├── py └── janitor │ ├── py.typed │ ├── site │ ├── templates │ │ ├── generic │ │ │ ├── summary.html │ │ │ ├── start.html │ │ │ ├── candidates.html │ │ │ ├── sidebar.html │ │ │ └── done.html │ │ ├── result-codes │ │ │ ├── unparseable-changelog.html │ │ │ ├── build-command-missing.html │ │ │ ├── build-failed-stage-build.html │ │ │ ├── native-package.html │ │ │ ├── quilt-refresh-error.html │ │ │ ├── upstream-merged-conflicts.html │ │ │ ├── autopkgtest-missing-node-module.html │ │ │ ├── dist-command-failed.html │ │ │ ├── dist-missing-file.html │ │ │ ├── dist-missing-automake-input.html │ │ │ ├── new-upstream-missing.html │ │ │ ├── command-failed.html │ │ │ ├── codemod-command-failed.html │ │ │ ├── install-deps-unsatisfied-dependencies.html │ │ │ ├── upstream-version-missing-in-upstream-branch.html │ │ │ ├── worker-failure.html │ │ │ ├── 502-bad-gateway.html │ │ │ ├── build-dh-addon-load-failure.html │ │ │ ├── build-missing-php-class.html │ │ │ ├── upstream-unsupported-vcs-hg.html │ │ │ ├── upstream-unsupported-vcs.html │ │ │ ├── uscan-error.html │ │ │ ├── build-upstart-file-present.html │ │ │ ├── upstream-unsupported-vcs-svn.html │ │ │ ├── watch-syntax-error.html │ │ │ ├── dist-apt-broken-packages.html │ │ │ ├── build-missing-python-module.html │ │ │ ├── control-files-in-root.html │ │ │ ├── package-in-subpath.html │ │ │ ├── timeout.html │ │ │ ├── no-upstream-locations-known.html │ │ │ ├── previous-upstream-missing.html │ │ │ ├── unpack-unexpected-local-upstream-changes.html │ │ │ ├── before-quilt-error.html │ │ │ ├── roundtripping-error.html │ │ │ ├── worker-timeout.html │ │ │ ├── unsupported-vcs-protocol.html │ │ │ ├── build-debhelper-pattern-not-found.html │ │ │ ├── build-missing-go-package.html │ │ │ ├── control-file-is-generated.html │ │ │ ├── invalid-path-normalization.html │ │ │ ├── missing-control-file.html │ │ │ ├── branch-unavailable.html │ │ │ ├── run-disappeared.html │ │ │ ├── upstream-branch-unavailable.html │ │ │ ├── 401-unauthorized.html │ │ │ ├── upstream-branch-unknown.html │ │ │ └── invalid-upstream-version-format.html │ │ ├── login.html │ │ ├── footer.html │ │ ├── faq-incorrect.html │ │ ├── faq-api.html │ │ ├── log-index.html │ │ ├── repo-list.html │ │ ├── cupboard │ │ │ ├── review-done.html │ │ │ ├── workers.html │ │ │ ├── changeset-list.html │ │ │ ├── reprocess-logs.html │ │ │ ├── rejected.html │ │ │ ├── ready-list.html │ │ │ ├── util.html │ │ │ ├── default-evaluate.html │ │ │ ├── publish.html │ │ │ ├── changeset.html │ │ │ ├── broken-merge-proposals.html │ │ │ ├── start.html │ │ │ ├── done-list.html │ │ │ └── never-processed.html │ │ ├── webhook.html │ │ ├── inputs.html │ │ ├── faq-out-of-date-proposal.html │ │ ├── index.html │ │ ├── faq-auto-push.html │ │ ├── ready-list.html │ │ ├── codeblock.html │ │ ├── faq-supported-vcs.html │ │ ├── lintian_util.html │ │ ├── about.html │ │ ├── layout.html │ │ └── credentials.html │ ├── _static │ │ ├── file.png │ │ ├── lintian.css │ │ ├── janitor.css │ │ └── janitor.js │ ├── cupboard │ │ ├── publish.py │ │ └── merge_proposals.py │ ├── pubsub.py │ └── merge_proposals.py │ ├── _runner.pyi │ ├── _common.pyi │ ├── _site.pyi │ ├── debian │ ├── debian.sql │ └── debdiff.py │ ├── _publish.pyi │ ├── config.py │ ├── artifacts.py │ ├── _launchpad.py │ ├── worker_creds.py │ ├── __init__.py │ └── review.py ├── pull_worker.sh ├── .github ├── CODEOWNERS ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── sql.yaml │ ├── lint.yaml │ └── rust.yml ├── site ├── src │ └── lib.rs └── Cargo.toml ├── TODO ├── .codespell-ignore-words ├── git-store ├── src │ ├── main.rs │ └── lib.rs └── Cargo.toml ├── archive ├── tests │ └── data │ │ ├── hello_2.10-3_amd64.deb │ │ ├── hello_2.10.orig.tar.gz │ │ ├── hello_2.10-3.debian.tar.xz │ │ ├── hello_2.10.orig.tar.gz.asc │ │ └── hello_2.10-3.dsc ├── Cargo.toml └── src │ └── lib.rs ├── .codespellrc ├── bzr-store ├── Cargo.toml └── src │ └── lib.rs ├── publish.sh ├── .containerignore ├── .testr.conf ├── .yamllint ├── TODO.rust ├── tox.ini ├── autopkgtest-wrapper ├── run_worker.sh ├── docs ├── structure.md ├── glossary.md ├── production.md └── flow.md ├── .dockerignore ├── worker ├── templates │ ├── log_index.html │ ├── artifact_index.html │ └── index.html ├── src │ ├── bin │ │ ├── debian-build.rs │ │ └── generic-build.rs │ └── tee.rs └── Cargo.toml ├── sieve ├── janitor.sieve └── README ├── auto-upload ├── Cargo.toml └── src │ └── lib.rs ├── AUTHORS ├── src ├── lib.rs ├── api │ ├── runner.rs │ └── mod.rs ├── prometheus.rs ├── logging.rs └── bin │ └── janitor-schedule.rs ├── .gitignore ├── runner-py ├── src │ └── lib.rs └── Cargo.toml ├── devnotes ├── glossary.rst ├── branch-names.rst └── adding-a-new-campaign.rst ├── site-py ├── Cargo.toml └── src │ └── lib.rs ├── mail-filter ├── src │ ├── tests.rs │ └── bin │ │ └── janitor-mail-filter.rs └── Cargo.toml ├── sbuildrc.example ├── common-py ├── src │ ├── io.rs │ ├── lib.rs │ └── debdiff.rs └── Cargo.toml ├── publish-py └── Cargo.toml ├── tests ├── test_launchpad.py ├── test_site_simple.py ├── conftest.py ├── test_core.py ├── __init__.py ├── test_site.py ├── test_queue.py ├── test_differ.py ├── test_artifacts.py ├── test_archive.py ├── test_logs.py ├── test_debdiff.py └── test_site_macros.py ├── Dockerfile_mail_filter ├── differ-py ├── Cargo.toml └── src │ └── lib.rs ├── Dockerfile_auto_upload ├── Dockerfile_differ ├── Dockerfile_publish ├── Dockerfile_archive ├── Dockerfile_bzr_store ├── Dockerfile_git_store ├── Dockerfile_runner ├── Dockerfile_site ├── differ ├── src │ └── lib.rs └── Cargo.toml ├── publish └── Cargo.toml ├── setup.py ├── Dockerfile_worker ├── runner ├── Cargo.toml └── src │ └── main.rs ├── helpers ├── cleanup-repositories.py └── render-publish-template.py ├── Dockerfile_ognibuild_dep ├── reprocess-build-results.py ├── Makefile └── reschedule.py /py/janitor/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pull_worker.sh: -------------------------------------------------------------------------------- 1 | run_worker.sh -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jelmer 2 | -------------------------------------------------------------------------------- /site/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod analyze; 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | --- 2 | github: jelmer 3 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - Split out generic code from debian-janitor specific bits 2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/generic/summary.html: -------------------------------------------------------------------------------- 1 | {{ command }} 2 | -------------------------------------------------------------------------------- /.codespell-ignore-words: -------------------------------------------------------------------------------- 1 | crate 2 | buildd 3 | fpr 4 | afile 5 | nd 6 | ser 7 | -------------------------------------------------------------------------------- /git-store/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("Hello, world!"); 3 | } 4 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/unparseable-changelog.html: -------------------------------------------------------------------------------- 1 |

The changelog could not be parsed.

2 | -------------------------------------------------------------------------------- /py/janitor/site/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jelmer/janitor/HEAD/py/janitor/site/_static/file.png -------------------------------------------------------------------------------- /archive/tests/data/hello_2.10-3_amd64.deb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jelmer/janitor/HEAD/archive/tests/data/hello_2.10-3_amd64.deb -------------------------------------------------------------------------------- /archive/tests/data/hello_2.10.orig.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jelmer/janitor/HEAD/archive/tests/data/hello_2.10.orig.tar.gz -------------------------------------------------------------------------------- /py/janitor/site/templates/login.html: -------------------------------------------------------------------------------- 1 | {% if openid_configured %}Login{% endif %} 2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-command-missing.html: -------------------------------------------------------------------------------- 1 |

A command was missing while trying to build the package.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-failed-stage-build.html: -------------------------------------------------------------------------------- 1 |

Building the package failed after changes were made to it.

2 | -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | skip = .git,.mypy_cache,build,testdata,target,htmlcov,Cargo.lock 3 | ignore-words = .codespell-ignore-words 4 | -------------------------------------------------------------------------------- /archive/tests/data/hello_2.10-3.debian.tar.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jelmer/janitor/HEAD/archive/tests/data/hello_2.10-3.debian.tar.xz -------------------------------------------------------------------------------- /py/janitor/_runner.pyi: -------------------------------------------------------------------------------- 1 | def committer_env(committer: str | None) -> dict[str, str]: ... 2 | def is_log_filename(filename: str) -> bool: ... 3 | -------------------------------------------------------------------------------- /bzr-store/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bzr-store" 3 | version = "0.1.0" 4 | edition.workspace = true 5 | 6 | [lib] 7 | 8 | [dependencies] 9 | -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PYTHONPATH="$PYTHONPATH:$(pwd)/lintian-brush:$(pwd)/silver-platter:$(pwd)/breezy" python3 -m janitor.publish "$@" 3 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/native-package.html: -------------------------------------------------------------------------------- 1 |

The package is native, which means there is no upstream source to be merged.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/quilt-refresh-error.html: -------------------------------------------------------------------------------- 1 |

Refreshing quilt patches after merging in a new upstream source failed.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/upstream-merged-conflicts.html: -------------------------------------------------------------------------------- 1 |

There were merge conflicts while merging a new upstream version.

2 | -------------------------------------------------------------------------------- /.containerignore: -------------------------------------------------------------------------------- 1 | .eggs 2 | .pytest_cache 3 | .mypy_cache 4 | __pycache__ 5 | lib 6 | lib64/ 7 | share/ 8 | bin/ 9 | man/ 10 | *~ 11 | .git/ 12 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/autopkgtest-missing-node-module.html: -------------------------------------------------------------------------------- 1 |

A node module could not be found while running autopkgtest tests.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/dist-command-failed.html: -------------------------------------------------------------------------------- 1 |

Creating a dist tarball from an upstream source repository snapshot failed.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/dist-missing-file.html: -------------------------------------------------------------------------------- 1 |

A file required by the upstream tarball creation process could not be found.

2 | -------------------------------------------------------------------------------- /git-store/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-git-store" 3 | version = "0.1.0" 4 | edition.workspace = true 5 | 6 | [lib] 7 | 8 | [dependencies] 9 | -------------------------------------------------------------------------------- /.testr.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_command=PYTHONPATH=`pwd`:$PYTHONPATH pytest --subunit tests 3 | test_id_option=--load-list $IDFILE 4 | test_list_option=--list 5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/dist-missing-automake-input.html: -------------------------------------------------------------------------------- 1 |

2 | One of the required input files could not be found while running automake. 3 |

4 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/new-upstream-missing.html: -------------------------------------------------------------------------------- 1 |

2 | There is configuration for finding an upstream source, but one can not be 3 | found. 4 |

5 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | # 80 chars should be enough, but don't fail if a line is longer 6 | line-length: 7 | max: 80 8 | level: warning 9 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/command-failed.html: -------------------------------------------------------------------------------- 1 |

The codemod failed to run for some reason, and didn't report why. This is almost always a bug in the codemod.

2 | -------------------------------------------------------------------------------- /TODO.rust: -------------------------------------------------------------------------------- 1 | * Convert queue to rust 2 | * Convert queue_processor to rust 3 | * Convert publish_one to rust 4 | * Convert LogFileManager to rust 5 | * Convert ArtifactManager to rust 6 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/codemod-command-failed.html: -------------------------------------------------------------------------------- 1 |

The codemod failed to run for some reason, and didn't report why. This is almost always a bug in the codemod.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/install-deps-unsatisfied-dependencies.html: -------------------------------------------------------------------------------- 1 |

Some of the dependencies of the source package failed to be satisfied during package build.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/upstream-version-missing-in-upstream-branch.html: -------------------------------------------------------------------------------- 1 |

The upstream version that is being merged does not exist in the upstream repository branch.

2 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/worker-failure.html: -------------------------------------------------------------------------------- 1 |

An undefined error occurred while processing the package in the worker.

2 |

This is usually a bug in the Janitor.

3 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | downloadcache = {toxworkdir}/cache/ 3 | envlist = py38, py39, py310, py311 4 | 5 | [testenv] 6 | commands = make check 7 | recreate = True 8 | whitelist_externals = make 9 | -------------------------------------------------------------------------------- /bzr-store/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Bzr Store crate for the Janitor project. 2 | //! 3 | //! This crate provides functionality for storing and managing Bazaar repositories. 4 | 5 | #![deny(missing_docs)] 6 | -------------------------------------------------------------------------------- /git-store/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Git Store crate for the Janitor project. 2 | //! 3 | //! This crate provides functionality for storing and managing Git repositories. 4 | 5 | #![deny(missing_docs)] 6 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/502-bad-gateway.html: -------------------------------------------------------------------------------- 1 |

2 | This error mostly occurs when Salsa is unreachable, usually during upgrades 3 | to a newer version of GitLab. 4 |

5 | -------------------------------------------------------------------------------- /autopkgtest-wrapper: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | autopkgtest "$@" 3 | # autopkgtest(1) mentions that 2 indicates a skipped test. Ignore those: 4 | aptexit=$(($?&~2)) 5 | echo "Exiting with $aptexit" 6 | exit $aptexit 7 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-dh-addon-load-failure.html: -------------------------------------------------------------------------------- 1 |

2 | A dh addon failed to load, possibly due to other dh options not being set or 3 | missing Build-Depends. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-missing-php-class.html: -------------------------------------------------------------------------------- 1 |

2 | The specified PHP class was missing, and there is no package in the archive 3 | that is known to provide it. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/upstream-unsupported-vcs-hg.html: -------------------------------------------------------------------------------- 1 |

2 | The upstream source repository is maintained in Mercurial, which is not 3 | currently supported by Breezy. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/upstream-unsupported-vcs.html: -------------------------------------------------------------------------------- 1 |

2 | The upstream package repository location is known, but the version control 3 | system it uses is unsupported. 4 |

5 | -------------------------------------------------------------------------------- /run_worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | WD=$(realpath $(dirname $0)) 4 | 5 | export SBUILD_CONFIG=${SBUILD_CONFIG:-$WD/sbuildrc} 6 | export AUTOPKGTEST=$WD/autopkgtest-wrapper 7 | 8 | janitor-worker --tee "$@" 9 | -------------------------------------------------------------------------------- /py/janitor/_common.pyi: -------------------------------------------------------------------------------- 1 | from breezy.branch import Branch 2 | 3 | def is_alioth_url(url: str) -> bool: ... 4 | def is_authenticated_url(url: str) -> bool: ... 5 | def get_branch_vcs_type(branch: Branch) -> str: ... 6 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/uscan-error.html: -------------------------------------------------------------------------------- 1 |

Running uscan to download the upstream source tarball failed.

2 |

3 | This is usually a problem with the debian/watch file. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-upstart-file-present.html: -------------------------------------------------------------------------------- 1 |

2 | The package failed to build because newer versions of debhelper no longer 3 | support upstart service files and one was present. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/upstream-unsupported-vcs-svn.html: -------------------------------------------------------------------------------- 1 |

2 | The upstream repository is stored in a Subversion repository. Breezy does 3 | not currently support Subversion repositories. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/watch-syntax-error.html: -------------------------------------------------------------------------------- 1 |

2 | The debian/watch file contains syntax that debmutate does not understand. 3 |

4 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/dist-apt-broken-packages.html: -------------------------------------------------------------------------------- 1 |

2 | While creating the .orig.tar.gz from a snapshot of the upstream 3 | repository, some apt dependencies could not be satisfied. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-missing-python-module.html: -------------------------------------------------------------------------------- 1 |

2 | The package failed to build because it depends on a Python module that is 3 | not available in the APT repository for the distribution. 4 |

5 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/control-files-in-root.html: -------------------------------------------------------------------------------- 1 |

The control files are in the root of the packaging branch.

2 |

At the moment, the Janitor can not build such packages due to limitations in Breezy-Debian.

3 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/package-in-subpath.html: -------------------------------------------------------------------------------- 1 |

This package is not stored at the root of a VCS repository.

2 |

3 | Tracking bug 4 |

5 | -------------------------------------------------------------------------------- /docs/structure.md: -------------------------------------------------------------------------------- 1 | ## Structure 2 | 3 | - `./reschedule.py` - a tool for users of the janitor and can be run by anybody locally 4 | - `./helpers/*` - all need to run inside of a janitor deployment (and talk to the database, etc) by an admin. 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | credentials.* 2 | *.secret 3 | .mypy_cache 4 | .pytest_cache 5 | .ruff_cache 6 | .git 7 | .bzr 8 | .eggs 9 | *~ 10 | target/ 11 | k8s 12 | *.conf 13 | bin/ 14 | htmlcov/ 15 | tests/ 16 | build/ 17 | lib/ 18 | include/ 19 | .venv 20 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/timeout.html: -------------------------------------------------------------------------------- 1 |

2 | The build of the package timed out. This often means that it is stuck in an 3 | endless loop or did not generate output to standard out for a long time 4 | (usually an hour). 5 |

6 | -------------------------------------------------------------------------------- /py/janitor/_site.pyi: -------------------------------------------------------------------------------- 1 | def find_dist_log_failure( 2 | logf: str, length: int 3 | ) -> tuple[int, tuple[int, int], list[int] | None]: ... 4 | def find_build_log_failure( 5 | logf: str, length: int 6 | ) -> tuple[int, tuple[int, int], list[int] | None]: ... 7 | -------------------------------------------------------------------------------- /worker/templates/log_index.html: -------------------------------------------------------------------------------- 1 | 2 | Log Index 3 | 4 |

Logs

5 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /sieve/janitor.sieve: -------------------------------------------------------------------------------- 1 | require ["vnd.dovecot.execute", "envelope"]; 2 | 3 | # Hand off all e-mails to the janitor from salsa. 4 | if allof(header :contains "To" "janitor@jelmer.uk", 5 | envelope "from" "gitlab@salsa.debian.org") { 6 | execute :pipe "janitor-mail-filter"; 7 | } 8 | -------------------------------------------------------------------------------- /worker/templates/artifact_index.html: -------------------------------------------------------------------------------- 1 | 2 | Artifact Index 3 | 4 |

Artifacts

5 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /py/janitor/site/templates/footer.html: -------------------------------------------------------------------------------- 1 |
2 | © 2018-2023 Jelmer Vernooij. File a bug. Source code. Published under the GNU GPLv2 or later. 3 |
4 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/no-upstream-locations-known.html: -------------------------------------------------------------------------------- 1 |

No location for the upstream source code is known.

2 |

3 | Upstream locations can either be specified in debian/watch or in the 4 | Repository field in debian/upstream/metadata. 5 |

6 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/previous-upstream-missing.html: -------------------------------------------------------------------------------- 1 |

2 | The previous upstream version merged into the package could not be found in 3 | the packaging branch. 4 |

5 |

6 | Usually this version is tagged with a tag starting with upstream/. 7 |

8 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/unpack-unexpected-local-upstream-changes.html: -------------------------------------------------------------------------------- 1 |

2 | There are changes between the upstream tarball and the non-debian/ part of 3 | the packaging repository that are not accounted for by any of the patches 4 | under debian/patches. 5 |

6 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/before-quilt-error.html: -------------------------------------------------------------------------------- 1 |

Applying the quilt patches in this package before making any changes failed.

2 |

3 | This can mean that the patches are out of date and need to be updated so 4 | they apply against the upstream source in the packaging branch. 5 |

6 | -------------------------------------------------------------------------------- /archive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-archive" 3 | version.workspace = true 4 | edition.workspace = true 5 | 6 | [dependencies] 7 | futures = "0.3.31" 8 | tokio = { workspace = true, features = ["full"] } 9 | tracing = "0.1.41" 10 | deb822-lossless.workspace = true 11 | debian-control.workspace = true 12 | -------------------------------------------------------------------------------- /auto-upload/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-auto-upload" 3 | version.workspace = true 4 | edition.workspace = true 5 | 6 | [dependencies] 7 | futures = "0.3.31" 8 | tokio = { workspace = true, features = ["full"] } 9 | tracing = "0.1.41" 10 | silver-platter = { workspace = true, features = ["debian"] } 11 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/roundtripping-error.html: -------------------------------------------------------------------------------- 1 |

2 | This package is maintained in Git, but the upstream branch uses Bazaar. Breezy does not currently support these merges. 3 |

4 |

5 | See this bug on salsa for details. 6 |

7 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/worker-timeout.html: -------------------------------------------------------------------------------- 1 |

The worker failed to respond to pings, and the run was therefore marked as lost.

2 |

3 | In some cases, this can happen because the runner is rejecting the uploads 4 | from the worker - e.g. because it consistently crashes while processing them. 5 |

6 | -------------------------------------------------------------------------------- /py/janitor/site/templates/generic/start.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include [suite + "/sidebar.html", "generic/sidebar.html"] %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | {{ suite }} 7 | {% endblock page_title %} 8 | {% block body %} 9 | TODO 10 | {% endblock body %} 11 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/unsupported-vcs-protocol.html: -------------------------------------------------------------------------------- 1 |

2 | The packaging branch is using a version control system that is not supported 3 | by the Janitor. At the moment, only Git and 4 | Bazaar repositories are supported. 5 |

6 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-debhelper-pattern-not-found.html: -------------------------------------------------------------------------------- 1 |

2 | A debhelper helper failed to run because it was unable to expand some of the 3 | patterns provided to it. 4 |

5 |

6 | This is often the result of an upgrade to debhelper 11 or later, which 7 | enforces that patterns can be expanded. 8 |

9 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/build-missing-go-package.html: -------------------------------------------------------------------------------- 1 |

The package needs a Go package to be installed in order to build.

2 |

3 | However, the Go package is not available in the APT repository for 4 | the distribution and can thus not be added to the build dependencies 5 | for the package that's being built. 6 |

7 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/control-file-is-generated.html: -------------------------------------------------------------------------------- 1 |

One or more control files in the packaging branch are generated from another file.

2 |

3 | The Janitor identifies generated files by looking for the string 4 | 

DO NOT EDIT
in control files, and the existence of files with the 5 | .in extension. 6 |

7 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/invalid-path-normalization.html: -------------------------------------------------------------------------------- 1 |

The package contains a path that isn't unicode normalized.

2 |

3 | Breezy currently prohibits this, because it results in issues checking out 4 | files on Mac OS X. See this bug 5 | about loosening the constraints. 6 |

7 | -------------------------------------------------------------------------------- /docs/glossary.md: -------------------------------------------------------------------------------- 1 | * *codebase*: A collection of source code files that are managed together in a 2 | version control system. Usually this will be the root of a specific branch in a 3 | vcs repository. Sometimes, it will be a subdirectory in a VCS. It can also be 4 | e.g. a tarball somewhere. 5 | 6 | * *cotenants*: Other codebases that share the same branch as the current codebase. 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Jelmer Vernooij 2 | 3 | There are a lot of people who contributed ideas and feedback to the Janitor. 4 | Some of them are listed here; if you're missing, please let me know! 5 | 6 | Thanks: 7 | 8 | Perry Lorrier 9 | Christoph Berg 10 | Raphael Hertzog 11 | Gregor Herrman 12 | Holger Levsen 13 | Helmut Grohne 14 | Mattia Rizzolo 15 | Colin Watson 16 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod analyze_log; 2 | pub mod api; 3 | pub mod artifacts; 4 | pub mod config; 5 | pub mod debdiff; 6 | pub mod logging; 7 | pub mod logs; 8 | pub mod prometheus; 9 | pub mod publish; 10 | pub mod queue; 11 | pub mod reprocess_logs; 12 | pub mod schedule; 13 | pub mod schema; 14 | pub mod state; 15 | pub mod vcs; 16 | 17 | /// The type of a run ID. 18 | pub type RunId = String; 19 | -------------------------------------------------------------------------------- /py/janitor/site/templates/faq-incorrect.html: -------------------------------------------------------------------------------- 1 |
3 |

4 | The bot is proposing an incorrect change. Where do I report this? 5 |

6 |

For issues with a fix that the bot has proposed, please just follow up on the merge proposal.

7 |
8 | -------------------------------------------------------------------------------- /py/janitor/site/templates/faq-api.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | How can I automate interactions with the Janitor? 4 |

5 |

6 | There is an JSON API that can be used to retrieve information about packages, to reschedule runs and trigger creation of merge proposals. 7 |

8 |
9 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/missing-control-file.html: -------------------------------------------------------------------------------- 1 |

2 | The packaging branch 3 | {% if run %}({{ run.branch_url }}){% endif %} 4 | is accessible, but does not contain 5 | a debian/control file. 6 |

7 |

8 | This may be because the Vcs-* 9 | URL points at a branch that contains the upstream sources, rather than the Debian packaging. 10 |

11 | -------------------------------------------------------------------------------- /py/janitor/site/templates/log-index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Log file index 5 | 6 | 7 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /auto-upload/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Auto-upload crate for the Janitor project. 2 | //! 3 | //! This crate provides functionality for automatically uploading Debian packages. 4 | 5 | #![deny(missing_docs)] 6 | 7 | /// Re-export for signing Debian packages 8 | pub use silver_platter::debian::uploader::debsign; 9 | 10 | /// Re-export for uploading Debian changes files 11 | pub use silver_platter::debian::uploader::dput_changes; 12 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/branch-unavailable.html: -------------------------------------------------------------------------------- 1 |

2 | The packaging branch 3 | {% if vcs_url %}at ({{ vcs_url }}){% endif %} 4 | can not be 5 | reached. 6 |

7 |

This problem may be intermittent, e.g. if the hosting site is temporarily down.

8 |

9 | If the repository has moved, please update the Vcs-* headers 10 | in the package and upload a new version. 11 |

12 | -------------------------------------------------------------------------------- /src/api/runner.rs: -------------------------------------------------------------------------------- 1 | /// Sent when the publish-status for a run changes. 2 | #[derive(Debug, serde::Deserialize, serde::Serialize, Clone)] 3 | struct PublishStatusPubsub { 4 | /// The codebase. 5 | codebase: String, 6 | 7 | /// The run ID. 8 | run_id: crate::RunId, 9 | 10 | /// The new publish-status. 11 | #[serde(rename = "publish-status")] 12 | publish_status: crate::api::RunPublishStatus, 13 | } 14 | -------------------------------------------------------------------------------- /site/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-site" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Basic site for the janitor" 7 | license = "GPL-3.0+" 8 | repository = "https://github.com/jelmer/janitor.git" 9 | homepage = "https://github.com/jelmer/janitor" 10 | 11 | [dependencies] 12 | buildlog-consultant = { version = "0.1.0", default-features = false } 13 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/run-disappeared.html: -------------------------------------------------------------------------------- 1 |

2 | The management plane for the worker no longer knows about the 3 | job that was processing the run. This can happen because e.g. 4 | the management plane for the worker was restarted, or because the worker 5 | failed to upload its results to the runner, possibly because the 6 | runner was down or rejected the uploads - it could be crashing 7 | while processing them. 8 |

9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | *.swp 3 | build/ 4 | __pycache__ 5 | .plugins 6 | .pybuild 7 | _build/* 8 | *.pyc 9 | *.rej 10 | *.orig 11 | *~ 12 | site/_build 13 | state.db 14 | site/history.rst 15 | build.log 16 | debian_janitor.egg-info/ 17 | .mypy_cache/ 18 | .venv 19 | .eggs 20 | janitor.egg-info 21 | pyvenv.cfg 22 | lib 23 | lib64 24 | /bin 25 | man 26 | share 27 | .hypothesis 28 | .tox 29 | Cargo.lock 30 | target 31 | .testrepository 32 | py/janitor/_*.cpython-*.so 33 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/upstream-branch-unavailable.html: -------------------------------------------------------------------------------- 1 |

The upstream branch could not be found; it may have been moved.

2 |

3 | The upstream branch will be taken from the Repository field in 4 | debian/upstream/metadata or guessed based on metadata in the source 5 | package. 6 |

7 |

8 | To fix this error, set the Repository field appropriately in 9 | debian/upstream/metadata. 10 |

11 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/401-unauthorized.html: -------------------------------------------------------------------------------- 1 |

2 | Attempting to access the packaging branch resulted in a HTTP 3 | 401 Unauthorized error. This can indicate that the repository is 4 | private, that accessing it may require log in, or that it simply does not 5 | exist. 6 |

7 |

8 | This can happen due to Breezy not sending credentials to GitLab 9 | repositories. See 10 | this bug 11 | for details. 12 |

13 | -------------------------------------------------------------------------------- /py/janitor/site/templates/repo-list.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block page_title %} 3 | Repository List - {{ vcs }} 4 | {% endblock page_title %} 5 | {% block body %} 6 |
7 |

Repository List - {{ vcs }}

8 |
    9 | {% for repo in repositories %} 10 |
  • 11 | {{ repo }} 12 |
    • 13 | {% endfor %} 14 |
15 |
16 | {% endblock body %} 17 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Please see the documentation for all configuration options: 2 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 3 | --- 4 | version: 2 5 | updates: 6 | - package-ecosystem: "cargo" 7 | directory: "/" 8 | schedule: 9 | interval: "weekly" 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: weekly 14 | - package-ecosystem: "pip" 15 | directory: "/" 16 | schedule: 17 | interval: "weekly" 18 | -------------------------------------------------------------------------------- /docs/production.md: -------------------------------------------------------------------------------- 1 | # Running Janitor in production 2 | 3 | There are [containers](Dockerfiles_.md) available for each of the Janitor services. 4 | 5 | [pre-built containers](https://github.com/jelmer?tab=packages&repo_name=janitor) are 6 | available, but you can also create them yourself: 7 | 8 | ```console 9 | $ sudo apt install \ 10 | buildah \ 11 | make 12 | $ make build-all 13 | ``` 14 | 15 | For a Janitor instance, you probably want a custom website in combination with 16 | the Janitor API. See the existing instances for inspiration. 17 | -------------------------------------------------------------------------------- /py/janitor/debian/debian.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION IF NOT EXISTS debversion; 2 | CREATE TABLE debian_build ( 3 | run_id text not null references run (id), 4 | -- Debian version text of the built package 5 | version debversion not null, 6 | -- Distribution the package was built for (e.g. "lintian-fixes") 7 | distribution text not null, 8 | source text not null, 9 | binary_packages text[], 10 | lintian_result json 11 | ); 12 | CREATE INDEX ON debian_build (run_id); 13 | CREATE INDEX ON debian_build (distribution, source, version); 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /archive/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Archive crate for the Janitor project. 2 | //! 3 | //! This crate provides functionality for working with package archives. 4 | 5 | #![deny(missing_docs)] 6 | 7 | use tracing::{debug, error, info}; 8 | 9 | /// Temporary prefix used for archive operations. 10 | pub const TMP_PREFIX: &str = "janitor-apt"; 11 | /// Default timeout for Google Cloud Storage operations in seconds. 12 | pub const DEFAULT_GCS_TIMEOUT: usize = 60 * 30; 13 | 14 | /// Scanner module for archive operations. 15 | pub mod scanner; 16 | 17 | // TODO(jelmer): Generate contents file 18 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/upstream-branch-unknown.html: -------------------------------------------------------------------------------- 1 |

2 | The location of the upstream repository of this package is unknown. The canonical location 3 | for this information is the Repository field in the debian/upstream/metadata file. 4 |

5 |

6 | The Janitor will also fall back to attempting to reading various other metadata files 7 | to figure out the upstream repository location, such as dist.ini (for perl packages) 8 | or setup.py (for Python packages). 9 |

10 | -------------------------------------------------------------------------------- /runner-py/src/lib.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | use std::collections::HashMap; 3 | 4 | #[pyfunction] 5 | #[pyo3(signature = (committer=None))] 6 | fn committer_env(committer: Option<&str>) -> HashMap { 7 | janitor_runner::committer_env(committer) 8 | } 9 | 10 | #[pyfunction] 11 | fn is_log_filename(filename: &str) -> bool { 12 | janitor_runner::is_log_filename(filename) 13 | } 14 | 15 | #[pymodule] 16 | fn _runner(m: &Bound) -> PyResult<()> { 17 | m.add_function(wrap_pyfunction!(committer_env, m)?)?; 18 | m.add_function(wrap_pyfunction!(is_log_filename, m)?)?; 19 | Ok(()) 20 | } 21 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/review-done.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard - Reviews Completed 7 | {% endblock page_title %} 8 | {% block body %} 9 |
10 |

Review

11 |

All done!

12 | {% if publishable_only %} 13 |

14 | Review unpublishable 15 |

16 | {% endif %} 17 |
18 | {% endblock body %} 19 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/workers.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard - Workers 7 | {% endblock page_title %} 8 | {% block body %} 9 |
10 |

Workers

11 | {% for worker in workers %} 12 |
13 |

{{ worker.name }}

14 |

Total runs: {{ worker.run_count }}

15 |
16 | {% endfor %} 17 |
18 | {% endblock body %} 19 | -------------------------------------------------------------------------------- /py/janitor/site/templates/webhook.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Janitor - Webhook API 6 | 8 | 9 | 10 |

This URL can be used as an endpoint for webhooks in GitLab and GitHub.

11 |

Sending updates for repositories here will cause the Janitor to trigger rescheduling of the affected runs.

12 | 13 | 14 | -------------------------------------------------------------------------------- /devnotes/glossary.rst: -------------------------------------------------------------------------------- 1 | Campaign 2 | ######## 3 | 4 | An effort to fix a particular thing in the set of packages the janitor instance 5 | is responsible for. 6 | 7 | E.g. lintian-fixes', 'fresh-upstreams', 'fresh-releases'. 8 | 9 | Codebase 10 | ######## 11 | 12 | Typically, a VCS tree of some sort. Usually identified by a branch in a 13 | particular repository. 14 | 15 | Candidate 16 | ######### 17 | 18 | A suite + codebase that has been identified as potentially being improveable. 19 | 20 | Target 21 | ###### 22 | 23 | Either "debian" or "upstream". The kind of environment to target. 24 | This determines how the resulting codebase will be built. 25 | -------------------------------------------------------------------------------- /py/janitor/site/templates/inputs.html: -------------------------------------------------------------------------------- 1 | {% macro codebase_input(name) %} 2 | 7 | 21 | {% endmacro %} 22 | -------------------------------------------------------------------------------- /site-py/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "site-py" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Site for the janitor - python bindings" 7 | publish = false 8 | license = "GPL-3.0+" 9 | repository = "https://github.com/jelmer/janitor.git" 10 | homepage = "https://github.com/jelmer/janitor" 11 | 12 | [lib] 13 | crate-type = ["cdylib"] 14 | 15 | [dependencies] 16 | pyo3 = {workspace = true, features=["serde", "chrono"]} 17 | janitor-site = { path = "../site" } 18 | pyo3-log = { workspace = true } 19 | log = "0.4" 20 | 21 | [features] 22 | extension-module = ["pyo3/extension-module"] 23 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/changeset-list.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard - Changeset List 7 | {% endblock page_title %} 8 | {% block body %} 9 |
10 |

Changesets

11 | 18 |
19 | {% endblock body %} 20 | -------------------------------------------------------------------------------- /py/janitor/site/templates/generic/candidates.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include [suite + "/sidebar.html", "generic/sidebar.html"] %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | {{ suite }} - Candidates 7 | {% endblock page_title %} 8 | {% block body %} 9 |
10 |

{{ suite }} - Candidates

11 |
    12 | {% for codebase, value in candidates %} 13 |
  • 14 | {{ codebase }} 15 |
    • 16 | {% endfor %} 17 |
18 |
19 | {% endblock body %} 20 | -------------------------------------------------------------------------------- /mail-filter/src/tests.rs: -------------------------------------------------------------------------------- 1 | use super::parse_email; 2 | 3 | #[test] 4 | fn test_parse_github_merged_email() { 5 | let email = include_bytes!("../tests/data/github-merged-email.txt"); 6 | 7 | assert_eq!( 8 | Some("https://github.com/UbuntuBudgie/budgie-desktop/pull/78"), 9 | parse_email(std::io::Cursor::new(email)).as_deref() 10 | ); 11 | } 12 | 13 | #[test] 14 | fn test_parse_gitlab_merged_email() { 15 | let email = include_bytes!("../tests/data/gitlab-merged-email.txt"); 16 | 17 | assert_eq!( 18 | Some("https://salsa.debian.org/debian/pkg-lojban-common/-/merge_requests/2"), 19 | parse_email(std::io::Cursor::new(email)).as_deref() 20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/reprocess-logs.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard - Reprocess Logs 7 | {% endblock page_title %} 8 | {% block body %} 9 |

Reprocess Logs

10 |
13 | Dry Run: 14 | 15 | Reschedule: 16 | 17 | 18 |
19 | {% endblock body %} 20 | -------------------------------------------------------------------------------- /src/api/mod.rs: -------------------------------------------------------------------------------- 1 | //! This module contains the API definitions for the various janitor components. 2 | pub mod runner; 3 | pub mod worker; 4 | 5 | use serde::{Deserialize, Serialize}; 6 | 7 | /// The publish status of a run. 8 | #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] 9 | pub enum RunPublishStatus { 10 | #[serde(rename = "unknown")] 11 | Unknown, 12 | 13 | #[serde(rename = "blocked")] 14 | Blocked, 15 | 16 | #[serde(rename = "needs-manual-review")] 17 | NeedsManualReview, 18 | 19 | #[serde(rename = "rejected")] 20 | Rejected, 21 | 22 | #[serde(rename = "approved")] 23 | Approved, 24 | 25 | #[serde(rename = "ignored")] 26 | Ignored, 27 | } 28 | -------------------------------------------------------------------------------- /sbuildrc.example: -------------------------------------------------------------------------------- 1 | $build_arch_all = 1; 2 | $apt_update = 1; 3 | $apt_distupgrade = 1; 4 | $run_lintian = 1; 5 | $run_piuparts = 0; 6 | $piuparts_require_success = 0; 7 | $piuparts_opts = ['--schroot', '%r-%a-sbuild' ]; 8 | $piuparts_root_args = ''; 9 | $run_autopkgtest = 1; 10 | $autopkgtest_require_success = 1; 11 | $autopkgtest_root_args = ''; 12 | $autopkgtest_opts = [ '--no-auto-control', '--', 'schroot', '%r-%a-sbuild' ]; 13 | $lintian_opts = ['--suppress-tags', 'bad-distribution-in-changes-file,no-nmu-in-changelog,source-nmu-has-incorrect-version-number']; 14 | $aspcud_criteria = '-removed,-changed,-new,-count(solution,APT-Release:=/o=Janitor/)'; 15 | $build_dep_resolver = 'aspcud'; 16 | $clean_source = 0; 17 | $verbose = 1; 18 | -------------------------------------------------------------------------------- /runner-py/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "runner-py" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | publish = false 6 | edition.workspace = true 7 | description = "Runner for the janitor - python bindings" 8 | license = "GPL-3.0+" 9 | repository = "https://github.com/jelmer/janitor.git" 10 | homepage = "https://github.com/jelmer/janitor" 11 | 12 | [lib] 13 | crate-type = ["cdylib"] 14 | 15 | [dependencies] 16 | pyo3 = {workspace = true, features=["serde", "chrono"]} 17 | janitor-runner = { path = "../runner" } 18 | pyo3-log = { workspace = true } 19 | breezyshim.workspace = true 20 | silver-platter = { workspace = true, features = ["debian"] } 21 | 22 | [features] 23 | extension-module = ["pyo3/extension-module"] 24 | -------------------------------------------------------------------------------- /sieve/README: -------------------------------------------------------------------------------- 1 | This directory contains a basic script to process e-mails from GitLab and use 2 | them to trigger immediate refreshing of the status of merge proposals. 3 | 4 | Dovecot 5 | ======= 6 | 7 | To use this script with dovecot's sieve filtering, enable the extprograms sieve plugin. 8 | 9 | To configure the plugin, set something like: 10 | 11 | plugin { 12 | sieve_plugins = sieve_extprograms 13 | sieve_extensions = +vnd.dovecot.execute +editheader 14 | sieve_execute_bin_dir = /usr/lib/dovecot/sieve-pipe 15 | sieve_global_dir = /var/lib/dovecot/sieve/global 16 | } 17 | 18 | The filter script should be placed in $sieve_execute_bin_dir. 19 | 20 | You can then include the sieve filter ("janitor.sieve") in this directory to 21 | trigger runs. 22 | -------------------------------------------------------------------------------- /common-py/src/io.rs: -------------------------------------------------------------------------------- 1 | use pyo3::exceptions::PyRuntimeError; 2 | use pyo3::prelude::*; 3 | use pyo3::types::PyBytes; 4 | use std::io::Read; 5 | 6 | #[pyclass] 7 | pub(crate) struct Readable(Box); 8 | 9 | impl Readable { 10 | pub fn new(read: Box) -> Self { 11 | Self(read) 12 | } 13 | } 14 | 15 | #[pymethods] 16 | impl Readable { 17 | #[pyo3(signature = (size=None))] 18 | fn read(&mut self, py: Python, size: Option) -> PyResult { 19 | let mut buf = vec![0; size.unwrap_or(4096)]; 20 | let n = self 21 | .0 22 | .read(&mut buf) 23 | .map_err(|e| PyRuntimeError::new_err(e))?; 24 | buf.truncate(n); 25 | Ok(PyBytes::new_bound(py, &buf).into()) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /mail-filter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-mail-filter" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Mail filter for the janitor" 7 | license = "Apache-2.0" 8 | repository = "https://github.com/jelmer/janitor.git" 9 | homepage = "https://github.com/jelmer/janitor" 10 | 11 | [dependencies] 12 | isahc = "1" 13 | serde_json = "1" 14 | select = "0.6" 15 | mailparse = "0.16" 16 | async-std = "1" 17 | log = "0.4" 18 | clap = { workspace = true, optional = true, features = ["derive"] } 19 | reqwest = { version = "0.12", features = ["blocking", "json"], optional = true } 20 | 21 | [[bin]] 22 | name="janitor-mail-filter" 23 | required-features=["cmdline"] 24 | 25 | [features] 26 | default = ["cmdline"] 27 | cmdline = ["dep:clap", "dep:reqwest"] 28 | -------------------------------------------------------------------------------- /publish-py/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "publish-py" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Publisher for the janitor - python bindings" 7 | publish = false 8 | license = "GPL-3.0+" 9 | repository = "https://github.com/jelmer/janitor.git" 10 | homepage = "https://github.com/jelmer/janitor" 11 | 12 | [lib] 13 | crate-type = ["cdylib"] 14 | 15 | [dependencies] 16 | pyo3 = {workspace = true, features=["serde", "chrono"]} 17 | janitor-publish = { path = "../publish" } 18 | pyo3-log = { workspace = true } 19 | log = "0.4" 20 | chrono = { workspace = true, features = ["serde"] } 21 | breezyshim.workspace = true 22 | silver-platter = { workspace = true, features = ["debian"] } 23 | url.workspace = true 24 | 25 | [features] 26 | extension-module = ["pyo3/extension-module"] 27 | -------------------------------------------------------------------------------- /py/janitor/site/templates/faq-out-of-date-proposal.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | What do I do with out-of-date merge proposals? 4 |

5 |

6 | The Janitor will automatically reschedule processing of packages with a 7 | conflicted merge proposal. Once a conflict appears, it may take a couple of 8 | hours before the merge proposal is updated. 9 |

10 |

11 | It will also regularly rebase merge proposals on the packaging branch. It 12 | can take several days before this happens, since there is no mechanism to 13 | notify the Janitor of new commits. You can manually trigger a rerun 14 | from the package-specific page linked from the merge proposal. 15 |

16 |
17 | -------------------------------------------------------------------------------- /site-py/src/lib.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | #[pyfunction] 4 | fn find_dist_log_failure(logf: &str, length: usize) -> (usize, (usize, usize), Option>) { 5 | let r = janitor_site::analyze::find_dist_log_failure(logf, length); 6 | (r.total_lines, r.include_lines, r.highlight_lines) 7 | } 8 | 9 | #[pyfunction] 10 | fn find_build_log_failure( 11 | logf: &[u8], 12 | length: usize, 13 | ) -> (usize, (usize, usize), Option>) { 14 | let r = janitor_site::analyze::find_build_log_failure(logf, length); 15 | (r.total_lines, r.include_lines, r.highlight_lines) 16 | } 17 | 18 | #[pymodule] 19 | fn _site(_py: Python, m: &Bound) -> PyResult<()> { 20 | pyo3_log::init(); 21 | m.add_function(wrap_pyfunction!(find_dist_log_failure, m)?)?; 22 | m.add_function(wrap_pyfunction!(find_build_log_failure, m)?)?; 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /py/janitor/site/templates/index.html: -------------------------------------------------------------------------------- 1 | {% set layout = "plain" %} 2 | {% extends "layout.html" %} 3 | {% block page_title %} 4 | Index 5 | {% endblock page_title %} 6 | {% block body %} 7 |
8 |

Janitor Instance

9 |

10 | The Janitor is a project to automatically make changes to Debian 11 | packages. 12 |

13 |

This is a minimal website for a Janitor instance.

14 |

Campaigns

15 | 22 |
23 |

24 | For Janitor internal status details, see the cupboard. 25 |

26 | {% endblock body %} 27 | -------------------------------------------------------------------------------- /py/janitor/site/templates/faq-auto-push.html: -------------------------------------------------------------------------------- 1 |
3 |

4 | This is great. How do I get it to automatically push improvements to my repository? 5 |

6 |

Simply give the bot commit access to your repository, and it will push fixes rather than proposing them.

7 |

8 | The bot will need permission to push to the relevant branches. For 9 | repositories on a GitLab instances (such as salsa) this means that 10 | it will need developer permissions if the relevant branch is unprotected, 11 | and maintainer permissions if the relevant branch is protected. 12 | See the GitLab permissions guide for details. 13 |

14 |
15 | -------------------------------------------------------------------------------- /tests/test_launchpad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | 19 | import janitor._launchpad # noqa: F401 20 | -------------------------------------------------------------------------------- /archive/tests/data/hello_2.10.orig.tar.gz.asc: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP SIGNATURE----- 2 | Version: GnuPG v2 3 | 4 | iQIcBAABCAAGBQJUaJPFAAoJEKlVMkX96bc5EuMP/2Z8T2r+ZjJbveEfVKuY3LbG 5 | sPqZqI/t0WISsfhRen3R/tiis0lN8TWIdTFRnLqtxyqfzDDtgzrPg8gwICFYxE6W 6 | ffVvgbbDA14EuatWHfnAK1SjWsaemJIO1rGROgNFqsatEDIOf7bg4NZ6Gs1QR0rJ 7 | p4W+LYKiP8UeJwV4Xd2d/h+rf4XBWo5HTNYwgZpZawklWupmIx0bXi3HiRs4MJQm 8 | mfbNrNE5YcAWQpBwAxgcUCwGHlvDonpvu0i0D4tNoMeneLAhZty1GCnamTlcuDXJ 9 | IZpg0Ky9mYEnyRhaRnLsyaZ2kzJhOSMNfVzSP2+ge+JfTuenw1yvhAZC9qTLoV+f 10 | 1xUhxUkmzgDV3pVpc9qB+LVGfJclrHtrgD2dakmph5JGGhAoAExwrkyO3qxE5jzJ 11 | x2C83aNpBjNqPhAVIcywXpFWBT8sbsXgwLufXWFwQtyxIm1dxrOku0SI5oYm1ZON 12 | l1rjkaQmpFKx1oo7eOG1XLbCQ1Ii1qEDSiXTvQwoaBTkAcPz1KOVGEyby6kf9AS9 13 | DjuJzh8oQgylaDk5FqGqsY6S90Naz7SJSrBi/3xOP51LvsLciNx+EPBNrRJzFDYw 14 | svn/ahaWx4hsXr0ErjqAzqE6ZNQQcyKa5qDDFD5dz14dSI78FjZ4u2WUwZCGyW+u 15 | OFOwPF0lXuPO6q2UFcpj 16 | =s6wC 17 | -----END PGP SIGNATURE----- 18 | -------------------------------------------------------------------------------- /Dockerfile_mail_filter: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ca-certificates \ 14 | cargo \ 15 | libpython3-dev \ 16 | libssl-dev \ 17 | pkg-config \ 18 | protobuf-compiler \ 19 | python3-minimal \ 20 | && apt-get clean 21 | 22 | COPY . /code 23 | 24 | RUN cargo build --release --manifest-path /code/mail-filter/Cargo.toml 25 | 26 | FROM docker.io/debian:testing-slim 27 | 28 | COPY --from=build /code/target/release/janitor-mail-filter /usr/local/bin/janitor-mail-filter 29 | 30 | ENTRYPOINT ["janitor-mail-filter"] 31 | -------------------------------------------------------------------------------- /differ-py/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "differ-py" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Differ for the janitor - python bindings" 7 | publish = false 8 | license = "GPL-3.0+" 9 | repository = "https://github.com/jelmer/janitor.git" 10 | homepage = "https://github.com/jelmer/janitor" 11 | 12 | [lib] 13 | crate-type = ["cdylib"] 14 | 15 | [dependencies] 16 | pyo3 = {workspace = true, features=["serde", "chrono"]} 17 | janitor-publish = { path = "../publish" } 18 | pyo3-log = { workspace = true } 19 | log = "0.4" 20 | chrono = { workspace = true, features = ["serde"] } 21 | breezyshim.workspace = true 22 | silver-platter = { workspace = true, features = ["debian"] } 23 | janitor-differ = { path = "../differ" } 24 | pyo3-async-runtimes = { workspace = true, features = ["tokio-runtime"] } 25 | 26 | [features] 27 | extension-module = ["pyo3/extension-module"] 28 | -------------------------------------------------------------------------------- /Dockerfile_auto_upload: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ## Standard packages: ./CONTRIBUTING.md 14 | cargo \ 15 | g++ \ 16 | gcc \ 17 | libpython3-dev \ 18 | libssl-dev \ 19 | pkg-config \ 20 | protobuf-compiler \ 21 | ## Extra packages 22 | python3-pip \ 23 | && apt-get clean 24 | 25 | COPY . /code 26 | 27 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,auto-upload]" \ 28 | && rm -rf /code 29 | 30 | EXPOSE 9933 31 | 32 | ENTRYPOINT ["janitor-auto-upload", "--port=9933", "--listen-address=0.0.0.0"] 33 | -------------------------------------------------------------------------------- /Dockerfile_differ: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ## Standard packages: ./CONTRIBUTING.md 14 | cargo \ 15 | g++ \ 16 | gcc \ 17 | libpython3-dev \ 18 | libssl-dev \ 19 | pkg-config \ 20 | protobuf-compiler \ 21 | ## Extra packages 22 | python3-gpg \ 23 | python3-pip \ 24 | && apt-get clean 25 | 26 | COPY . /code 27 | 28 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,differ]" \ 29 | && rm -rf /code 30 | 31 | EXPOSE 9920 32 | 33 | ENTRYPOINT ["janitor-differ", "--port=9920", "--listen-address=0.0.0.0"] 34 | -------------------------------------------------------------------------------- /Dockerfile_publish: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ## Standard packages: ./CONTRIBUTING.md 14 | cargo \ 15 | g++ \ 16 | gcc \ 17 | libpython3-dev \ 18 | libssl-dev \ 19 | pkg-config \ 20 | protobuf-compiler \ 21 | ## Extra packages 22 | python3-gpg \ 23 | python3-pip \ 24 | && apt-get clean 25 | 26 | COPY . /code 27 | 28 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,publish]" \ 29 | && rm -rf /code 30 | 31 | EXPOSE 9912 32 | 33 | ENTRYPOINT ["janitor-publish", "--port=9912", "--listen-address=0.0.0.0"] 34 | -------------------------------------------------------------------------------- /devnotes/branch-names.rst: -------------------------------------------------------------------------------- 1 | Goal 2 | ==== 3 | 4 | For runs: 5 | * runs/$uuid/$function tags 6 | * runs/$uuid/tags/$tag tags for new/updated upstream tags 7 | 8 | There is a symref at refs/$suite/$function that points at said tag and updated 9 | 10 | Also, track this information in the database. 11 | 12 | For related repositories, create remotes: 13 | 14 | * remotes/origin/ for Debian packaging 15 | * remotes/upstream/ for Upstream 16 | 17 | Names for functions: 18 | 19 | * "main" for the main branch (packaging or otherwise) 20 | * "upstream" for the upstream import branch for Debian packages 21 | * "pristine-tar" for the pristine-tar branch 22 | 23 | Roadmap 24 | ======= 25 | 26 | * Update publisher to use new tag names 27 | 28 | * Send requests to publisher to mirror origin/upstream repositories 29 | To start off with just: 30 | * name of remote ("origin", "upstream") 31 | * URL of remote 32 | 33 | * Push symrefs (refs/$suite/$function => refs/tags/$uuid/$function) 34 | + needs to be done by publisher 35 | -------------------------------------------------------------------------------- /py/janitor/site/templates/ready-list.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% if not suite %} 4 | {% include "cupboard/sidebar.html" %} 5 | {% else %} 6 | {% include [suite + "/sidebar.html", "generic/sidebar.html"] %} 7 | {% endif %} 8 | {% endblock sidebar %} 9 | {% block page_title %} 10 | Changes Ready to Publish 11 | {% if suite %}- {{ suite }}{% endif %} 12 | {% endblock page_title %} 13 | {% block body %} 14 |
15 |

Ready to publish

16 |
    17 | {% for run in runs %} 18 |
  • 19 | {{ run.codebase }} 20 | {% set command = run.command -%} 21 | {% set result = run.result -%} 22 | {% include [run.suite + "/summary.html", "generic/summary.html"] %} 23 |
    • 24 | {% endfor %} 25 |
26 |
27 | {% endblock body %} 28 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/rejected.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard - Rejected in Review 7 | {% endblock page_title %} 8 | {% block body %} 9 |
10 |

Rejected runs during review

11 |
    12 | {% for run in runs %} 13 |
  • 14 | {{ run.suite }} for {{ run.package }} 15 |
      16 | {% for review in reviews[run.id] %} 17 |
    • 18 | {{ review.reviewer }}: {{ review.verdict }}: {{ review.comment }} 19 |
      • 20 | {% endfor %} 21 |
    22 |
    • 23 | {% endfor %} 24 |
25 |
26 | {% endblock body %} 27 | -------------------------------------------------------------------------------- /Dockerfile_archive: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ## Standard packages: ./CONTRIBUTING.md 14 | cargo \ 15 | g++ \ 16 | gcc \ 17 | libpython3-dev \ 18 | libssl-dev \ 19 | pkg-config \ 20 | protobuf-compiler \ 21 | ## Extra packages 22 | dpkg-dev \ 23 | git \ 24 | libapt-pkg-dev \ 25 | python3-gpg \ 26 | python3-pip \ 27 | && apt-get clean 28 | 29 | COPY . /code 30 | 31 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,archive]" \ 32 | && rm -rf /code 33 | 34 | EXPOSE 9914 35 | 36 | ENTRYPOINT ["janitor-archive", "--port=9914", "--listen-address=0.0.0.0"] 37 | -------------------------------------------------------------------------------- /Dockerfile_bzr_store: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ## Standard packages: ./CONTRIBUTING.md 14 | cargo \ 15 | g++ \ 16 | gcc \ 17 | libpython3-dev \ 18 | libssl-dev \ 19 | pkg-config \ 20 | protobuf-compiler \ 21 | ## Extra packages 22 | python3-gpg \ 23 | python3-pip \ 24 | && apt-get clean 25 | 26 | COPY . /code 27 | 28 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,bzr-store]" \ 29 | && rm -rf /code 30 | 31 | VOLUME /bzr 32 | 33 | EXPOSE 9929 34 | 35 | EXPOSE 9930 36 | 37 | ENTRYPOINT ["janitor-bzr-store", "--port=9929", "--public-port=9930", "--listen-address=0.0.0.0"] 38 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/ready-list.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% if not suite %} 4 | {% include "cupboard/sidebar.html" %} 5 | {% else %} 6 | {% include [suite + "/sidebar.html", "generic/sidebar.html"] %} 7 | {% endif %} 8 | {% endblock sidebar %} 9 | {% block page_title %} 10 | Cupboard - Changes Ready to Publish 11 | {% if suite %}- {{ suite }}{% endif %} 12 | {% endblock page_title %} 13 | {% block body %} 14 |
15 |

Ready to publish

16 |
    17 | {% for run in runs %} 18 |
  • 19 | {{ run.codebase }} 20 | {% set command = run.command -%} 21 | {% set result = run.result -%} 22 | {% include [run.suite + "/summary.html", "generic/summary.html"] %} 23 |
    • 24 | {% endfor %} 25 |
26 |
27 | {% endblock body %} 28 | -------------------------------------------------------------------------------- /.github/workflows/sql.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: SQL - Check database 3 | 4 | "on": 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | schedule: 10 | - cron: '0 6 * * *' # Daily 6AM UTC build 11 | 12 | jobs: 13 | build: 14 | name: Test SQL 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@v4 20 | 21 | - name: Install dependencies 22 | run: | 23 | set -x 24 | sudo apt-get update --yes 25 | PSQL_DEB=$( apt-cache search 'postgresql-.*-debversion' \ 26 | | awk '{print $1}' \ 27 | | tail -n 1 ) 28 | sudo apt-get satisfy --yes --no-install-recommends \ 29 | postgresql \ 30 | ${PSQL_DEB} \ 31 | postgresql-common 32 | 33 | - name: Load SQL 34 | run: | 35 | set -x 36 | set -o pipefail 37 | cat py/janitor/state.sql py/janitor/debian/debian.sql \ 38 | | pg_virtualenv psql -v ON_ERROR_STOP=1 39 | -------------------------------------------------------------------------------- /Dockerfile_git_store: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ## Standard packages: ./CONTRIBUTING.md 14 | cargo \ 15 | g++ \ 16 | gcc \ 17 | libpython3-dev \ 18 | libssl-dev \ 19 | pkg-config \ 20 | protobuf-compiler \ 21 | ## Extra packages 22 | git \ 23 | python3-gpg \ 24 | python3-pip \ 25 | && apt-get clean 26 | 27 | COPY . /code 28 | 29 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,git-store]" \ 30 | && rm -rf /code 31 | 32 | VOLUME /git 33 | 34 | EXPOSE 9923 35 | 36 | EXPOSE 9924 37 | 38 | ENTRYPOINT ["janitor-git-store", "--port=9923", "--public-port=9924", "--listen-address=0.0.0.0"] 39 | -------------------------------------------------------------------------------- /src/prometheus.rs: -------------------------------------------------------------------------------- 1 | use reqwest::Client; 2 | use std::collections::HashMap; 3 | use url::Url; 4 | 5 | pub async fn push_to_gateway( 6 | prometheus: &Url, 7 | job: &str, 8 | grouping_key: HashMap<&str, &str>, 9 | registry: &prometheus::Registry, 10 | ) -> Result<(), Box> { 11 | let client = Client::new(); 12 | let mut buffer = String::new(); 13 | let encoder = prometheus::TextEncoder::new(); 14 | let metric_families = registry.gather(); 15 | encoder.encode_utf8(&metric_families, &mut buffer).unwrap(); 16 | let mut url = prometheus.join("/metrics/job/").unwrap().join(job).unwrap(); 17 | for (k, v) in grouping_key { 18 | url.query_pairs_mut().append_pair(k, v); 19 | } 20 | let response = client 21 | .post(url) 22 | .header("Content-Type", "text/plain") 23 | .body(buffer) 24 | .send() 25 | .await?; 26 | if !response.status().is_success() { 27 | return Err(format!("Unexpected status code: {}", response.status()).into()); 28 | } 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /common-py/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "common-py" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | publish = false 6 | edition.workspace = true 7 | description = "Common bindings for the janitor - python" 8 | license = "GPL-3.0+" 9 | repository = "https://github.com/jelmer/janitor.git" 10 | homepage = "https://github.com/jelmer/janitor" 11 | 12 | [lib] 13 | crate-type = ["cdylib"] 14 | 15 | [dependencies] 16 | pyo3 = {workspace = true, features=["serde", "chrono"]} 17 | janitor = { path = ".." } 18 | reqwest = { version = "0.12", features = ["json"] } 19 | serde_json = "1" 20 | pyo3-log = { workspace = true } 21 | log = "0.4" 22 | chrono = { workspace = true, features = ["serde"] } 23 | breezyshim.workspace = true 24 | silver-platter = { workspace = true, features = ["debian", "pyo3"] } 25 | url.workspace = true 26 | pyo3-async-runtimes = { workspace = true, features = ["tokio-runtime"] } 27 | tokio.workspace = true 28 | maplit.workspace = true 29 | pyo3-filelike.workspace = true 30 | 31 | [features] 32 | extension-module = ["pyo3/extension-module"] 33 | -------------------------------------------------------------------------------- /Dockerfile_runner: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get upgrade --yes \ 12 | && apt-get satisfy --yes --no-install-recommends \ 13 | ## Standard packages: ./CONTRIBUTING.md 14 | cargo \ 15 | g++ \ 16 | gcc \ 17 | libpython3-dev \ 18 | libssl-dev \ 19 | pkg-config \ 20 | protobuf-compiler \ 21 | ## Extra packages 22 | dpkg-dev \ 23 | git \ 24 | libapt-pkg-dev \ 25 | python3-gpg \ 26 | python3-pip \ 27 | && apt-get clean 28 | 29 | COPY . /code 30 | 31 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,runner]" \ 32 | && rm -rf /code 33 | 34 | EXPOSE 9911 35 | 36 | EXPOSE 9919 37 | 38 | ENTRYPOINT ["janitor-runner", "--port=9911", "--public-port=9919", "--listen-address=0.0.0.0"] 39 | -------------------------------------------------------------------------------- /Dockerfile_site: -------------------------------------------------------------------------------- 1 | # TODO: config 2 | # TODO: service discovery 3 | 4 | # https://hub.docker.com/_/debian 5 | FROM docker.io/debian:testing-slim AS build 6 | MAINTAINER Jelmer Vernooij 7 | 8 | ARG DEBIAN_FRONTEND=noninteractive 9 | 10 | RUN apt-get update --yes \ 11 | && apt-get install --yes --no-install-recommends \ 12 | auto-apt-proxy \ 13 | iproute2 \ 14 | && apt-get upgrade --yes \ 15 | && apt-get satisfy --yes --no-install-recommends \ 16 | ## Standard packages: ./CONTRIBUTING.md 17 | cargo \ 18 | g++ \ 19 | gcc \ 20 | libpython3-dev \ 21 | libssl-dev \ 22 | pkg-config \ 23 | protobuf-compiler \ 24 | ## Extra packages 25 | libjs-jquery-datatables \ 26 | python3-gpg \ 27 | python3-pip \ 28 | && apt-get clean 29 | 30 | COPY . /code 31 | 32 | RUN pip3 install --break-system-packages --upgrade "/code[gcp,site]" \ 33 | && rm -rf /code 34 | 35 | EXPOSE 8080 36 | 37 | EXPOSE 8090 38 | 39 | ENTRYPOINT ["janitor-site", "--port=8080", "--public-port=8090", "--host=0.0.0.0"] 40 | -------------------------------------------------------------------------------- /py/janitor/site/_static/lintian.css: -------------------------------------------------------------------------------- 1 | ul.lintian-filelist { 2 | list-style-type: disclosure-closed; 3 | } 4 | 5 | ul.lintian-hintlist { 6 | list-style: none; 7 | padding-left: 5px; 8 | } 9 | 10 | ul.lintian-commentlist { 11 | list-style: none; 12 | padding-left: 0px; 13 | } 14 | 15 | li.lintian-fileitem { 16 | margin: 5px 0 5px 0; 17 | } 18 | 19 | #lintian-code-informative { 20 | color: #111; 21 | background-color: #C7EA3C; 22 | } 23 | 24 | #lintian-code-warning { 25 | color: #111; 26 | background-color: #FFEB44; 27 | } 28 | 29 | #lintian-code-error { 30 | color: #111; 31 | background-color: #FF6700; 32 | } 33 | 34 | #lintian-code-X { 35 | color: #111; 36 | background-color: #EE99EE; 37 | } 38 | 39 | #lintian-code-O { 40 | color: #111; 41 | background-color: #DDD; 42 | } 43 | 44 | #lintian-code-pedantic { 45 | color: #111; 46 | background-color: #C7EA3C; 47 | } 48 | 49 | #lintian-code-classification { 50 | color: #111; 51 | background-color: lightblue; 52 | } 53 | 54 | li.lintian-code-O { 55 | color: #444; 56 | } 57 | -------------------------------------------------------------------------------- /py/janitor/site/templates/codeblock.html: -------------------------------------------------------------------------------- 1 | {% macro include_console_log(f, include_lines=None, highlight_lines=None, id=None) %} 2 | {% set lines = read_file(f) %} 3 |
4 | 5 | 6 | 12 | 19 | 20 |
7 |
8 | 
{% for i, line in enumerate(lines, 1) %}{% if in_line_boundaries(i, include_lines) %}{{ i }}
 9 | {% endif %}{% endfor %}
10 |
11 | 		13 |
14 | 
15 | {% for i, line in enumerate(lines, 1) %}{% if in_line_boundaries(i, include_lines) %}{{ line.rstrip('\n') }}
16 | {% endif %}{% endfor %}
17 |
18 |
21 |
22 | {% endmacro %} 23 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/util.html: -------------------------------------------------------------------------------- 1 | {% macro reprocess_logs_button(run_id, title="Reprocess logs", id="reprocess-logs", accesskey="l") %} 2 | 4 | 20 | {% endmacro %} 21 | -------------------------------------------------------------------------------- /py/janitor/site/templates/faq-supported-vcs.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | What repositories are supported? 4 |

5 |

Repositories on the following hosting platforms are supported:

6 | 17 |

18 | Work is under way to also support Mercurial. Subversion support may 19 | also be an option, though I have yet to work out what the equivalent of 20 | pull requests in Subversion would be. 21 |

22 |
23 | -------------------------------------------------------------------------------- /differ/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Differ crate for the Janitor project. 2 | //! 3 | //! This crate provides functionality for finding and comparing binary files. 4 | 5 | #![deny(missing_docs)] 6 | 7 | /// Module for interacting with diffoscope 8 | pub mod diffoscope; 9 | 10 | use std::ffi::{OsStr, OsString}; 11 | use std::path::{Path, PathBuf}; 12 | 13 | /// Find binary files in a directory. 14 | /// 15 | /// # Arguments 16 | /// * `path` - The directory to search 17 | /// 18 | /// # Returns 19 | /// An iterator of (filename, path) pairs 20 | pub fn find_binaries(path: &Path) -> impl Iterator { 21 | std::fs::read_dir(path).unwrap().filter_map(|entry| { 22 | let entry = entry.ok()?; 23 | let path = entry.path(); 24 | Some((entry.file_name(), path)) 25 | }) 26 | } 27 | 28 | /// Check if a filename is a binary package. 29 | /// 30 | /// # Arguments 31 | /// * `name` - The filename to check 32 | /// 33 | /// # Returns 34 | /// `true` if the file is a binary package, `false` otherwise 35 | pub fn is_binary(name: &str) -> bool { 36 | name.ends_with(".deb") || name.ends_with(".udeb") 37 | } 38 | -------------------------------------------------------------------------------- /src/logging.rs: -------------------------------------------------------------------------------- 1 | #[derive(clap::Args, Debug, Clone)] 2 | #[group()] 3 | pub struct LoggingArgs { 4 | /// Enable debug mode. 5 | #[arg(long, default_value_t = false)] 6 | pub debug: bool, 7 | 8 | /// Use Google cloud logging. 9 | #[cfg(feature = "gcp")] 10 | #[arg(long, default_value_t = false)] 11 | pub gcp_logging: bool, 12 | } 13 | 14 | impl LoggingArgs { 15 | pub fn init(&self) { 16 | #[cfg(feature = "gcp")] 17 | let gcp_logging = self.gcp_logging; 18 | 19 | #[cfg(not(feature = "gcp"))] 20 | let gcp_logging = false; 21 | init_logging(gcp_logging, self.debug); 22 | } 23 | } 24 | 25 | pub fn init_logging(gcp_logging: bool, debug_mode: bool) { 26 | #[cfg(feature = "gcp")] 27 | if gcp_logging { 28 | stackdriver_logger::init_with_cargo!("../Cargo.toml"); 29 | return; 30 | } 31 | 32 | #[cfg(not(feature = "gcp"))] 33 | assert!(!gcp_logging, "GCP logging is not enabled"); 34 | 35 | use log::LevelFilter; 36 | let level = if debug_mode { 37 | LevelFilter::Debug 38 | } else { 39 | LevelFilter::Info 40 | }; 41 | env_logger::builder().filter(None, level).init(); 42 | } 43 | -------------------------------------------------------------------------------- /tests/test_site_simple.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | from janitor.config import read_string as read_config_string 19 | from janitor.site.simple import create_app 20 | 21 | 22 | def create_config(): 23 | return read_config_string(""" 24 | campaign { 25 | name: "lintian-fixes" 26 | } 27 | """) 28 | 29 | 30 | async def test_create_app(): 31 | await create_app(config=create_config()) 32 | -------------------------------------------------------------------------------- /py/janitor/_publish.pyi: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | def calculate_next_try_time(finish_time: datetime, attempt_count: int) -> datetime: ... 4 | def get_merged_by_user_url(url: str, user: str) -> str | None: ... 5 | def branches_match(url_a: str | None, url_b: str | None) -> bool: ... 6 | def role_branch_url(url: str, remote_branch_name: str | None) -> str: ... 7 | 8 | class RateLimiter: 9 | def set_mps_per_bucket(self, mps_per_bucket: dict[str, dict[str, int]]) -> None: ... 10 | def check_allowed(self, bucket: str) -> None: ... 11 | def inc(self, bucket: str) -> None: ... 12 | def get_stats(self) -> dict[str, tuple[int, int | None]]: ... 13 | 14 | class SlowStartRateLimiter(RateLimiter): 15 | def __init__(self, mps_per_bucket: int | None) -> None: ... 16 | 17 | class NonRateLimiter(RateLimiter): 18 | def __init__(self) -> None: ... 19 | 20 | class FixedRateLimiter(RateLimiter): 21 | def __init__(self, mps_per_bucket: int) -> None: ... 22 | 23 | class RateLimited(Exception): 24 | def __init__(self, message: str) -> None: ... 25 | 26 | class BucketRateLimited(RateLimited): 27 | def __init__(self, bucket: str, open_mps: int, max_open_mps: int) -> None: ... 28 | 29 | bucket: str 30 | open_mps: int 31 | max_open_mps: int 32 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import importlib.resources 2 | from collections.abc import AsyncGenerator 3 | 4 | import asyncpg 5 | import pytest_asyncio 6 | import testing.postgresql 7 | 8 | from janitor.state import create_pool 9 | 10 | pytest_plugins = ["aiohttp"] 11 | 12 | 13 | @pytest_asyncio.fixture() 14 | async def db(): 15 | with testing.postgresql.Postgresql() as postgresql: 16 | conn = await asyncpg.connect(postgresql.url()) 17 | files = importlib.resources.files("janitor") 18 | debian_files = importlib.resources.files("janitor.debian") 19 | try: 20 | with files.joinpath("state.sql").open() as f: 21 | await conn.execute(f.read()) 22 | with debian_files.joinpath("debian.sql").open() as f: 23 | await conn.execute(f.read()) 24 | finally: 25 | await conn.close() 26 | 27 | db = await create_pool(postgresql.url()) 28 | 29 | yield db 30 | 31 | await db.close() 32 | 33 | 34 | @pytest_asyncio.fixture() 35 | async def con(db: asyncpg.Pool) -> AsyncGenerator[asyncpg.Connection, None]: 36 | async with db.acquire() as con: 37 | yield con 38 | 39 | 40 | async def test_db_returns_janitor_db(db) -> None: 41 | assert isinstance(db, asyncpg.Pool) 42 | -------------------------------------------------------------------------------- /worker/src/bin/debian-build.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | 3 | #[derive(Parser)] 4 | struct Args { 5 | /// Path to configuration (JSON) 6 | #[clap(short, long)] 7 | config: Option, 8 | /// Output directory 9 | #[clap(short, long)] 10 | output_directory: std::path::PathBuf, 11 | } 12 | 13 | fn main() { 14 | let args = Args::parse(); 15 | 16 | breezyshim::init(); 17 | 18 | let (wt, subpath) = 19 | breezyshim::workingtree::open_containing(std::path::Path::new(".")).unwrap(); 20 | 21 | let config = if let Some(config) = args.config { 22 | let config = std::fs::read_to_string(config).unwrap(); 23 | serde_json::from_str(&config).unwrap() 24 | } else { 25 | serde_json::Value::Null 26 | }; 27 | 28 | match janitor_worker::debian::build_from_config( 29 | &wt, 30 | &subpath, 31 | &args.output_directory, 32 | &config, 33 | &std::env::vars().collect::>(), 34 | ) { 35 | Ok(result) => serde_json::to_writer(std::io::stdout(), &result).unwrap(), 36 | Err(e) => { 37 | serde_json::to_writer(std::io::stdout(), &serde_json::to_value(e).unwrap()).unwrap(); 38 | std::process::exit(1); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /publish/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-publish" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Publisher for the janitor" 7 | license = "GPL-3.0+" 8 | repository = "https://github.com/jelmer/janitor.git" 9 | homepage = "https://github.com/jelmer/janitor" 10 | 11 | [dependencies] 12 | axum = { workspace = true } 13 | breezyshim = { workspace = true, features = ["sqlx"] } 14 | chrono.workspace = true 15 | clap = { workspace = true, features = ["derive"] } 16 | debian-changelog = "0.2.0" 17 | janitor = { path = ".." } 18 | log.workspace = true 19 | minijinja = { version = "2", features = ["loader"] } 20 | pyo3.workspace = true 21 | redis = { workspace = true, features = ["tokio-comp", "json", "connection-manager"] } 22 | rslock = { workspace = true, default-features = false, features = ["tokio-comp"] } 23 | reqwest.workspace = true 24 | serde.workspace = true 25 | serde_json.workspace = true 26 | shlex.workspace = true 27 | silver-platter.workspace = true 28 | tokio = { workspace = true, features = ["full"] } 29 | url = { workspace = true, features = ["serde"] } 30 | sqlx = { workspace = true, features = ["chrono"] } 31 | maplit.workspace = true 32 | prometheus = "0.14.0" 33 | 34 | [dev-dependencies] 35 | maplit = { workspace = true } 36 | -------------------------------------------------------------------------------- /tests/test_core.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | 19 | from janitor import splitout_env 20 | 21 | 22 | def test_splitout_env(): 23 | assert splitout_env("ls") == ({}, "ls") 24 | assert splitout_env("PATH=/bin ls") == ({"PATH": "/bin"}, "ls") 25 | assert splitout_env("PATH=/bin FOO=bar ls") == ( 26 | {"PATH": "/bin", "FOO": "bar"}, 27 | "ls", 28 | ) 29 | assert splitout_env("PATH=/bin FOO=bar ls -l") == ( 30 | {"PATH": "/bin", "FOO": "bar"}, 31 | "ls -l", 32 | ) 33 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2019 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import unittest 19 | 20 | from breezy.tests import TestCaseWithTransport # noqa: F401 21 | 22 | 23 | def test_suite(): 24 | names = [ 25 | "archive", 26 | "artifacts", 27 | "debdiff", 28 | "debian", 29 | "runner", 30 | "site", 31 | "vcs", 32 | ] 33 | module_names = [__name__ + ".test_" + name for name in names] 34 | loader = unittest.TestLoader() 35 | return loader.loadTestsFromNames(module_names) 36 | -------------------------------------------------------------------------------- /py/janitor/debian/debdiff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # Copyright (C) 2019 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | from .._common import debdiff # type: ignore 19 | 20 | debdiff_is_empty = debdiff.debdiff_is_empty # type: ignore 21 | filter_boring = debdiff.filter_boring # type: ignore 22 | section_is_wdiff = debdiff.section_is_wdiff # type: ignore 23 | markdownify_debdiff = debdiff.markdownify_debdiff # type: ignore 24 | htmlize_debdiff = debdiff.htmlize_debdiff # type: ignore 25 | DebdiffError = debdiff.DebdiffError # type: ignore 26 | run_debdiff = debdiff.run_debdiff # type: ignore 27 | -------------------------------------------------------------------------------- /worker/src/bin/generic-build.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | 3 | #[derive(Parser)] 4 | struct Args { 5 | /// Path to configuration (JSON) 6 | #[clap(short, long)] 7 | config: Option, 8 | /// Output directory 9 | #[clap(short, long)] 10 | output_directory: std::path::PathBuf, 11 | } 12 | 13 | fn main() { 14 | let args = Args::parse(); 15 | 16 | breezyshim::init(); 17 | 18 | let (wt, subpath) = 19 | breezyshim::workingtree::open_containing(std::path::Path::new(".")).unwrap(); 20 | 21 | let config: janitor::api::worker::GenericBuildConfig = if let Some(config) = args.config { 22 | let config = std::fs::read_to_string(config).unwrap(); 23 | serde_json::from_str(&config).unwrap() 24 | } else { 25 | serde_json::from_value(serde_json::json!({})).unwrap() 26 | }; 27 | 28 | match janitor_worker::generic::build_from_config( 29 | &wt, 30 | &subpath, 31 | &args.output_directory, 32 | &config, 33 | &std::env::vars().collect::>(), 34 | ) { 35 | Ok(result) => serde_json::to_writer(std::io::stdout(), &result).unwrap(), 36 | Err(e) => { 37 | serde_json::to_writer(std::io::stdout(), &serde_json::to_value(e).unwrap()).unwrap(); 38 | std::process::exit(1); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /devnotes/adding-a-new-campaign.rst: -------------------------------------------------------------------------------- 1 | Adding a new campaign 2 | ===================== 3 | 4 | Create a new codemod script 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | At the core of every campaign is a script that can make changes 8 | to a version controlled branch. 9 | 10 | This script will be executed in a version controlled checkout of 11 | a source codebase, and can make changes to the codebase as it sees fit. 12 | See `this blog post `_ for more 13 | information about creating codemod scripts. 14 | 15 | You can test the script independently by running silver-platter, e.g. 16 | 17 | ``./debian-svp apply --command=myscript --dry-run --diff`` (from a checkout) 18 | or 19 | 20 | ``./debian-svp run --command=myscript --dry-run --diff package-name`` 21 | 22 | Add configuration for the campaign 23 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 24 | 25 | In janitor.conf, add a section for the campaign. E.g.:: 26 | 27 | campaign { 28 | name: "some-name" 29 | branch_name: "some-name" 30 | debian_build { 31 | build_suffix: "suf" 32 | } 33 | } 34 | 35 | Add script for finding candidates 36 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 37 | 38 | Add a script that can gather candidates for the new campaign. This script should 39 | be run regularly to find new candidates to schedule, with its JSON output 40 | uploaded to $RUNNER_URL/candidates. 41 | -------------------------------------------------------------------------------- /py/janitor/site/templates/result-codes/invalid-upstream-version-format.html: -------------------------------------------------------------------------------- 1 |

2 | The upstream version that was picked contains characters that 3 | are invalid in Debian version strings. 4 |

5 |

6 | For version strings that come from upstream tags, this can 7 | be because the upstream tags use characters that are not valid 8 | in Debian version strings. The Janitor currently only applies 9 | very basic version mangling to upstream tags: 10 |

    11 |
  • 12 | Strip release- prefixes and -release suffixes 13 |
    • 14 |
  • 15 | Strip package- prefixes 16 |
    • 17 |
  • 18 | Strip v prefixes 19 |
    • 20 |
  • 21 | Replace any underscores with dots if there are no other 22 | dots in the version. This is done for compatibility with CVS style tags, 23 | which usually did not use dots. 24 |
    • 25 |
26 |

27 | For version strings that come from uscan, no additional mangling 28 | is performed besides the mangling that uscan already does. 29 |

30 |

31 | In some cases, the version string matching is overly broad - and the 32 | lintian-brush could possibly replace the first group with @ANY_VERSION@ to 33 | fix the watch file. 34 |

35 |

36 | -------------------------------------------------------------------------------- /differ/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-differ" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Differ for the janitor" 7 | license = "GPL-3.0+" 8 | repository = "https://github.com/jelmer/janitor.git" 9 | homepage = "https://github.com/jelmer/janitor" 10 | 11 | [dependencies] 12 | janitor = { path = ".." } 13 | clap = { optional = true, workspace = true } 14 | env_logger = { optional = true, workspace = true } 15 | serde_json.workspace = true 16 | tokio = { workspace = true, features = ["full"] } 17 | tracing = "0.1.41" 18 | serde.workspace = true 19 | shlex.workspace = true 20 | patchkit = "0.2.1" 21 | axum.workspace = true 22 | sqlx.workspace = true 23 | redis = { workspace = true, features = ["aio", "connection-manager", "tokio", "tokio-comp", "json"] } 24 | tempfile.workspace = true 25 | breezyshim = { workspace = true, features = ["sqlx"] } 26 | nix = { version = "0.30.0", features = ["resource"] } 27 | axum-extra = { version = "0.10.1", features = ["typed-header"] } 28 | mime = "0.3.17" 29 | accept-header = "0.2.3" 30 | pyo3.workspace = true 31 | 32 | [dev-dependencies] 33 | maplit = { workspace = true } 34 | static_assertions = { workspace = true } 35 | 36 | [features] 37 | cli = ["dep:clap", "dep:env_logger"] 38 | default = ["cli"] 39 | 40 | [[bin]] 41 | name = "janitor-differ" 42 | path = "src/main.rs" 43 | required-features = ["cli"] 44 | -------------------------------------------------------------------------------- /tests/test_site.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2019 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | from datetime import datetime, timedelta 19 | 20 | from janitor.site import format_duration, format_timestamp 21 | 22 | 23 | def test_duration(): 24 | assert "10s" == format_duration(timedelta(seconds=10)) 25 | assert "1m10s" == format_duration(timedelta(seconds=70)) 26 | assert "1h0m" == format_duration(timedelta(hours=1)) 27 | assert "1d1h" == format_duration(timedelta(days=1, hours=1)) 28 | assert "2w1d" == format_duration(timedelta(weeks=2, days=1)) 29 | 30 | 31 | def test_timestamp(): 32 | assert "2022-10-01T11:10" == format_timestamp(datetime(2022, 10, 1, 11, 10, 22)) 33 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from setuptools import setup 3 | from setuptools_rust import Binding, RustBin, RustExtension 4 | 5 | setup( 6 | rust_extensions=[ 7 | RustExtension( 8 | "janitor._common", 9 | "common-py/Cargo.toml", 10 | binding=Binding.PyO3, 11 | features=["extension-module"], 12 | ), 13 | RustExtension( 14 | "janitor._differ", 15 | "differ-py/Cargo.toml", 16 | binding=Binding.PyO3, 17 | features=["extension-module"], 18 | ), 19 | RustExtension( 20 | "janitor._publish", 21 | "publish-py/Cargo.toml", 22 | binding=Binding.PyO3, 23 | features=["extension-module"], 24 | ), 25 | RustExtension( 26 | "janitor._runner", 27 | "runner-py/Cargo.toml", 28 | binding=Binding.PyO3, 29 | features=["extension-module"], 30 | ), 31 | RustExtension( 32 | "janitor._site", 33 | "site-py/Cargo.toml", 34 | binding=Binding.PyO3, 35 | features=["extension-module"], 36 | ), 37 | RustBin("janitor-mail-filter", "mail-filter/Cargo.toml", features=["cmdline"]), 38 | RustBin("janitor-worker", "worker/Cargo.toml", features=["cli", "debian"]), 39 | RustBin("janitor-dist", "worker/Cargo.toml", features=["cli", "debian"]), 40 | ] 41 | ) 42 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Linting 3 | 4 | "on": 5 | push: 6 | pull_request: 7 | schedule: 8 | - cron: '0 6 * * *' # Daily 6AM UTC build 9 | 10 | jobs: 11 | yamllint: 12 | name: YAML Lint 13 | runs-on: ubuntu-latest 14 | 15 | # Steps to perform in job 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v4 19 | 20 | - name: YAML style checks (yamllint) 21 | if: always() 22 | run: | 23 | set -x 24 | pip3 install --break-system-packages --upgrade \ 25 | yamllint 26 | make yamllint 27 | 28 | djlint: 29 | name: HTML Lint 30 | runs-on: ubuntu-latest 31 | 32 | # Steps to perform in job 33 | steps: 34 | - name: Checkout code 35 | uses: actions/checkout@v4 36 | 37 | - name: HTML style checks (djLint) 38 | if: always() 39 | run: | 40 | set -x 41 | pip3 install --break-system-packages --upgrade \ 42 | djlint 43 | make djlint 44 | 45 | codespell: 46 | name: Check common misspellings 47 | runs-on: ubuntu-latest 48 | 49 | # Steps to perform in job 50 | steps: 51 | - name: Checkout code 52 | uses: actions/checkout@v4 53 | 54 | - name: Check common misspellings (codespell) 55 | if: always() 56 | run: | 57 | set -x 58 | pip3 install --break-system-packages --upgrade \ 59 | codespell 60 | codespell 61 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Rust build 3 | 4 | "on": 5 | push: 6 | pull_request: 7 | 8 | env: 9 | CARGO_TERM_COLOR: always 10 | 11 | jobs: 12 | rust-build: 13 | 14 | runs-on: ${{ matrix.os }} 15 | strategy: 16 | matrix: 17 | os: [ubuntu-latest] 18 | fail-fast: false 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Install dependencies 23 | run: | 24 | sudo apt -y update 25 | sudo apt -y install devscripts libapt-pkg-dev libtdb-dev libssl-dev \ 26 | pkg-config libgpgme-dev protobuf-compiler diffoscope 27 | - name: Upgrade pip 28 | run: python -m pip install --upgrade pip setuptools_rust setuptools 29 | - name: Install breezy, diffoscope 30 | run: python -m pip install --upgrade breezy diffoscope jsondiff \ 31 | "brz-debian@git+https://github.com/breezy-team/breezy-debian" 32 | # TODO(jelmer): Add proper test isolation so this isn't necessary 33 | - name: Setup bzr identity 34 | run: brz whoami "CI " 35 | - name: Build 36 | run: cargo build --verbose --workspace 37 | - name: Run tests 38 | run: cargo test --verbose --workspace 39 | 40 | rust-fmt: 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v4 44 | - name: Install rustfmt 45 | run: sudo apt -y install rustfmt cargo 46 | - name: Check formatting 47 | run: cargo fmt --all -- --check 48 | -------------------------------------------------------------------------------- /Dockerfile_worker: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS build 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get satisfy --yes --no-install-recommends \ 12 | ca-certificates \ 13 | cargo \ 14 | libpython3-dev \ 15 | libssl-dev \ 16 | pkg-config \ 17 | protobuf-compiler \ 18 | python3-minimal \ 19 | && apt-get clean 20 | 21 | COPY . /code 22 | 23 | RUN cargo build --verbose --release --manifest-path /code/worker/Cargo.toml 24 | 25 | 26 | 27 | # https://hub.docker.com/_/debian 28 | FROM docker.io/debian:testing-slim 29 | MAINTAINER Jelmer Vernooij 30 | 31 | ARG DEBIAN_FRONTEND=noninteractive 32 | 33 | RUN apt-get update --yes \ 34 | && apt-get install --yes --no-install-recommends \ 35 | auto-apt-proxy \ 36 | iproute2 \ 37 | && apt-get upgrade --yes \ 38 | && apt-get satisfy --yes --no-install-recommends \ 39 | libpython3-dev \ 40 | python3-breezy \ 41 | dpkg-dev \ 42 | && apt-get clean 43 | 44 | COPY --from=build /code/target/release/janitor-worker /usr/local/bin/janitor-worker 45 | 46 | COPY autopkgtest-wrapper /usr/local/bin/autopkgtest-wrapper 47 | 48 | ENV AUTOPKGTEST=/usr/local/bin/autopkgtest-wrapper 49 | 50 | EXPOSE 9821 51 | 52 | ENTRYPOINT ["janitor-worker", "--port=9821", "--listen-address=0.0.0.0"] 53 | -------------------------------------------------------------------------------- /tests/test_queue.py: -------------------------------------------------------------------------------- 1 | from janitor.queue import Queue 2 | 3 | 4 | async def test_get_buckets(con): 5 | queue = Queue(con) 6 | assert await queue.get_buckets() == [] 7 | 8 | 9 | async def test_add(con): 10 | queue = Queue(con) 11 | await con.execute("INSERT INTO codebase (name) VALUES ('foo')") 12 | assert await queue.add(codebase="foo", campaign="bar", command="true") == ( 13 | 1, 14 | "default", 15 | ) 16 | queue_item, vcs_info = await queue.next_item() 17 | assert queue_item 18 | assert queue_item.codebase == "foo" 19 | assert queue_item.campaign == "bar" 20 | 21 | 22 | async def test_double_add(con): 23 | queue = Queue(con) 24 | await con.execute("INSERT INTO codebase (name) VALUES ('foo')") 25 | assert await queue.add(codebase="foo", campaign="bar", command="true") == ( 26 | 1, 27 | "default", 28 | ) 29 | assert await queue.add(codebase="foo", campaign="bar", command="true") == ( 30 | 1, 31 | "default", 32 | ) 33 | 34 | 35 | async def test_vcs_only(con): 36 | queue = Queue(con) 37 | await con.execute( 38 | "INSERT INTO codebase (name, vcs_type, branch_url) VALUES ('foo', 'git', NULL)" 39 | ) 40 | assert await queue.add(codebase="foo", campaign="bar", command="true") == ( 41 | 1, 42 | "default", 43 | ) 44 | queue_item, vcs_info = await queue.next_item() 45 | assert queue_item 46 | assert queue_item.codebase == "foo" 47 | assert queue_item.campaign == "bar" 48 | assert vcs_info == {"vcs_type": "git"} 49 | -------------------------------------------------------------------------------- /runner/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-runner" 3 | version.workspace = true 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Runner for the janitor" 7 | license = "GPL-3.0+" 8 | repository = "https://github.com/jelmer/janitor.git" 9 | homepage = "https://github.com/jelmer/janitor" 10 | 11 | [dependencies] 12 | breezyshim.workspace = true 13 | tokio = { workspace = true, features = ["full"] } 14 | sqlx.workspace = true 15 | sqlx-core.workspace = true 16 | sqlx-postgres.workspace = true 17 | redis = { workspace = true, features = ["aio", "tokio-comp"] } 18 | chrono = { workspace = true, features = ["serde"] } 19 | serde.workspace = true 20 | serde_json.workspace = true 21 | janitor = { path = "..", default-features = false, features = ["debian"] } 22 | async-trait = "0.1.88" 23 | url.workspace = true 24 | debversion = { workspace = true, optional = true, features = ["sqlx"] } 25 | debian-control = { version = "0.1.28", optional = true } 26 | log.workspace = true 27 | silver-platter = { workspace = true, features = ["debian"] } 28 | reqwest.workspace = true 29 | clap = { workspace = true, features = ["derive"], optional = true } 30 | axum.workspace = true 31 | serde_with = { version = "3.9.0", features = ["chrono_0_4"] } 32 | 33 | [dev-dependencies] 34 | maplit = { workspace = true } 35 | 36 | [features] 37 | default = ["debian"] 38 | debian = ["janitor/debian", "dep:debversion", "dep:debian-control"] 39 | cli = ["dep:clap"] 40 | 41 | [[bin]] 42 | name = "janitor-runner" 43 | path = "src/main.rs" 44 | required-features = ["cli"] 45 | -------------------------------------------------------------------------------- /py/janitor/site/cupboard/publish.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from aiohttp import web 4 | 5 | 6 | async def iter_publish_history(conn, limit: Optional[int] = None): 7 | query = """ 8 | SELECT 9 | publish.id, publish.timestamp, publish.branch_name, 10 | publish.mode, publish.merge_proposal_url, publish.result_code, 11 | publish.description, codebase.web_url, publish.codebase AS codebase 12 | FROM 13 | publish 14 | LEFT JOIN codebase ON codebase.name = publish.codebase 15 | ORDER BY timestamp DESC 16 | """ 17 | if limit: 18 | query += f" LIMIT {limit}" 19 | return await conn.fetch(query) 20 | 21 | 22 | async def write_history(conn, limit: Optional[int] = None): 23 | return { 24 | "count": limit, 25 | "history": await iter_publish_history(conn, limit=limit), 26 | } 27 | 28 | 29 | async def write_publish(conn, publish_id): 30 | query = """ 31 | SELECT 32 | publish.id AS id, 33 | publish.timestamp AS timestamp, 34 | publish.branch_name AS branch_name, 35 | publish.mode AS mode, 36 | publish.merge_proposal_url AS merge_proposal_url, 37 | publish.result_code AS result_code, 38 | publish.description AS description, 39 | codebase.web_url AS vcs_browse, 40 | codebase.name AS codebase 41 | FROM 42 | publish 43 | LEFT JOIN codebase ON codebase.name = publish.codebase 44 | WHERE id = $1 45 | """ 46 | publish = await conn.fetchrow(query, publish_id) 47 | if publish is None: 48 | raise web.HTTPNotFound(text=f"no such publish: {publish_id}") 49 | return {"publish": publish} 50 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/default-evaluate.html: -------------------------------------------------------------------------------- 1 |
2 | Score: {{ value }} 3 |
4 |
5 | Command: {{ command }} 6 |
7 |
8 | Finish Time: {{ format_timestamp(finish_time) }} 9 |
10 | {% for role, remote_branch_name, base_revision, revision in branches %} 11 |
12 |

{{ role }}

13 |
14 | 24 |
25 | {%- set DIFF_INLINE_THRESHOLD = 200 -%} 26 | {%- set diff = show_diff(role) -%} 27 | {% if diff.splitlines(False)|length < DIFF_INLINE_THRESHOLD %} 28 |
{{ highlight_diff(diff) |safe }}
29 | {% else %} 30 |
31 | The full diff is too large to include here. 32 |
33 | {% endif %} 34 |
35 | {% endfor %} 36 | -------------------------------------------------------------------------------- /py/janitor/site/templates/generic/sidebar.html: -------------------------------------------------------------------------------- 1 | {% from "inputs.html" import codebase_input %} 2 | 38 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/publish.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% from "run_util.html" import display_publish_result_code %} 6 | {% block page_title %} 7 | Cupboard - Publish {{ publish.id }} 8 | {% endblock page_title %} 9 | {% block body %} 10 |
11 |

Publish {{ publish.id }}

12 |
    13 |
  • Timestamp: {{ format_timestamp(publish.timestamp) }}
    • 14 |
  • 15 | Codebase: {{ publish.codebase }} 16 |
    • 17 | {% if publish.branch_name %}
  • Branch Name: {{ publish.branch_name }}
    • {% endif %} 18 |
  • Mode: {{ publish.mode }}
    • 19 |
  • 20 | {{ display_publish_result_code(publish.result_code) }}: 21 | {% if '\n' in publish.description %} 22 | 
    {{ publish.description }}
    23 | {% else %} 24 | {{ publish.description }} 25 | {% endif %} 26 | {% if publish.mode == 'propose' %} 27 | {% if publish.merge_proposal_url %}- Merge Proposal{% endif %} 28 | {% endif %} 29 | {% if publish.mode == 'push' and result_code == 'success' %} 30 | {% if publish.vcs_browse %}- Branch{% endif %} 31 | {% endif %} 32 |
    • 33 |
34 |
35 | {% endblock body %} 36 | -------------------------------------------------------------------------------- /py/janitor/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # Copyright (C) 2018 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | __all__ = [ 19 | "Config", 20 | "Campaign", 21 | "AptRepository", 22 | "read_config", 23 | "read_string", 24 | "get_campaign_config", 25 | "get_distribution", 26 | ] 27 | 28 | 29 | from ._common import config as _config_rs # type: ignore 30 | 31 | Config = _config_rs.Config 32 | Campaign = _config_rs.Campaign 33 | AptRepository = _config_rs.AptRepository 34 | read_config = _config_rs.read_config 35 | read_string = _config_rs.read_string 36 | get_distribution = _config_rs.get_distribution 37 | get_campaign_config = _config_rs.get_campaign_config 38 | 39 | 40 | if __name__ == "__main__": 41 | import argparse 42 | 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument("config_file", type=str, help="Configuration file to read") 45 | args = parser.parse_args() 46 | with open(args.config_file) as f: 47 | config = read_config(f) 48 | -------------------------------------------------------------------------------- /mail-filter/src/bin/janitor-mail-filter.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | 3 | use log::debug; 4 | use std::process; 5 | 6 | use clap::Parser; 7 | use log::{error, info}; 8 | use reqwest::blocking::Client; 9 | 10 | #[derive(Parser)] 11 | struct Args { 12 | #[clap( 13 | short, 14 | long, 15 | default_value = "https://janitor.debian.net/api/refresh-proposal-status" 16 | )] 17 | refresh_url: String, 18 | #[clap(short, long, default_value = "/dev/stdin")] 19 | input: String, 20 | } 21 | 22 | fn refresh_merge_proposal(api_url: &str, merge_proposal_url: &str) -> Result<(), String> { 23 | let client = Client::new(); 24 | let res = client 25 | .post(api_url) 26 | .json(&serde_json::json!({"url": merge_proposal_url})) 27 | .send() 28 | .map_err(|e| e.to_string())?; 29 | 30 | match res.status().as_u16() { 31 | 200 | 202 => Ok(()), 32 | status => Err(format!( 33 | "error {} triggering refresh for {}", 34 | status, api_url 35 | )), 36 | } 37 | } 38 | 39 | fn main() { 40 | let args = Args::parse(); 41 | 42 | let f = File::open(args.input).unwrap(); 43 | 44 | match janitor_mail_filter::parse_email(f) { 45 | Some(merge_proposal_url) => { 46 | info!("Found merge proposal URL: {}", merge_proposal_url); 47 | match refresh_merge_proposal(&args.refresh_url, &merge_proposal_url) { 48 | Ok(()) => process::exit(0), 49 | Err(e) => { 50 | error!("Error: {}", e); 51 | process::exit(1); 52 | } 53 | } 54 | } 55 | None => { 56 | debug!("No merge proposal URL found."); 57 | process::exit(0); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /helpers/cleanup-repositories.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Cleanup owned repositories that are no longer needed for merge proposals. 3 | 4 | This is necessary in particular because some hosting sites 5 | (e.g. default GitLab) have restrictions on the number of repositories 6 | that a single user can own (in the case of GitLab, 1000). 7 | """ 8 | 9 | import logging 10 | import sys 11 | 12 | import breezy 13 | import breezy.bzr 14 | import breezy.git # noqa: F401 15 | import breezy.plugins 16 | import breezy.plugins.github # noqa: F401 17 | import breezy.plugins.gitlab # noqa: F401 18 | import breezy.plugins.launchpad # noqa: F401 19 | from breezy.forge import UnsupportedForge, iter_forge_instances 20 | 21 | 22 | def projects_to_remove(instance): 23 | in_use = set() 24 | for mp in instance.iter_my_proposals(): 25 | if not mp.is_closed() and not mp.is_merged(): 26 | in_use.add(mp.get_source_project()) 27 | for project in instance.iter_my_forks(): 28 | if project in in_use: 29 | continue 30 | yield project 31 | 32 | 33 | def main(argv=None): 34 | import argparse 35 | 36 | parser = argparse.ArgumentParser() 37 | parser.add_argument("--dry-run", action="store_true", help="Dry run.") 38 | args = parser.parse_args() 39 | 40 | logging.basicConfig(format="%(message)s") 41 | 42 | for instance in iter_forge_instances(): 43 | try: 44 | for project in projects_to_remove(instance): 45 | logging.info(f"Deleting {project} from {instance!r}") 46 | if not args.dry_run: 47 | instance.delete_project(project) 48 | except UnsupportedForge as e: 49 | logging.warning("Ignoring unsupported instance %s: %s", instance, e) 50 | 51 | 52 | if __name__ == "__main__": 53 | sys.exit(main()) 54 | -------------------------------------------------------------------------------- /py/janitor/site/templates/lintian_util.html: -------------------------------------------------------------------------------- 1 | {%- macro lintian_severity_short(severity) -%} 2 | {%- if severity == 'error' -%}e 3 | {%- elif severity == 'warning' -%}w 4 | {%- elif severity == 'informational' -%}i 5 | {%- elif severity == 'pedantic' -%}p 6 | {%- else -%}? 7 | {%- endif -%} 8 | {%- endmacro -%} 9 | {% macro display_lintian_hintitem(hint) %} 10 |
  • 11 | {{ lintian_severity_short(hint.severity) }} 12 | {{ hint.name }} 13 | {{ hint.context }} 14 |
    • 15 | {% endmacro %} 16 | {% macro display_lintian_result(lintian_result) %} 17 | {% for group in lintian_result['groups'] %} 18 |
      19 | {% for file in group.get('input-files') or group.get('input_files', []) %} 20 |
    • 21 | {{ file.path.split('/')[-1] }} 22 |
      • 23 |
        24 | {% if not file.tags -%} 25 | {% else %} 26 | {% for hint in file.tags %}{{ display_lintian_hintitem(hint) }}{% endfor %} 27 | {% endif %} 28 |
      29 | {% endfor %} 30 |
    31 | {% endfor %} 32 | {% if 'lintian-version' in lintian_result %} 33 | 34 | {% endif %} 35 | {% if 'lintian_version' in lintian_result %} 36 | 37 | {% endif %} 38 | {% endmacro %} 39 | -------------------------------------------------------------------------------- /worker/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | Job{% if let Some(assignment) = assignment %}{{ assignment.id }}{% endif %} 3 | 4 | 5 | {% if let Some(assignment) = assignment %} 6 |

    Run Details

    7 | 8 |
      9 |
    • Raw Assignment
      • 10 |
    • Campaign: {{ assignment.campaign }}
      • 11 |
    • Codebase: {{ assignment.codebase }}
      • 12 | {% if metadata.is_some() && metadata.unwrap().start_time.is_some() %} 13 |
    • Start Time: {{ metadata.unwrap().start_time .unwrap()}} 14 |
    • Current duration: {{ chrono::Utc::now() - metadata.unwrap().start_time.unwrap() }} 15 | {% endif %} 16 |
    • Environment:
        17 | {% for (key, value) in assignment.env.iter() %} 18 |
      • {{ key }}: {{ value }}
        • 19 | {% endfor %} 20 | 21 |
      22 | 23 |

      Codemod

      24 | 25 |
        26 |
      • Command: {{ assignment.codemod.command }}
        • 27 |
      • Environment:
          28 | {% for (key, value) in assignment.codemod.environment.iter() %} 29 |
        • {{ key }}: {{ value }}
          • 30 | {% endfor %} 31 |
        32 |
        • 33 |
      34 | 35 |

      Build

      36 | 37 |
        38 |
      • Target: {{ assignment.build.target }}
        • 39 |
      • Force Build: {{ assignment.force_build }}
        • 40 | {% if let Some(build_env) = assignment.build.environment %} 41 |
      • Environment:
          42 | {% for (key, value) in build_env.iter() %} 43 |
        • {{ key }}: {{ value }}
          • 44 | {% endfor %} 45 |
        46 |
        • 47 | {% endif %} 48 |
      49 | 50 | {% if let Some(lognames) = lognames %} 51 |

      Logs

      52 |
        53 | {% for name in lognames.iter() %} 54 |
      • {{ name }}
        • 55 | {% endfor %} 56 |
      57 | {% endif %} 58 | 59 | {% else %} 60 | 61 |

      No current assignment.

      62 | 63 | {% endif %} 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /py/janitor/site/_static/janitor.css: -------------------------------------------------------------------------------- 1 | /* -- result codes */ 2 | span.result-code-bug, span.publish-bug { 3 | color: darkred; 4 | } 5 | 6 | span.result-code-failure, span.publish-failure { 7 | color: red; 8 | } 9 | 10 | span.result-code-transient-failure, span.publish-transient { 11 | color: orange; 12 | } 13 | 14 | span.result-code-success, span.publish-success { 15 | color: green; 16 | } 17 | 18 | span.result-code-nothing-new-to-do, span.publish-nothing-to-do { 19 | color: lightgreen; 20 | } 21 | 22 | span.result-code-nothing-to-do, span.publish-missing { 23 | color: grey; 24 | } 25 | 26 | blockquote.result-code-explanation { 27 | background-color: #FE9; 28 | } 29 | 30 | tr.package-error { 31 | background-color: coral; 32 | } 33 | 34 | tr.package-unabsorbed { 35 | background-color: azure; 36 | } 37 | 38 | tr.package-candidates { 39 | background-color: beige; 40 | } 41 | 42 | tr.package-nothing-to-do, tr.package-nothing-new-to-do { 43 | background-color: grey; 44 | opacity: 0.5; 45 | } 46 | 47 | tr.package-proposal { 48 | background-color: lightgreen; 49 | } 50 | 51 | li.not-in-archive { 52 | font-weight: bolder; 53 | } 54 | 55 | ul.metadata { 56 | list-style: none; 57 | background-color: lightyellow; 58 | } 59 | 60 | tr.row-disabled, td.old-version { 61 | color: lightgrey; 62 | } 63 | 64 | span.tinycode { 65 | font-family: monospace; 66 | font-weight: bold; 67 | font-size: 1.1em; 68 | padding: 1px 3px; 69 | } 70 | 71 | span.tinyhint { 72 | font-weight: 700; 73 | font-size: 0.9em; 74 | } 75 | 76 | span.tinycontext { 77 | font-style: italic; 78 | font-size: 0.9em; 79 | } 80 | 81 | span.tinycomment { 82 | font-size: 0.9em; 83 | } 84 | 85 | td.version { 86 | text-align: center; 87 | } 88 | 89 | /* Queue active lines */ 90 | tr.active { 91 | background-color: bisque; 92 | } 93 | -------------------------------------------------------------------------------- /archive/tests/data/hello_2.10-3.dsc: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP SIGNED MESSAGE----- 2 | Hash: SHA256 3 | 4 | Format: 3.0 (quilt) 5 | Source: hello 6 | Binary: hello 7 | Architecture: any 8 | Version: 2.10-3 9 | Maintainer: Santiago Vila 10 | Homepage: https://www.gnu.org/software/hello/ 11 | Standards-Version: 4.6.2 12 | Vcs-Browser: https://salsa.debian.org/sanvila/hello 13 | Vcs-Git: https://salsa.debian.org/sanvila/hello.git 14 | Testsuite: autopkgtest 15 | Build-Depends: debhelper-compat (= 13), help2man, texinfo 16 | Package-List: 17 | hello deb devel optional arch=any 18 | Checksums-Sha1: 19 | f7bebf6f9c62a2295e889f66e05ce9bfaed9ace3 725946 hello_2.10.orig.tar.gz 20 | 9dc7a584db576910856ac7aa5cffbaeefe9cf427 819 hello_2.10.orig.tar.gz.asc 21 | a2d122fd090dbab3d40b219a237fbb7d74f8023a 12684 hello_2.10-3.debian.tar.xz 22 | Checksums-Sha256: 23 | 31e066137a962676e89f69d1b65382de95a7ef7d914b8cb956f41ea72e0f516b 725946 hello_2.10.orig.tar.gz 24 | 4ea69de913428a4034d30dcdcb34ab84f5c4a76acf9040f3091f0d3fac411b60 819 hello_2.10.orig.tar.gz.asc 25 | 60ee7a466808301fbaa7fea2490b5e7a6d86f598956fb3e79c71b3295dc1f249 12684 hello_2.10-3.debian.tar.xz 26 | Files: 27 | 6cd0ffea3884a4e79330338dcc2987d6 725946 hello_2.10.orig.tar.gz 28 | e6074bb23a0f184e00fdfb5c546b3bc2 819 hello_2.10.orig.tar.gz.asc 29 | 27ab798c1d8d9048ffc8127e9b8dbfca 12684 hello_2.10-3.debian.tar.xz 30 | 31 | -----BEGIN PGP SIGNATURE----- 32 | 33 | iQEzBAEBCAAdFiEE1Uw7+v+wQt44LaXXQc5/C58bizIFAmOp5ssACgkQQc5/C58b 34 | izJEQwgAiB73GKmfMV5PPyysZhoruCJo5I3/egZXfzA4lKofsqDh/ItYdvev5Ijl 35 | Rrbmd6xOXxJcHSYIeeuMYwK3X0moOY1Qk3CqTO00yH9lHgZg1r7G0xqhTM0EX7Jz 36 | suUpdRX7liENsAdwHBAmW3F6Lh4MFmBYJ2UcaX/HN6XwEF19b+JRvN76kUH9kaUJ 37 | 037mwN/M9qfwaLvp3Dpm+p7G4CGLLEzIEJCwx8IPew74+YhsXknGuK/96VFngdwK 38 | LFH3j/Kutc952HdLkKOWCibYd1fklW7SMXmJfiYF0qN4LDmhVMdkfd9ToAGxPhwn 39 | pMBP096Z2WoW+Z7QVgzwG+MjbyPz2w== 40 | =kNoz 41 | -----END PGP SIGNATURE----- 42 | -------------------------------------------------------------------------------- /py/janitor/site/_static/janitor.js: -------------------------------------------------------------------------------- 1 | var handlers = []; 2 | 3 | registerHandler = function(kind, cb) { 4 | handlers.push({'kind': kind, 'callback': cb}); 5 | }; 6 | 7 | var ws_url; 8 | window.onload = function() { 9 | if(location.protocol == 'http:') { 10 | ws_url = 'ws://' + location.hostname + '/ws/notifications'; 11 | } else if(location.protocol == 'https:') { 12 | ws_url = 'wss://' + location.hostname + '/ws/notifications'; 13 | } else { 14 | console.log('Unknown protocol: ' + location.protocol); 15 | ws_url = undefined; 16 | } 17 | 18 | const connection = new WebSocket(ws_url); 19 | 20 | connection.onerror = (error) => { 21 | console.log('WebSocket error: '); 22 | console.log(error); 23 | } 24 | 25 | connection.onmessage = (e) => { 26 | data = JSON.parse(e.data); 27 | handlers.forEach(function(handler) { 28 | if (handler.kind == data[0]) { handler.callback(data[1]); } 29 | }); 30 | console.log(data); 31 | } 32 | } 33 | 34 | windowbeforeunload = function(){ 35 | socket.close(); 36 | }; 37 | 38 | // Please keep this logic in sync with janitor/site/__init__.py:format_duration 39 | format_duration = function(n) { 40 | var d = moment.duration(n, "s"); 41 | var ret = ""; 42 | if (d.weeks() > 0) { 43 | return d.weeks() + "w" + (d.days() % 7) + "d"; 44 | } 45 | if (d.days() > 0) { 46 | return d.days() + "d" + (d.hours() % 24) + "h"; 47 | } 48 | if (d.hours() > 0) { 49 | return d.hours() + "h" + (d.minutes() % 60) + "m"; 50 | } 51 | if (d.minutes() > 0) { 52 | return d.minutes() + "m" + (d.seconds() % 60) + "s"; 53 | } 54 | return d.seconds() + "s"; 55 | }; 56 | 57 | 58 | window.chartColors = { 59 | red: 'rgb(255, 99, 132)', 60 | orange: 'rgb(255, 159, 64)', 61 | yellow: 'rgb(255, 205, 86)', 62 | green: 'rgb(75, 192, 192)', 63 | blue: 'rgb(54, 162, 235)', 64 | purple: 'rgb(153, 102, 255)', 65 | grey: 'rgb(201, 203, 207)' 66 | }; 67 | -------------------------------------------------------------------------------- /py/janitor/artifacts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # Copyright (C) 2020 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | """Artifacts.""" 19 | 20 | import asyncio 21 | 22 | from ._common import artifacts # type: ignore 23 | 24 | ArtifactManager = artifacts.ArtifactManager 25 | GCSArtifactManager = artifacts.GCSArtifactManager 26 | LocalArtifactManager = artifacts.LocalArtifactManager 27 | ServiceUnavailable = artifacts.ServiceUnavailable 28 | ArtifactsMissing = artifacts.ArtifactsMissing 29 | get_artifact_manager = artifacts.get_artifact_manager 30 | list_ids = artifacts.list_ids 31 | upload_backup_artifacts = artifacts.upload_backup_artifacts 32 | store_artifacts_with_backup = artifacts.store_artifacts_with_backup 33 | 34 | DEFAULT_GCS_TIMEOUT = 60 35 | 36 | 37 | if __name__ == "__main__": 38 | import argparse 39 | 40 | parser = argparse.ArgumentParser() 41 | subparsers = parser.add_subparsers(dest="command") 42 | list_parser = subparsers.add_parser("list") 43 | list_parser.add_argument("location", type=str) 44 | args = parser.parse_args() 45 | if args.command == "list": 46 | manager = get_artifact_manager(args.location) 47 | asyncio.run(list_ids(manager)) 48 | -------------------------------------------------------------------------------- /tests/test_differ.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | 19 | import tempfile 20 | 21 | from janitor.artifacts import LocalArtifactManager 22 | from janitor.differ import create_app 23 | 24 | 25 | async def create_client(aiohttp_client, db): 26 | td = tempfile.TemporaryDirectory().name 27 | atd = tempfile.TemporaryDirectory().name 28 | afm = LocalArtifactManager(atd) 29 | return await aiohttp_client(create_app(td, afm, db=db)) 30 | 31 | 32 | async def test_health(aiohttp_client, db): 33 | client = await create_client(aiohttp_client, db) 34 | 35 | resp = await client.get("/health") 36 | assert resp.status == 200 37 | text = await resp.text() 38 | assert text == "ok" 39 | 40 | 41 | async def test_ready(aiohttp_client, db): 42 | client = await create_client(aiohttp_client, db) 43 | 44 | resp = await client.get("/ready") 45 | assert resp.status == 200 46 | text = await resp.text() 47 | assert text == "ok" 48 | 49 | 50 | async def test_precache_all(aiohttp_client, db): 51 | client = await create_client(aiohttp_client, db) 52 | 53 | resp = await client.post("/precache-all") 54 | assert resp.status == 200 55 | assert {"count": 0} == await resp.json() 56 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/changeset.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% from "run_util.html" import display_result_code %} 3 | {% block sidebar %} 4 | {% include "cupboard/sidebar.html" %} 5 | {% endblock sidebar %} 6 | {% block page_title %} 7 | Cupboard - Changeset 8 | {% endblock page_title %} 9 | {% block body %} 10 |
      11 |

      Changeset {{ changeset.id }}

      12 |

      13 | Status: {{ changeset.state }} 14 |

      15 |

      Runs

      16 | 23 |

      Left to process

      24 | 31 |

      APT Repository

      32 |

      33 | To install the packages from this changeset, use the following 34 | sources configuration (with archive keyring stored in 35 | /etc/apt/keyrings/debian-janitor.gpg): 36 |

      37 |

      38 |

      39 |
      40 | 
      41 | deb "[arch=amd64 signed-by=/etc/apt/keyrings/debian-janitor.gpg]" {{ url.join(URL('/')) }} cs/{{ changeset.id }} main
42 | deb-src "[arch=amd64 signed-by=/etc/apt/keyrings/debian-janitor.gpg]" {{ url.join(URL('/')) }} cs/{{ changeset.id }} main
43 |
      44 |
      45 |
      46 |

      47 |
      48 | {% endblock body %} 49 | -------------------------------------------------------------------------------- /py/janitor/site/templates/about.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block page_title %} 3 | About 4 | {% endblock page_title %} 5 | {% block body %} 6 |
      7 |

      About

      8 |

      9 | The initial version of the Janitor was written by Jelmer Vernooij. 10 |

      11 |

      12 | The code for the Debian-specific instance has now been split out into a separate 13 | repository. 14 |

      15 |

      Many thanks to everybody who has provided feedback and helped make the Janitor better.

      16 |

      Source Code

      17 |

      The Janitor itself is mostly written in Python, using the following libraries and services:

      18 |
        19 |
      • 20 | The style for the website is based on Alabaster 0.7.8 theme for sphinx. 21 |
        • 22 |
      • 23 | Breezy 24 | provides abstractions over the version control system (Git, Bazaar, 25 | Mercurial, Subversion) and the supported hosting platforms (GitHub, 26 | GitLab, Launchpad). 27 |
        • 28 |
      • 29 | Silver-Platter 31 | ties this all together; it manages branches, invokes codemods and pushes back or creates 32 | merge proposals. 33 |
        • 34 |
      35 |

      36 | The source code is hosted on GitHub. 37 |

      38 |

      Bugs

      39 |

      40 | Please report bugs in the bug tracker. 41 |

      42 |
      43 | {% endblock body %} 44 | -------------------------------------------------------------------------------- /py/janitor/_launchpad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # Copyright (C) 2019 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | 19 | def override_launchpad_consumer_name(): 20 | from breezy.forge import ForgeLoginRequired 21 | from breezy.plugins.launchpad import lp_api 22 | from launchpadlib.credentials import RequestTokenAuthorizationEngine 23 | from launchpadlib.launchpad import Launchpad 24 | 25 | class LoginRequiredAuthorizationEngine(RequestTokenAuthorizationEngine): 26 | def make_end_user_authorize_token(self, credentials, request_token): 27 | raise ForgeLoginRequired(self.web_root) 28 | 29 | def connect_launchpad( 30 | base_url, timeout=None, proxy_info=None, version=Launchpad.DEFAULT_VERSION 31 | ): 32 | cache_directory = lp_api.get_cache_directory() 33 | credential_store = lp_api.BreezyCredentialStore() 34 | authorization_engine = LoginRequiredAuthorizationEngine( 35 | base_url, consumer_name="Janitor" 36 | ) 37 | return Launchpad.login_with( 38 | "Janitor", 39 | base_url, 40 | cache_directory, 41 | timeout=timeout, 42 | credential_store=credential_store, 43 | authorization_engine=authorization_engine, 44 | version=version, 45 | ) 46 | 47 | lp_api.connect_launchpad = connect_launchpad 48 | -------------------------------------------------------------------------------- /py/janitor/site/templates/layout.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Janitor - 6 | {% block page_title %} 7 | {% endblock page_title %} 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | {% include "analytics.html" ignore missing without context %} 26 |
      27 | {% block sidebar %} 28 | {% endblock sidebar %} 29 | {% if layout != 'plain' %} 30 |
      31 |
      32 | {% endif %} 33 |
      34 | {% block body %} 35 | {% endblock body %} 36 |
      37 | {% if layout != 'plain' %} 38 |
      39 |
      40 | {% endif %} 41 |
      42 | {% include "footer.html" %} 43 | 44 | 45 | -------------------------------------------------------------------------------- /Dockerfile_ognibuild_dep: -------------------------------------------------------------------------------- 1 | # https://hub.docker.com/_/debian 2 | FROM docker.io/debian:testing-slim AS m4 3 | MAINTAINER Jelmer Vernooij 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt-get update --yes \ 8 | && apt-get install --yes --no-install-recommends \ 9 | auto-apt-proxy \ 10 | iproute2 \ 11 | && apt-get satisfy --yes --no-install-recommends \ 12 | apt-file \ 13 | aptitude \ 14 | && apt-get clean \ 15 | && apt-file update \ 16 | && apt-file search /usr/share/aclocal/.*.m4 --regex -l | xargs aptitude -y install 17 | 18 | 19 | 20 | # https://hub.docker.com/_/debian 21 | FROM docker.io/debian:testing-slim AS build 22 | MAINTAINER Jelmer Vernooij 23 | 24 | ARG DEBIAN_FRONTEND=noninteractive 25 | 26 | RUN apt-get update --yes \ 27 | && apt-get install --yes --no-install-recommends \ 28 | auto-apt-proxy \ 29 | iproute2 \ 30 | && apt-get satisfy --yes --no-install-recommends \ 31 | ca-certificates \ 32 | cargo \ 33 | gcc \ 34 | git \ 35 | libc6-dev \ 36 | libpython3-dev \ 37 | libssl-dev \ 38 | pkg-config \ 39 | python3-minimal \ 40 | && apt-get clean \ 41 | && git clone https://github.com/jelmer/ognibuild.git /code/ \ 42 | && cd /code/ \ 43 | && cargo build --verbose --release #-p dep-server 44 | 45 | 46 | 47 | # https://hub.docker.com/_/debian 48 | FROM docker.io/debian:testing-slim 49 | MAINTAINER Jelmer Vernooij 50 | 51 | ARG DEBIAN_FRONTEND=noninteractive 52 | 53 | RUN apt-get update --yes \ 54 | && apt-get install --yes --no-install-recommends \ 55 | auto-apt-proxy \ 56 | iproute2 \ 57 | && apt-get upgrade --yes \ 58 | && apt-get satisfy --yes --no-install-recommends \ 59 | libpython3-dev \ 60 | python3-breezy \ 61 | && apt-get clean \ 62 | && rm -rf /usr/share/aclocal 63 | 64 | COPY --from=m4 /usr/share/aclocal /usr/share/aclocal 65 | 66 | COPY --from=build /code/target/release/ /usr/local/bin/ 67 | 68 | EXPOSE 9934 69 | 70 | # $ janitor-ognibuild 71 | ENTRYPOINT ["dep-server", "--port=9934", "--listen-address=0.0.0.0"] 72 | -------------------------------------------------------------------------------- /py/janitor/worker_creds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # Copyright (C) 2018-2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | from typing import Optional 19 | 20 | import aiohttp 21 | from aiohttp import BasicAuth, web 22 | 23 | 24 | async def is_worker(db, request: web.Request) -> Optional[str]: 25 | auth_header = request.headers.get(aiohttp.hdrs.AUTHORIZATION) 26 | if not auth_header: 27 | return None 28 | auth = BasicAuth.decode(auth_header=auth_header) 29 | async with db.acquire() as conn: 30 | val = await conn.fetchval( 31 | "select 1 from worker where name = $1 AND password = crypt($2, password)", 32 | auth.login, 33 | auth.password, 34 | ) 35 | if val: 36 | return auth.login 37 | return None 38 | 39 | 40 | async def check_worker_creds(db, request: web.Request) -> Optional[str]: 41 | auth_header = request.headers.get(aiohttp.hdrs.AUTHORIZATION) 42 | if not auth_header: 43 | raise web.HTTPUnauthorized( 44 | text="worker login required", 45 | headers={"WWW-Authenticate": 'Basic Realm="Janitor"'}, 46 | ) 47 | login = await is_worker(db, request) 48 | if not login: 49 | raise web.HTTPUnauthorized( 50 | text="worker login required", 51 | headers={"WWW-Authenticate": 'Basic Realm="Janitor"'}, 52 | ) 53 | 54 | return login 55 | -------------------------------------------------------------------------------- /src/bin/janitor-schedule.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | 3 | #[derive(Parser)] 4 | struct Args { 5 | #[clap(long)] 6 | /// Create branches but don't push or propose anything. 7 | dry_run: bool, 8 | 9 | #[clap(long)] 10 | /// Prometheus push gateway to export to. 11 | prometheus: Option, 12 | 13 | #[clap(long, default_value = "janitor.conf")] 14 | /// Path to configuration. 15 | config: std::path::PathBuf, 16 | 17 | #[clap(long)] 18 | /// Restrict to a specific campaign. 19 | campaign: Option, 20 | 21 | /// Codebase to process. 22 | codebases: Vec, 23 | 24 | #[clap(long)] 25 | /// Bucket to use. 26 | bucket: Option, 27 | 28 | #[clap(long)] 29 | /// Requester to use. 30 | requester: Option, 31 | 32 | #[clap(long)] 33 | /// Refresh the queue. 34 | refresh: bool, 35 | 36 | #[clap(flatten)] 37 | logging: janitor::logging::LoggingArgs, 38 | } 39 | 40 | #[tokio::main] 41 | async fn main() -> Result<(), i32> { 42 | let args = Args::parse(); 43 | 44 | args.logging.init(); 45 | 46 | log::info!("Reading configuration"); 47 | 48 | let config = janitor::config::read_file(&args.config).unwrap(); 49 | 50 | let db = janitor::state::create_pool(&config).await.unwrap(); 51 | 52 | log::info!("Finding candidates with policy"); 53 | log::info!("Determining schedule for candidates"); 54 | let todo = janitor::schedule::iter_schedule_requests_from_candidates( 55 | &db, 56 | if args.codebases.is_empty() { 57 | None 58 | } else { 59 | Some(args.codebases.iter().map(|x| x.as_str()).collect()) 60 | }, 61 | args.campaign.as_deref(), 62 | ) 63 | .await 64 | .map_err(|e| { 65 | log::error!("Error: {}", e); 66 | 1 67 | })? 68 | .collect::>(); 69 | 70 | log::info!("Adding {} items to queue", todo.len()); 71 | 72 | janitor::schedule::bulk_add_to_queue( 73 | &db, 74 | &todo, 75 | args.dry_run, 76 | 0.0, 77 | args.bucket.as_deref(), 78 | args.requester.as_deref(), 79 | args.refresh, 80 | ) 81 | .await 82 | .unwrap(); 83 | Ok(()) 84 | } 85 | -------------------------------------------------------------------------------- /common-py/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Necessary since create_exception!() uses cfg!(feature = "gil-refs"), 2 | // but we don't have that feature. 3 | #![allow(unexpected_cfgs)] 4 | use pyo3::exceptions::PyValueError; 5 | use pyo3::prelude::*; 6 | 7 | mod artifacts; 8 | mod config; 9 | mod debdiff; 10 | mod io; 11 | mod logs; 12 | mod vcs; 13 | 14 | #[pyfunction] 15 | fn get_branch_vcs_type(branch: PyObject) -> PyResult { 16 | let branch = breezyshim::branch::GenericBranch::new(branch); 17 | janitor::vcs::get_branch_vcs_type(&branch) 18 | .map_err(|e| PyValueError::new_err((format!("{}", e),))) 19 | .map(|vcs| vcs.to_string()) 20 | } 21 | 22 | #[pyfunction] 23 | fn is_authenticated_url(url: &str) -> PyResult { 24 | Ok(janitor::vcs::is_authenticated_url( 25 | &url::Url::parse(url) 26 | .map_err(|e| PyValueError::new_err((format!("Invalid URL: {}", e),)))?, 27 | )) 28 | } 29 | 30 | #[pyfunction] 31 | fn is_alioth_url(url: &str) -> PyResult { 32 | Ok(janitor::vcs::is_alioth_url(&url::Url::parse(url).map_err( 33 | |e| PyValueError::new_err((format!("Invalid URL: {}", e),)), 34 | )?)) 35 | } 36 | 37 | #[pymodule] 38 | pub fn _common(py: Python, m: &Bound) -> PyResult<()> { 39 | pyo3_log::init(); 40 | m.add_function(wrap_pyfunction!(is_authenticated_url, m)?)?; 41 | m.add_function(wrap_pyfunction!(is_alioth_url, m)?)?; 42 | m.add_function(wrap_pyfunction!(get_branch_vcs_type, m)?)?; 43 | 44 | let artifactsm = pyo3::types::PyModule::new_bound(py, "artifacts")?; 45 | crate::artifacts::init(py, &artifactsm)?; 46 | m.add_submodule(&artifactsm)?; 47 | 48 | let vcsm = pyo3::types::PyModule::new_bound(py, "vcs")?; 49 | crate::vcs::init(py, &vcsm)?; 50 | m.add_submodule(&vcsm)?; 51 | 52 | let logsm = pyo3::types::PyModule::new_bound(py, "logs")?; 53 | crate::logs::init(py, &logsm)?; 54 | m.add_submodule(&logsm)?; 55 | 56 | let configm = pyo3::types::PyModule::new_bound(py, "config")?; 57 | crate::config::init(py, &configm)?; 58 | m.add_submodule(&configm)?; 59 | 60 | let debdiff = PyModule::new_bound(py, "debdiff")?; 61 | crate::debdiff::init_module(py, &debdiff)?; 62 | m.add_submodule(&debdiff)?; 63 | 64 | Ok(()) 65 | } 66 | -------------------------------------------------------------------------------- /helpers/render-publish-template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | import asyncio 5 | import logging 6 | import os 7 | import sys 8 | 9 | from janitor.config import read_config 10 | from janitor.publish_one import load_template_env 11 | 12 | sys.path.insert(0, os.path.dirname(__file__)) 13 | 14 | from janitor import state # noqa: E402 15 | from janitor.debian.debdiff import debdiff_is_empty, markdownify_debdiff # noqa: E402 16 | 17 | loop = asyncio.get_event_loop() 18 | 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument( 21 | "--config", type=str, default="janitor.conf", help="Path to configuration." 22 | ) 23 | parser.add_argument("-r", "--run-id", type=str, help="Run id to process") 24 | parser.add_argument("--role", type=str, help="Role", default="main") 25 | parser.add_argument("--format", type=str, choices=["md", "txt"], default="md") 26 | 27 | parser.add_argument("--template-env-path", type=str, help="Path to template env") 28 | 29 | args = parser.parse_args() 30 | 31 | logging.basicConfig(level=logging.INFO, format="%(message)s") 32 | 33 | try: 34 | with open(args.config) as f: 35 | config = read_config(f) 36 | except FileNotFoundError: 37 | parser.error(f"config path {args.config} does not exist") 38 | 39 | template_env = load_template_env(args.template_env_path) 40 | 41 | 42 | async def process_build(db_location, run_id, role, format): 43 | async with state.create_pool(db_location) as conn: 44 | query = """ 45 | SELECT 46 | package.name AS package, 47 | suite AS campaign, 48 | id AS log_id, 49 | result AS _result 50 | FROM run 51 | LEFT JOIN package ON run.codebase = package.codebase 52 | WHERE 53 | id = $1 54 | """ 55 | row = await conn.fetchrow(query, run_id) 56 | vs = {} 57 | vs.update(row) 58 | if row["_result"]: 59 | vs.update(row["_result"]) 60 | vs["external_url"] = "https://janitor.debian.net/" 61 | vs["markdownify_debdiff"] = markdownify_debdiff 62 | vs["debdiff_is_empty"] = debdiff_is_empty 63 | print(template_env.get_template(vs["suite"] + "." + format).render(vs)) 64 | 65 | 66 | loop.run_until_complete( 67 | process_build(config.database_location, args.run_id, args.role, args.format) 68 | ) 69 | -------------------------------------------------------------------------------- /py/janitor/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2018 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | 19 | import shlex 20 | from urllib.request import URLopener, build_opener, install_opener 21 | 22 | from breezy.transport import http as _mod_http 23 | from breezy.transport.http import urllib as _mod_urllib 24 | 25 | __version__ = (0, 1, 0) 26 | version_string = ".".join(map(str, __version__)) 27 | 28 | 29 | def get_core_schema() -> str: 30 | """Return the core janitor database schema SQL.""" 31 | import pkg_resources 32 | 33 | return pkg_resources.resource_string("janitor", "state.sql").decode("utf-8") 34 | 35 | 36 | def get_debian_schema() -> str: 37 | """Return the Debian-specific database schema SQL.""" 38 | import pkg_resources 39 | 40 | return pkg_resources.resource_string("janitor.debian", "debian.sql").decode("utf-8") 41 | 42 | 43 | def set_user_agent(user_agent): 44 | _mod_http.default_user_agent = lambda: user_agent 45 | _mod_urllib.AbstractHTTPHandler._default_headers["User-agent"] = user_agent 46 | URLopener.version = user_agent 47 | opener = build_opener() 48 | opener.addheaders = [("User-agent", user_agent)] 49 | install_opener(opener) 50 | 51 | 52 | CAMPAIGN_REGEX = "[a-z0-9-]+" 53 | 54 | 55 | def splitout_env(command): 56 | args = shlex.split(command) 57 | env = {} 58 | while len(args) > 0 and "=" in args[0]: 59 | (key, value) = args.pop(0).split("=", 1) 60 | env[key] = value 61 | return env, shlex.join(args) 62 | -------------------------------------------------------------------------------- /runner/src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use std::path::PathBuf; 3 | 4 | #[derive(Parser)] 5 | struct Args { 6 | #[clap(long, default_value = "localhost")] 7 | listen_address: String, 8 | 9 | #[clap(long, default_value = "9911")] 10 | port: u16, 11 | 12 | #[clap(long, default_value = "9919")] 13 | public_port: u16, 14 | 15 | #[clap(long)] 16 | /// Command to run to check codebase before pushing 17 | post_check: Option, 18 | 19 | #[clap(long)] 20 | /// Command to run to check whether to process codebase 21 | pre_check: Option, 22 | 23 | #[clap(long)] 24 | /// Use cached branches only. 25 | use_cached_only: bool, 26 | 27 | #[clap(long, default_value = "janitor.conf")] 28 | /// Path to configuration. 29 | config: Option, 30 | 31 | #[clap(long)] 32 | /// Backup directory to write files to if artifact or log manager is unreachable. 33 | backup_directory: Option, 34 | 35 | #[clap(long)] 36 | /// Public vcs location (used for URLs handed to worker) 37 | public_vcs_location: Option, 38 | 39 | #[clap(long)] 40 | /// Base location for our own APT archive 41 | public_apt_archive_location: Option, 42 | 43 | #[clap(long)] 44 | public_dep_server_url: Option, 45 | 46 | #[clap(flatten)] 47 | logging: janitor::logging::LoggingArgs, 48 | 49 | #[clap(long)] 50 | /// Print debugging info 51 | debug: bool, 52 | 53 | #[clap(long, default_value = "60")] 54 | /// Time before marking a run as having timed out (minutes) 55 | run_timeout: u64, 56 | 57 | #[clap(long)] 58 | /// Avoid processing runs on a host (e.g. 'salsa.debian.org') 59 | avoid_host: Vec, 60 | } 61 | 62 | #[tokio::main] 63 | async fn main() -> Result<(), i32> { 64 | let args = Args::parse(); 65 | 66 | args.logging.init(); 67 | 68 | let state = Arc::new(AppState {}); 69 | 70 | let app = janitor_runner::web::app(state.clone()); 71 | 72 | // Run it 73 | let addr = SocketAddr::new(args.listen_address, args.port); 74 | log::info!("listening on {}", addr); 75 | 76 | let listener = tokio::net::TcpListener::bind(addr).await?; 77 | axum::serve(listener, app.into_make_service()).await?; 78 | 79 | Ok(()) 80 | } 81 | -------------------------------------------------------------------------------- /common-py/src/debdiff.rs: -------------------------------------------------------------------------------- 1 | use pyo3::create_exception; 2 | use pyo3::prelude::*; 3 | use pyo3::types::PyBytes; 4 | 5 | #[pyfunction] 6 | fn debdiff_is_empty(debdiff: &str) -> PyResult { 7 | Ok(janitor::debdiff::debdiff_is_empty(debdiff)) 8 | } 9 | 10 | #[pyfunction] 11 | fn filter_boring(debdiff: &str, old_version: &str, new_version: &str) -> PyResult { 12 | Ok(janitor::debdiff::filter_boring( 13 | debdiff, 14 | old_version, 15 | new_version, 16 | )) 17 | } 18 | 19 | #[pyfunction] 20 | fn section_is_wdiff(title: &str) -> PyResult<(bool, Option<&str>)> { 21 | Ok(janitor::debdiff::section_is_wdiff(title)) 22 | } 23 | 24 | #[pyfunction] 25 | fn markdownify_debdiff(debdiff: &str) -> PyResult { 26 | Ok(janitor::debdiff::markdownify_debdiff(debdiff)) 27 | } 28 | 29 | #[pyfunction] 30 | fn htmlize_debdiff(debdiff: &str) -> PyResult { 31 | Ok(janitor::debdiff::htmlize_debdiff(debdiff)) 32 | } 33 | 34 | create_exception!( 35 | janitor.debian.debdiff, 36 | DebdiffError, 37 | pyo3::exceptions::PyException 38 | ); 39 | 40 | #[pyfunction] 41 | fn run_debdiff<'a>( 42 | py: Python<'a>, 43 | old_binaries: Vec, 44 | new_binaries: Vec, 45 | ) -> PyResult> { 46 | pyo3_async_runtimes::tokio::future_into_py(py, async move { 47 | let r = janitor::debdiff::run_debdiff( 48 | old_binaries.iter().map(|x| x.as_str()).collect::>(), 49 | new_binaries.iter().map(|x| x.as_str()).collect::>(), 50 | ) 51 | .await 52 | .map_err(|e| DebdiffError::new_err((e.to_string(),)))?; 53 | 54 | Ok(Python::with_gil(|py| { 55 | PyBytes::new_bound(py, &r).to_object(py) 56 | })) 57 | }) 58 | } 59 | 60 | pub(crate) fn init_module(py: Python, m: &Bound) -> PyResult<()> { 61 | m.add_function(wrap_pyfunction!(debdiff_is_empty, m)?)?; 62 | m.add_function(wrap_pyfunction!(filter_boring, m)?)?; 63 | m.add_function(wrap_pyfunction!(section_is_wdiff, m)?)?; 64 | m.add_function(wrap_pyfunction!(markdownify_debdiff, m)?)?; 65 | m.add_function(wrap_pyfunction!(htmlize_debdiff, m)?)?; 66 | m.add_function(wrap_pyfunction!(run_debdiff, m)?)?; 67 | m.add("DebdiffError", py.get_type_bound::())?; 68 | Ok(()) 69 | } 70 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/broken-merge-proposals.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% from "run_util.html" import display_result_code %} 6 | {% block page_title %} 7 | Cupboard - Merge Proposals with Broken Runs 8 | {% endblock page_title %} 9 | {% block body %} 10 |
      11 |

      Merge Proposals With Broken Runs

      12 |

      13 | This is an overview of all open merge proposals for which the 14 | last relevant run failed. 15 |

      16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | {% for mp_url, campaign, codebase, run_id, result_code, finish_time, description in broken_mps %} 29 | 30 | 33 | 36 | 39 | 43 | 44 | 45 | 46 | {% endfor %} 47 | 48 |
      Merge ProposalCodebaseSuiteResult codeFinish timeDescription
      31 | Merge Proposal 32 | 34 | {{ codebase }} 35 | 37 | {{ campaign }} 38 | 40 | {{ display_result_code(result_code) }} 42 | {{ format_timestamp(finish_time) }}{{ description }}
      49 |
      50 | {% endblock body %} 51 | -------------------------------------------------------------------------------- /worker/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "janitor-worker" 3 | version = "0.0.0" 4 | authors = ["Jelmer Vernooij "] 5 | edition.workspace = true 6 | description = "Worker for the janitor" 7 | license = "GPL-3.0+" 8 | repository = "https://github.com/jelmer/janitor.git" 9 | homepage = "https://github.com/jelmer/janitor" 10 | default-run = "janitor-worker" 11 | 12 | [dependencies] 13 | url = { workspace = true, features = ["serde"] } 14 | tokio = { workspace = true, features = ["full"] } 15 | reqwest = { workspace = true, features = ["json", "multipart"] } 16 | backoff = { version = "0.4", features = ["tokio"] } 17 | serde_json = "1" 18 | log = "0.4" 19 | serde = { version = "1.0", features = ["derive"] } 20 | chrono = { version = "0.4", features = ["serde"] } 21 | clap = { workspace = true, features = ["derive", "env"], optional = true } 22 | pyo3 = { workspace = true, features = ["auto-initialize", "serde"], optional = true } 23 | janitor = { path = ".." } 24 | pyo3-log = { workspace = true, optional = true } 25 | breezyshim.workspace = true 26 | silver-platter = { workspace = true, features = ["debian"] } 27 | debian-analyzer = { workspace = true, optional = true } 28 | shlex = "1.3.0" 29 | axum = "0.7.9" # Depends on axum-core v0.4.5, which is the same for askama_axum 30 | askama_axum = { version = "0.4.0", features = ["serde-json", "serde-yaml", "humansize", "urlencode", "markdown"] } 31 | nix = { version = "0.30.0", features = ["fs"] } 32 | percent-encoding = "2.3.1" 33 | maplit = "1.0.2" 34 | tempfile = "3.20.0" 35 | prometheus = "0.14.0" 36 | askama = "0.12.1" 37 | gethostname = "1.0.2" 38 | ognibuild = { workspace = true } 39 | debversion = { workspace = true, optional = true } 40 | debian-changelog = { workspace = true, optional = true } 41 | tokio-util = "0.7.14" 42 | 43 | [features] 44 | default = ["debian", "cli"] 45 | debian = ["dep:debversion", "dep:debian-analyzer", "dep:debian-changelog", "janitor/debian"] 46 | cli = ["dep:clap", "dep:pyo3", "dep:pyo3-log"] 47 | 48 | [dev-dependencies] 49 | http-body-util = "0.1.3" 50 | hyper = "1.6.0" 51 | serial_test = "3.1.1" 52 | tempfile = "3.20.0" 53 | test-log = "0.2.17" 54 | tower = { version = "0.5.2", features = ["util"] } 55 | 56 | [[bin]] 57 | name = "janitor-worker" 58 | path = "src/bin/worker.rs" 59 | required-features = ["cli"] 60 | 61 | [[bin]] 62 | name = "janitor-dist" 63 | path = "src/bin/dist.rs" 64 | required-features = ["cli"] 65 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/start.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard 7 | {% endblock page_title %} 8 | {% block body %} 9 |
      10 |

      Cupboard

      11 |
      12 | 51 |
      52 |
      53 | {% endblock body %} 54 | -------------------------------------------------------------------------------- /tests/test_artifacts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2021 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import os 19 | import shutil 20 | import tempfile 21 | 22 | import pytest 23 | 24 | from janitor.artifacts import ArtifactsMissing, LocalArtifactManager 25 | 26 | 27 | @pytest.fixture 28 | def local_artifact_manager(): 29 | path = tempfile.mkdtemp() 30 | try: 31 | yield LocalArtifactManager(path) 32 | finally: 33 | shutil.rmtree(path) 34 | 35 | 36 | @pytest.mark.asyncio 37 | async def test_store_twice(local_artifact_manager): 38 | manager = local_artifact_manager 39 | with tempfile.TemporaryDirectory() as td: 40 | with open(os.path.join(td, "somefile"), "w") as f: 41 | f.write("lalala") 42 | await manager.store_artifacts("some-run-id", td) 43 | await manager.store_artifacts("some-run-id", td) 44 | 45 | 46 | @pytest.mark.asyncio 47 | async def test_store_and_retrieve(local_artifact_manager): 48 | manager = local_artifact_manager 49 | with tempfile.TemporaryDirectory() as td: 50 | with open(os.path.join(td, "somefile"), "w") as f: 51 | f.write("lalala") 52 | await manager.store_artifacts("some-run-id", td) 53 | with tempfile.TemporaryDirectory() as td: 54 | await manager.retrieve_artifacts("some-run-id", td) 55 | assert ["somefile"] == os.listdir(td) 56 | 57 | 58 | @pytest.mark.asyncio 59 | async def test_retrieve_nonexistent(local_artifact_manager): 60 | manager = local_artifact_manager 61 | with tempfile.TemporaryDirectory() as td: 62 | with pytest.raises(ArtifactsMissing): 63 | await manager.retrieve_artifacts("some-run-id", td) 64 | -------------------------------------------------------------------------------- /py/janitor/review.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # Copyright (C) 2018 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | from typing import Optional 19 | 20 | from yarl import URL 21 | 22 | from .schedule import do_schedule 23 | 24 | 25 | async def store_review( 26 | conn, 27 | session, 28 | runner_url, 29 | run_id: str, 30 | verdict: str, 31 | comment: Optional[str], 32 | reviewer: Optional[str], 33 | is_qa_reviewer: bool, 34 | ): 35 | async with conn.transaction(): 36 | if verdict == "reschedule": 37 | verdict = "rejected" 38 | 39 | run = await conn.fetchrow( 40 | "SELECT suite, codebase FROM run WHERE id = $1", run_id 41 | ) 42 | await do_schedule( 43 | conn, 44 | campaign=run["suite"], 45 | refresh=True, 46 | requester=f"reviewer ({reviewer})", 47 | bucket="default", 48 | codebase=run["codebase"], 49 | ) 50 | 51 | if verdict != "abstained" and is_qa_reviewer: 52 | async with session.post( 53 | URL(runner_url) / "runs" / run_id, 54 | json={"publish_status": verdict}, 55 | raise_for_status=True, 56 | ): 57 | pass 58 | await conn.execute( 59 | "INSERT INTO review (run_id, comment, reviewer, verdict) VALUES " 60 | " ($1, $2, $3, $4) ON CONFLICT (run_id, reviewer) " 61 | "DO UPDATE SET verdict = EXCLUDED.verdict, comment = EXCLUDED.comment, " 62 | "reviewed_at = NOW()", 63 | run_id, 64 | comment, 65 | reviewer, 66 | verdict, 67 | ) 68 | -------------------------------------------------------------------------------- /tests/test_archive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | 19 | import hashlib 20 | import os 21 | from tempfile import TemporaryDirectory 22 | 23 | from debian.deb822 import Release 24 | 25 | from janitor.config import read_string as read_config_string 26 | from janitor.debian.archive import HashedFileWriter, create_app 27 | 28 | 29 | async def create_client(aiohttp_client, config=None): 30 | if config is None: 31 | config = read_config_string("") 32 | return await aiohttp_client( 33 | await create_app(None, config, "/tmp", None, gpg_context=None) 34 | ) 35 | 36 | 37 | async def test_health(aiohttp_client): 38 | client = await create_client(aiohttp_client) 39 | 40 | resp = await client.get("/health") 41 | assert resp.status == 200 42 | text = await resp.text() 43 | assert text == "ok" 44 | 45 | 46 | async def test_ready(aiohttp_client): 47 | client = await create_client(aiohttp_client) 48 | 49 | resp = await client.get("/ready") 50 | assert resp.status == 200 51 | text = await resp.text() 52 | assert text == "" 53 | 54 | 55 | def test_hash_file_writer(): 56 | with TemporaryDirectory() as td: 57 | r = Release() 58 | with HashedFileWriter(r, td, "foo/bar") as w: 59 | w.write(b"chunk1") 60 | w.write(b"chunk2") 61 | w.done() 62 | md5hex = hashlib.md5(b"chunk1chunk2").hexdigest() 63 | with open(os.path.join(td, "foo", "by-hash", "MD5Sum", md5hex), "rb") as f: 64 | assert f.read() == b"chunk1chunk2" 65 | with open(os.path.join(td, "foo", "bar"), "rb") as f: 66 | assert f.read() == b"chunk1chunk2" 67 | assert r["MD5Sum"] == [{"md5sum": md5hex, "name": "foo/bar", "size": 12}] 68 | -------------------------------------------------------------------------------- /tests/test_logs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import os 19 | import tempfile 20 | from datetime import datetime 21 | 22 | import pytest 23 | 24 | from janitor.logs import FileSystemLogFileManager, GCSLogFileManager, S3LogFileManager 25 | 26 | 27 | def test_s3_log_file_manager(): 28 | S3LogFileManager("https://some-url/") 29 | 30 | 31 | def test_gcs_log_file_manager(): 32 | GCSLogFileManager("gs://foo/") 33 | 34 | 35 | async def test_file_log_file_manager(): 36 | with tempfile.TemporaryDirectory() as td: 37 | async with FileSystemLogFileManager(td) as lm: 38 | assert not await lm.has_log("mypkg", "run-id", "foo.log") 39 | with pytest.raises(FileNotFoundError): 40 | await lm.get_log("mypkg", "run-id", "foo.log") 41 | with pytest.raises(FileNotFoundError): 42 | await lm.get_ctime("mypkg", "run-id", "foo.log") 43 | with pytest.raises(FileNotFoundError): 44 | await lm.delete_log("mypkg", "run-id", "foo.log") 45 | with tempfile.NamedTemporaryFile(suffix=".log") as f: 46 | f.write(b"foo bar\n") 47 | f.flush() 48 | await lm.import_log("mypkg", "run-id", f.name) 49 | logname = os.path.basename(f.name) 50 | assert await lm.has_log("mypkg", "run-id", logname) 51 | assert (await lm.get_log("mypkg", "run-id", logname)).read() == b"foo bar\n" 52 | assert isinstance(await lm.get_ctime("mypkg", "run-id", logname), datetime) 53 | assert [x async for x in lm.iter_logs()] == [("mypkg", "run-id", [logname])] 54 | await lm.delete_log("mypkg", "run-id", logname) 55 | assert not await lm.has_log("mypkg", "run-id", logname) 56 | -------------------------------------------------------------------------------- /differ-py/src/lib.rs: -------------------------------------------------------------------------------- 1 | use pyo3::exceptions::{PyRuntimeError, PyTimeoutError, PyValueError}; 2 | use pyo3::prelude::*; 3 | 4 | #[pyfunction] 5 | #[pyo3(signature = (old_binaries, new_binaries, timeout = None, memory_limit = None, diffoscope_command = None))] 6 | fn run_diffoscope<'a>( 7 | py: Python<'a>, 8 | old_binaries: Vec<(String, String)>, 9 | new_binaries: Vec<(String, String)>, 10 | timeout: Option, 11 | memory_limit: Option, 12 | diffoscope_command: Option, 13 | ) -> PyResult> { 14 | pyo3_async_runtimes::tokio::future_into_py(py, async move { 15 | let old_binaries = old_binaries 16 | .iter() 17 | .map(|(path, hash)| (path.as_str(), hash.as_str())) 18 | .collect::>(); 19 | let new_binaries = new_binaries 20 | .iter() 21 | .map(|(path, hash)| (path.as_str(), hash.as_str())) 22 | .collect::>(); 23 | 24 | let o = janitor_differ::diffoscope::run_diffoscope( 25 | old_binaries.as_slice(), 26 | new_binaries.as_slice(), 27 | timeout, 28 | memory_limit, 29 | diffoscope_command.as_deref(), 30 | ) 31 | .await 32 | .map_err(|e| match e { 33 | janitor_differ::diffoscope::DiffoscopeError::Timeout => { 34 | PyTimeoutError::new_err("Diffoscope timed out") 35 | } 36 | janitor_differ::diffoscope::DiffoscopeError::Io(e) => e.into(), 37 | janitor_differ::diffoscope::DiffoscopeError::Other(e) => PyRuntimeError::new_err(e), 38 | janitor_differ::diffoscope::DiffoscopeError::Serde(e) => { 39 | PyValueError::new_err(e.to_string()) 40 | } 41 | })?; 42 | Ok(Python::with_gil(|py| o.to_object(py))) 43 | }) 44 | } 45 | 46 | #[pyfunction] 47 | fn filter_boring_udiff( 48 | udiff: &str, 49 | old_version: &str, 50 | new_version: &str, 51 | display_version: &str, 52 | ) -> PyResult { 53 | let o = janitor_differ::diffoscope::filter_boring_udiff( 54 | udiff, 55 | old_version, 56 | new_version, 57 | display_version, 58 | ) 59 | .map_err(|e| PyValueError::new_err(e.to_string()))?; 60 | Ok(o) 61 | } 62 | 63 | #[pymodule] 64 | pub fn _differ(m: &Bound) -> PyResult<()> { 65 | pyo3_log::init(); 66 | 67 | m.add_function(wrap_pyfunction!(run_diffoscope, m)?)?; 68 | m.add_function(wrap_pyfunction!(filter_boring_udiff, m)?)?; 69 | 70 | Ok(()) 71 | } 72 | -------------------------------------------------------------------------------- /docs/flow.md: -------------------------------------------------------------------------------- 1 | Package metadata 2 | ================ 3 | 4 | Package metadata contains mostly static information about a package, imported 5 | straight from the archive. This includes the package name, maintainer email, 6 | uploader emails (if any) as well as the version control information 7 | (vcs type, URL, subpath) and optionally popularity. 8 | 9 | The "schedule" job regularly imports package metadata. On Debian, this information 10 | comes from UDD. On other Debian-like distributions, it's imported from the 11 | apt sources file. 12 | 13 | The importing has two components: 14 | 15 | * A script that can output Package() protobufs (see janitor/package_metadata.proto) to standard out 16 | * An importer that reads these protobufs on standard in and updates the database (janitor.package_metadata) 17 | 18 | Candidates 19 | ========== 20 | 21 | Once the janitor knows about a package, candidates can be created. A candidate 22 | is a bit of data that a particular suite (TODO: better name) (e.g. lintian-fixes) 23 | can be run on a particular package and that there is some chance it will yield 24 | changes. 25 | 26 | Candidates include information like: 27 | 28 | * value: a relative number that explains how useful this change would be 29 | * success_chance: an estimate of how likely this change is to succeed and result in a build 30 | * context: some indicator of the current state of the world. Used to avoid retrying 31 | builds if nothing has really changed. e.g. for new upstream releases, this 32 | is the upstream version number of the latest release 33 | 34 | Like package metadata, candidates are generated by a script that writes 35 | YAML to standard output. Candidate generation scripts 36 | can be really complicated - allowing for more optimal scheduling - or really 37 | simple, in which case they just output a candidate for each package in a suite 38 | with fixed settings for value and succes_chance. 39 | 40 | Scheduling 41 | ========== 42 | 43 | Once candidates have been created, the schedule job (``janitor.schedule``) 44 | inserts new entries into the queue, taking into account a variety of factors: 45 | 46 | * success chance 47 | * value 48 | * popularity of the package if known (from popcon) 49 | * previous success rate (for the suite/package combination and the package itself) 50 | * previous run duration 51 | * whether the context has changed since the last run 52 | 53 | The queue consists of prioritized buckets. Manually requested runs, runs triggered 54 | by the publisher (e.g. to resolve merge conflicts) and retried runs are always 55 | executed before runs that were scheduled by the scheduler. 56 | -------------------------------------------------------------------------------- /tests/test_debdiff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2019 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | from janitor.debian.debdiff import filter_boring 19 | 20 | 21 | def test_just_versions(): 22 | debdiff = """\ 23 | File lists identical (after any substitutions) 24 | 25 | Control files of package acpi-fakekey: lines which differ (wdiff format) 26 | ------------------------------------------------------------------------ 27 | Version: [-0.143-4~jan+unchanged1-] {+0.143-5~jan+lint1+} 28 | 29 | Control files of package acpi-fakekey-dbgsym: lines which differ (wdiff format) 30 | ------------------------------------------------------------------------------- 31 | Depends: acpi-fakekey (= [-0.143-4~jan+unchanged1)-] {+0.143-5~jan+lint1)+} 32 | Version: [-0.143-4~jan+unchanged1-] {+0.143-5~jan+lint1+} 33 | 34 | Control files of package acpi-support: lines which differ (wdiff format) 35 | ------------------------------------------------------------------------ 36 | Version: [-0.143-4~jan+unchanged1-] {+0.143-5~jan+lint1+} 37 | 38 | Control files of package acpi-support-base: lines which differ (wdiff format) 39 | ----------------------------------------------------------------------------- 40 | Version: [-0.143-4~jan+unchanged1-] {+0.143-5~jan+lint1+} 41 | """ 42 | newdebdiff = filter_boring(debdiff, "0.143-4~jan+unchanged1", "0.143-5~jan+lint1") 43 | assert ( 44 | newdebdiff 45 | == """\ 46 | File lists identical (after any substitutions) 47 | 48 | No differences were encountered between the control files of package \ 49 | acpi-fakekey 50 | 51 | No differences were encountered between the control files of package \ 52 | acpi-fakekey-dbgsym 53 | 54 | No differences were encountered between the control files of package \ 55 | acpi-support 56 | 57 | No differences were encountered between the control files of package \ 58 | acpi-support-base 59 | """ 60 | ) 61 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/done-list.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard - Changes That Have Been Merged or Pushed 7 | {% endblock page_title %} 8 | {% block body %} 9 |
      10 |

      Changes that have been merged or pushed

      11 |
      12 | 13 | 19 | 20 |
      21 | {% set ns = namespace(last_date=None, run_date=None) %} 22 |
        23 | {% for run in runs %} 24 | {% if run.absorbed_at %} 25 | {% set ns.run_date = run.absorbed_at.date().isoformat() %} 26 | {% else %} 27 | {% set ns.run_date = "unknown" %} 28 | {% endif %} 29 | {% if ns.run_date != ns.last_date %} 30 | {% if ns.last_date %}
      {% endif %} 31 | {% set ns.last_date = ns.run_date %} 32 |

      {{ ns.last_date }}

      33 |
        34 | {% endif %} 35 |
      • 36 | {{ run.codebase }} 37 | {% if run.merged_by %} 38 | (merged by 39 | {% if run.merged_by_url %} 40 | {{ run.merged_by }} 41 | {% else %} 42 | {{ run.merged_by }} 43 | {% endif %} 44 | ) 45 | {% else %} 46 | (pushed) 47 | {% endif %} 48 | {% set command = run.command -%} 49 | {% set result = run.result -%} 50 | {% include [run.campaign + "/summary.html", "generic/summary.html"] %} 51 |
        • 52 | {% endfor %} 53 |
      54 |
      55 | {% endblock body %} 56 | -------------------------------------------------------------------------------- /tests/test_site_macros.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (C) 2022 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | from jinja2 import Environment, select_autoescape 19 | 20 | from janitor.site import template_loader 21 | 22 | env = Environment(loader=template_loader, autoescape=select_autoescape(["html", "xml"])) 23 | 24 | 25 | def test_display_branch_url(): 26 | template = env.get_template("run_util.html") 27 | assert ( 28 | str( 29 | template.module.display_branch_url( # type: ignore 30 | None, "https://github.com/jelmer/example.git" 31 | ) 32 | ) 33 | == """\ 34 | 35 | 36 | https://github.com/jelmer/example.git 37 | 38 | """ 39 | ) 40 | assert ( 41 | str( 42 | template.module.display_branch_url( # type: ignore 43 | "https://github.com/jelmer/example.git", 44 | "https://github.com/jelmer/example", 45 | ) 46 | ) 47 | == """\ 48 | 49 | 50 | https://github.com/jelmer/example 51 | 52 | """ 53 | ) 54 | 55 | 56 | def test_display_publish_blockers(): 57 | template = env.get_template("run_util.html") 58 | assert ( 59 | str( 60 | template.module.display_publish_blockers( # type: ignore 61 | {} 62 | ) 63 | ) 64 | == """\ 65 | 66 |
        67 | 68 |
      69 | """ 70 | ) 71 | assert ( 72 | str( 73 | template.module.display_publish_blockers( # type: ignore 74 | {"inactive": {"result": True, "details": {}}} 75 | ) 76 | ) 77 | == """\ 78 | 79 |
        80 | 81 |
      • ☑ 82 | codebase is not inactive
        • 83 | 84 |
      85 | """ 86 | ) 87 | -------------------------------------------------------------------------------- /reprocess-build-results.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (C) 2019-2020 Jelmer Vernooij 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | 19 | import argparse 20 | import asyncio 21 | import logging 22 | import sys 23 | 24 | from aiohttp import ClientSession 25 | from yarl import URL 26 | 27 | loop = asyncio.get_event_loop() 28 | 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument( 31 | "--log-timeout", 32 | type=int, 33 | default=60, 34 | help="Default timeout when retrieving log files.", 35 | ) 36 | parser.add_argument( 37 | "-r", "--run-id", type=str, action="append", help="Run id to process" 38 | ) 39 | parser.add_argument( 40 | "--reschedule", 41 | action="store_true", 42 | help="Schedule rebuilds for runs for which result code has changed.", 43 | ) 44 | parser.add_argument("--dry-run", action="store_true") 45 | parser.add_argument( 46 | "--base-url", type=str, default="https://janitor.debian.net", help="Instance URL" 47 | ) 48 | 49 | args = parser.parse_args() 50 | 51 | logging.basicConfig(level=logging.INFO, format="%(message)s") 52 | 53 | 54 | async def reprocess_logs(base_url, run_ids=None, dry_run=False, reschedule=False): 55 | params = {} 56 | if dry_run: 57 | params["dry_run"] = "1" 58 | if reschedule: 59 | params["reschedule"] = "1" 60 | if run_ids: 61 | params["run_ids"] = run_ids 62 | url = URL(base_url) / "cupboard/api/mass-reschedule" 63 | async with ClientSession() as session, session.post(url, params=params) as resp: 64 | if resp.status != 200: 65 | logging.fatal("rescheduling failed: %d", resp.status) 66 | return 1 67 | for entry in await resp.json(): 68 | logging.info("%r", entry) 69 | 70 | 71 | sys.exit( 72 | asyncio.run( 73 | reprocess_logs( 74 | args.base_url, args.run_id, dry_run=args.dry_run, reschedule=args.reschedule 75 | ) 76 | ) 77 | ) 78 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DOCKER_TAG ?= latest 2 | PYTHON ?= python3 3 | SHA = $(shell git rev-parse HEAD) 4 | DOCKERFILES = $(shell ls Dockerfile_* | sed 's/Dockerfile_//' ) 5 | DOCKER_TARGETS := $(patsubst %,docker-%,$(DOCKERFILES)) 6 | BUILD_TARGETS := $(patsubst %,build-%,$(DOCKERFILES)) 7 | PUSH_TARGETS := $(patsubst %,push-%,$(DOCKERFILES)) 8 | 9 | .PHONY: all check 10 | 11 | build-inplace: 12 | $(PYTHON) setup.py build_ext -i 13 | 14 | all: core 15 | 16 | core: py/janitor/site/_static/pygments.css build-inplace 17 | 18 | check:: typing 19 | 20 | check:: test 21 | 22 | check:: style 23 | 24 | check:: ruff 25 | 26 | check:: check-format 27 | 28 | check-format:: check-ruff-format 29 | 30 | check-ruff-format: 31 | ruff format --check py tests 32 | 33 | check-format:: check-cargo-format 34 | 35 | check-cargo-format: 36 | cargo fmt --check --all 37 | 38 | ruff: 39 | ruff check py tests 40 | 41 | fix:: ruff-fix 42 | 43 | fix:: clippy-fix 44 | 45 | fix:: reformat 46 | 47 | clippy-fix: 48 | cargo clippy --fix --allow-dirty --allow-staged 49 | 50 | ruff-fix: 51 | ruff check --fix . 52 | 53 | reformat-ruff: 54 | ruff format py tests 55 | 56 | reformat:: reformat-ruff 57 | 58 | reformat:: 59 | cargo fmt --all 60 | 61 | suite-references: 62 | git grep "\\(lintian-brush\|lintian-fixes\|debianize\|fresh-releases\|fresh-snapshots\\)" | grep -v .example 63 | 64 | test:: build-inplace 65 | PYTHONPATH=$(shell pwd)/py:$(PYTHONPATH) PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python $(PYTHON) -m pytest -vv tests 66 | 67 | test:: 68 | cargo test 69 | 70 | style:: yamllint 71 | 72 | yamllint: 73 | yamllint -s .github/ 74 | 75 | style:: djlint 76 | 77 | check-format:: check-html-format 78 | 79 | check-html-format: 80 | djlint --check py/janitor/site/templates/ 81 | 82 | djlint: 83 | djlint py/janitor/site/templates 84 | 85 | typing: 86 | $(PYTHON) -m mypy py/janitor tests 87 | 88 | py/janitor/site/_static/pygments.css: 89 | pygmentize -S default -f html > $@ 90 | 91 | clean: 92 | 93 | docker-%: 94 | $(MAKE) build-$* 95 | $(MAKE) push-$* 96 | 97 | build-%: 98 | buildah build --no-cache -t ghcr.io/jelmer/janitor/$*:$(DOCKER_TAG) -t ghcr.io/jelmer/janitor/$*:$(SHA) -f Dockerfile_$* . 99 | 100 | push-%: 101 | buildah push ghcr.io/jelmer/janitor/$*:$(DOCKER_TAG) 102 | buildah push ghcr.io/jelmer/janitor/$*:$(SHA) 103 | 104 | docker-all: $(DOCKER_TARGETS) 105 | 106 | build-all: $(BUILD_TARGETS) 107 | 108 | push-all: $(PUSH_TARGETS) 109 | 110 | reformat:: reformat-html 111 | 112 | reformat-html: 113 | djlint --reformat py/janitor/site/templates/ 114 | 115 | codespell: 116 | codespell 117 | -------------------------------------------------------------------------------- /py/janitor/site/templates/credentials.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block page_title %} 3 | Credentials 4 | {% endblock page_title %} 5 | {% block body %} 6 |
      7 |

      Credentials

      8 |
      9 |

      User accounts on hosting sites

      10 |
      11 |
      12 |
        13 | {% for h in hosting %} 14 | {% if h.user %} 15 |
      • 16 | {{ h.user }} on {{ h.name }} 17 |
        • 18 | {% endif %} 19 | {% endfor %} 20 |
      21 |
      22 |
      23 |
      24 |
      25 |

      PGP

      26 |

      27 | These can also be downloaded in plain format from /pgp_keys.asc 28 |

      29 | {% macro pgp_flags(k) %} 30 | {% if k.can_certify %}C{% endif %} 31 | {% if k.can_sign %}S{% endif %} 32 | {% if k.can_encrypt %}E{% endif %} 33 | {% endmacro %} 34 | {% for k in pgp_keys %} 35 |
      36 |
      37 | 
      pub    [{{ pgp_flags(k) }}] {{ k.fpr }}
38 | {% for uid in k.uids %}uid           [{{ pgp_validity(uid.validity) }}] {{ uid.name }} <{{ uid.email }}>
39 | {% endfor %}{% for sk in k.subkeys %}sub   {{ pgp_algo(sk.pubkey_algo) }}{{ sk.length }} {{ format_pgp_date(sk.timestamp) }} [{{ pgp_flags(sk) }}] {% if sk.expires %}[expires: {{ format_pgp_date(sk.expires) }}]{% endif %}
40 | {% endfor %}
41 |
      42 |
      43 | {% endfor %} 44 |
      45 |
      46 |
      47 |

      SSH

      48 |

      49 | These can also be downloaded in plain format from /ssh_keys 50 |

      51 |
      52 |
      53 | 
      {{ ssh_keys|join('\n') }}
54 |
      55 |
      56 |
      57 |
      58 |
      59 | {% endblock body %} 60 | -------------------------------------------------------------------------------- /py/janitor/site/pubsub.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # Copyright (C) 2018 Jelmer Vernooij 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 | 18 | import asyncio 19 | import json 20 | 21 | from aiohttp import web 22 | from aiohttp_openmetrics import Gauge 23 | 24 | subscription_count = Gauge( 25 | "subscriptions", "Subscriptions per topic", labelnames=("topic",) 26 | ) 27 | 28 | 29 | class Subscription: 30 | """A pubsub subscription.""" 31 | 32 | def __init__(self, topic: "Topic") -> None: 33 | self.topic = topic 34 | self.queue: asyncio.Queue = asyncio.Queue() 35 | if topic.last: 36 | self.queue.put_nowait(topic.last) 37 | 38 | def __enter__(self): 39 | self.topic.subscriptions.add(self.queue) 40 | subscription_count.labels(self.topic.name).inc() 41 | return self.queue 42 | 43 | def __exit__(self, type, value, traceback): 44 | subscription_count.labels(self.topic.name).dec() 45 | self.topic.subscriptions.remove(self.queue) 46 | 47 | 48 | class Topic: 49 | """A pubsub topic.""" 50 | 51 | def __init__(self, name, repeat_last: bool = False) -> None: 52 | self.name = name 53 | self.subscriptions: set[asyncio.Queue] = set() 54 | self.last = None 55 | self.repeat_last = repeat_last 56 | 57 | def publish(self, message): 58 | if self.repeat_last: 59 | self.last = message 60 | for queue in self.subscriptions: 61 | queue.put_nowait(message) 62 | 63 | 64 | async def pubsub_handler(topic: Topic, request) -> web.WebSocketResponse: 65 | ws = web.WebSocketResponse() 66 | await ws.prepare(request) 67 | 68 | with Subscription(topic) as queue: 69 | while True: 70 | msg = await queue.get() 71 | try: 72 | await ws.send_str(json.dumps(msg)) 73 | except TypeError as e: 74 | raise TypeError(f"not jsonable: {msg!r}") from e 75 | except ConnectionResetError: 76 | break 77 | 78 | return ws 79 | -------------------------------------------------------------------------------- /py/janitor/site/templates/cupboard/never-processed.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% include "cupboard/sidebar.html" %} 4 | {% endblock sidebar %} 5 | {% block page_title %} 6 | Cupboard - Never Processed 7 | {% if campaign %}- for {{ campaign }}{% endif %} 8 | {% endblock page_title %} 9 | {% block body %} 10 |
      11 |

      never-processed

      12 | {% if is_admin %} 13 |
      16 | {% if campaign %}{% endif %} 17 | 18 | 19 |
      20 | 30 | {% endif %} 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | {% for codebase, campaign in never_processed %} 44 | 45 | 48 | 51 | 52 | {% endfor %} 53 | 54 |
      CodebaseCampaign
      46 | {{ codebase }} 47 | 49 | {{ campaign }} 50 |
      55 | 64 |
      65 | {% endblock body %} 66 | -------------------------------------------------------------------------------- /worker/src/tee.rs: -------------------------------------------------------------------------------- 1 | use nix::unistd::{close, dup, dup2}; 2 | use std::fs::File; 3 | use std::io::{self, Write}; 4 | use std::os::unix::io::{AsRawFd, RawFd}; 5 | use std::process::{Command, Stdio}; 6 | 7 | pub struct CopyOutput { 8 | old_stdout: RawFd, 9 | old_stderr: RawFd, 10 | tee: bool, 11 | process: Option, 12 | newfd: Option, 13 | } 14 | 15 | impl CopyOutput { 16 | pub fn new(output_log: &std::path::Path, tee: bool) -> io::Result { 17 | let old_stdout = dup(nix::libc::STDOUT_FILENO).expect("Failed to duplicate stdout"); 18 | let old_stderr = dup(nix::libc::STDERR_FILENO).expect("Failed to duplicate stderr"); 19 | 20 | let mut process = None; 21 | let newfd: Option; 22 | 23 | if tee { 24 | let p = Command::new("tee") 25 | .arg(output_log) 26 | .stdin(Stdio::piped()) 27 | .spawn()?; 28 | dup2( 29 | p.stdin.as_ref().unwrap().as_raw_fd(), 30 | nix::libc::STDOUT_FILENO, 31 | ) 32 | .expect("Failed to redirect stdout to tee"); 33 | dup2( 34 | p.stdin.as_ref().unwrap().as_raw_fd(), 35 | nix::libc::STDERR_FILENO, 36 | ) 37 | .expect("Failed to redirect stderr to tee"); 38 | process = Some(p); 39 | newfd = None; 40 | } else { 41 | let file = File::create(output_log)?; 42 | dup2(file.as_raw_fd(), nix::libc::STDOUT_FILENO) 43 | .expect("Failed to redirect stdout to file"); 44 | dup2(file.as_raw_fd(), nix::libc::STDERR_FILENO) 45 | .expect("Failed to redirect stderr to file"); 46 | newfd = Some(file); 47 | } 48 | 49 | Ok(Self { 50 | old_stdout, 51 | old_stderr, 52 | tee, 53 | process, 54 | newfd, 55 | }) 56 | } 57 | } 58 | 59 | impl Drop for CopyOutput { 60 | fn drop(&mut self) { 61 | // Restore original stdout and stderr 62 | dup2(self.old_stdout, nix::libc::STDOUT_FILENO).expect("Failed to restore stdout"); 63 | dup2(self.old_stderr, nix::libc::STDERR_FILENO).expect("Failed to restore stderr"); 64 | close(self.old_stdout).expect("Failed to close old stdout"); 65 | close(self.old_stderr).expect("Failed to close old stderr"); 66 | 67 | // Ensure process or file is cleaned up 68 | if self.tee { 69 | if let Some(ref mut process) = self.process { 70 | process.wait().expect("Failed to wait on tee process"); 71 | } 72 | } else if let Some(ref mut file) = self.newfd { 73 | file.flush().expect("Failed to flush output file"); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /py/janitor/site/templates/generic/done.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block sidebar %} 3 | {% if not campaign %} 4 | {% include "cupboard/sidebar.html" %} 5 | {% else %} 6 | {% include [campaign + "/sidebar.html", "generic/sidebar.html"] %} 7 | {% endif %} 8 | {% endblock sidebar %} 9 | {% block page_title %} 10 | Changes That Have Been Merged or Pushed 11 | {% if campaign %}- {{ campaign }}{% endif %} 12 | {% endblock page_title %} 13 | {% block body %} 14 |
      15 |

      Changes that have been merged or pushed

      16 |
      17 | 18 | 24 | 25 |
      26 | {% set ns = namespace(last_date=None, run_date=None) %} 27 |
        28 | {% for run in runs %} 29 | {% if run.absorbed_at %} 30 | {% set ns.run_date = run.absorbed_at.date().isoformat() %} 31 | {% else %} 32 | {% set ns.run_date = "unknown" %} 33 | {% endif %} 34 | {% if ns.run_date != ns.last_date %} 35 | {% if ns.last_date %}
      {% endif %} 36 | {% set ns.last_date = ns.run_date %} 37 |

      {{ ns.last_date }}

      38 |
        39 | {% endif %} 40 |
      • 41 | {{ run.codebase }} 42 | {% if run.merged_by %} 43 | (merged by 44 | {% if run.merged_by_url %} 45 | {{ run.merged_by }} 46 | {% else %} 47 | {{ run.merged_by }} 48 | {% endif %} 49 | ) 50 | {% else %} 51 | (pushed) 52 | {% endif %} 53 | {% set command = run.command -%} 54 | {% set result = run.result -%} 55 | {% include [run.campaign + "/summary.html", "generic/summary.html"] %} 56 |
        • 57 | {% endfor %} 58 |
      59 |
      60 | {% endblock body %} 61 | -------------------------------------------------------------------------------- /reschedule.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (C) 2020 Jelmer Vernooij 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 | 19 | import argparse 20 | import asyncio 21 | import logging 22 | import sys 23 | 24 | from aiohttp import ClientSession 25 | from yarl import URL 26 | 27 | parser = argparse.ArgumentParser("reschedule") 28 | parser.add_argument("result_code", type=str) 29 | parser.add_argument("description_re", type=str, nargs="?") 30 | parser.add_argument("--refresh", action="store_true", help="Force run from scratch.") 31 | parser.add_argument("--offset", type=int, default=0, help="Schedule offset.") 32 | parser.add_argument( 33 | "--rejected", action="store_true", help="Process rejected runs only." 34 | ) 35 | parser.add_argument("--campaign", type=str, help="Campaign to process.") 36 | parser.add_argument( 37 | "--min-age", type=int, default=0, help="Only reschedule runs older than N days." 38 | ) 39 | parser.add_argument( 40 | "--base-url", type=str, default="https://janitor.debian.net", help="Instance URL" 41 | ) 42 | args = parser.parse_args() 43 | 44 | logging.basicConfig() 45 | 46 | 47 | async def main(base_url, result_code, campaign, description_re, rejected, min_age=0): 48 | params = {"result_code": result_code} 49 | if campaign: 50 | params["suite"] = campaign 51 | if description_re: 52 | params["description_re"] = description_re 53 | if rejected: 54 | params["rejected"] = "1" 55 | if min_age: 56 | params["min_age"] = str(min_age) 57 | url = URL(base_url) / "cupboard/api/mass-reschedule" 58 | async with ClientSession() as session, session.post(url, params=params) as resp: 59 | if resp.status != 200: 60 | logging.fatal("rescheduling failed: %d", resp.status) 61 | return 1 62 | for entry in await resp.json(): 63 | logging.info("%r", entry) 64 | 65 | 66 | sys.exit( 67 | asyncio.run( 68 | main( 69 | args.base_url, 70 | args.result_code, 71 | args.campaign, 72 | args.description_re, 73 | args.rejected, 74 | args.min_age, 75 | ) 76 | ) 77 | ) 78 | -------------------------------------------------------------------------------- /py/janitor/site/cupboard/merge_proposals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from typing import Optional 4 | 5 | import asyncpg 6 | 7 | 8 | async def get_proposals_with_run(conn: asyncpg.Connection, campaign: Optional[str]): 9 | query = """ 10 | SELECT 11 | DISTINCT ON (merge_proposal.url) 12 | run.codebase AS codebase, 13 | run.suite AS suite, 14 | merge_proposal.url AS url, 15 | merge_proposal.status AS status 16 | FROM 17 | merge_proposal 18 | LEFT JOIN new_result_branch ON new_result_branch.revision = merge_proposal.revision 19 | LEFT JOIN run ON run.id = new_result_branch.run_id 20 | """ 21 | args = [] 22 | if campaign: 23 | args.append(campaign) 24 | query += """ 25 | WHERE suite = $1 26 | """ 27 | query += """ 28 | ORDER BY merge_proposal.url, run.finish_time DESC 29 | """ 30 | return await conn.fetch(query, *args) 31 | 32 | 33 | async def write_merge_proposals(db, suite): 34 | async with db.acquire() as conn: 35 | proposals_by_status: dict[str, list[asyncpg.Record]] = {} 36 | for row in await get_proposals_with_run(conn, campaign=suite): 37 | proposals_by_status.setdefault(row["status"], []).append(row) 38 | 39 | merged = proposals_by_status.get("merged", []) + proposals_by_status.get( 40 | "applied", [] 41 | ) 42 | return { 43 | "suite": suite, 44 | "open_proposals": proposals_by_status.get("open", []), 45 | "merged_proposals": merged, 46 | "closed_proposals": proposals_by_status.get("closed", []), 47 | "rejected_proposals": proposals_by_status.get("rejected", []), 48 | "abandoned_proposals": proposals_by_status.get("abandoned", []), 49 | } 50 | 51 | 52 | async def get_proposal_with_run(conn: asyncpg.Connection, url: str): 53 | query = """ 54 | SELECT 55 | run.codebase AS codebase, 56 | run.suite AS suite, 57 | merge_proposal.url AS url, 58 | merge_proposal.status AS status, 59 | merge_proposal.merged_at AS merged_at, 60 | merge_proposal.merged_by AS merged_by, 61 | merge_proposal.last_scanned AS last_scanned, 62 | merge_proposal.can_be_merged AS can_be_merged 63 | FROM 64 | merge_proposal 65 | LEFT JOIN new_result_branch ON new_result_branch.revision = merge_proposal.revision 66 | LEFT JOIN run ON run.id = new_result_branch.run_id 67 | WHERE url = $1 68 | """ 69 | return await conn.fetchrow(query, url) 70 | 71 | 72 | async def get_publishes(conn, url): 73 | return await conn.fetch( 74 | """ 75 | SELECT * FROM publish WHERE merge_proposal_url = $1 76 | ORDER BY timestamp DESC 77 | """, 78 | url, 79 | ) 80 | 81 | 82 | async def write_merge_proposal(db, url): 83 | async with db.acquire() as conn: 84 | proposal = await get_proposal_with_run(conn, url) 85 | 86 | publishes = await get_publishes(conn, url) 87 | 88 | return { 89 | "proposal": proposal, 90 | "publishes": publishes, 91 | } 92 | -------------------------------------------------------------------------------- /py/janitor/site/merge_proposals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from typing import Optional 4 | 5 | import asyncpg 6 | 7 | 8 | async def get_proposals_with_run(conn: asyncpg.Connection, suite: Optional[str]): 9 | query = """ 10 | SELECT 11 | DISTINCT ON (merge_proposal.url) 12 | run.codebase AS codebase, 13 | run.suite AS suite, 14 | merge_proposal.url AS url, 15 | merge_proposal.status AS status 16 | FROM 17 | merge_proposal 18 | LEFT JOIN new_result_branch ON new_result_branch.revision = merge_proposal.revision 19 | LEFT JOIN run ON run.id = new_result_branch.run_id 20 | """ 21 | args = [] 22 | if suite: 23 | args.append(suite) 24 | query += """ 25 | WHERE suite = $1 26 | """ 27 | query += """ 28 | ORDER BY merge_proposal.url, run.finish_time DESC 29 | """ 30 | return await conn.fetch(query, *args) 31 | 32 | 33 | async def write_merge_proposals(db, suite): 34 | async with db.acquire() as conn: 35 | proposals_by_status: dict[str, list[asyncpg.Record]] = {} 36 | for row in await get_proposals_with_run(conn, suite=suite): 37 | proposals_by_status.setdefault(row["status"], []).append(row) 38 | 39 | merged = proposals_by_status.get("merged", []) + proposals_by_status.get( 40 | "applied", [] 41 | ) 42 | return { 43 | "suite": suite, 44 | "campaign": suite, 45 | "open_proposals": proposals_by_status.get("open", []), 46 | "merged_proposals": merged, 47 | "closed_proposals": proposals_by_status.get("closed", []), 48 | "rejected_proposals": proposals_by_status.get("rejected", []), 49 | "abandoned_proposals": proposals_by_status.get("abandoned", []), 50 | } 51 | 52 | 53 | async def get_proposal_with_run(conn: asyncpg.Connection, url: str): 54 | query = """ 55 | SELECT 56 | run.codebase AS codebase, 57 | run.suite AS suite, 58 | merge_proposal.url AS url, 59 | merge_proposal.status AS status, 60 | merge_proposal.merged_at AS merged_at, 61 | merge_proposal.merged_by AS merged_by, 62 | merge_proposal.last_scanned AS last_scanned, 63 | merge_proposal.can_be_merged AS can_be_merged 64 | FROM 65 | merge_proposal 66 | LEFT JOIN new_result_branch ON new_result_branch.revision = merge_proposal.revision 67 | LEFT JOIN run ON run.id = new_result_branch.run_id 68 | WHERE url = $1 69 | """ 70 | return await conn.fetchrow(query, url) 71 | 72 | 73 | async def get_publishes(conn, url): 74 | return await conn.fetch( 75 | """ 76 | SELECT * FROM publish WHERE merge_proposal_url = $1 77 | ORDER BY timestamp ASC 78 | """, 79 | url, 80 | ) 81 | 82 | 83 | async def write_merge_proposal(db, url): 84 | async with db.acquire() as conn: 85 | proposal = await get_proposal_with_run(conn, url) 86 | 87 | publishes = await get_publishes(conn, url) 88 | 89 | return { 90 | "proposal": proposal, 91 | "publishes": publishes, 92 | } 93 | --------------------------------------------------------------------------------