├── .gitignore
├── CHANGES.txt
├── LICENSE
├── README.md
├── default.nix
├── nixborg-module.nix
├── nixborg
    ├── __init__.py
    ├── celery.py
    ├── default_settings.py
    ├── helper.py
    ├── hydra_jobsets.py
    ├── models.py
    ├── pr_merge.py
    ├── receiver.py
    ├── tasks.py
    └── views.py
├── setup.py
└── version


/.gitignore:
--------------------------------------------------------------------------------
1 | /build/
2 | /nixborg.egg-info/
3 | /result
4 | __pycache__
5 | development.cfg
6 | 


--------------------------------------------------------------------------------
/CHANGES.txt:
--------------------------------------------------------------------------------
1 | 0.0
2 | ---
3 | 
4 | -  Initial version
5 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright [2016] Domen Kožar, Robin Gloster, Franz Pletz
 2 | 
 3 | Licensed under the Apache License, Version 2.0 (the "License");
 4 | you may not use this file except in compliance with the License.
 5 | You may obtain a copy of the License at
 6 | 
 7 |     http://www.apache.org/licenses/LICENSE-2.0
 8 | 
 9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | nixborg
 2 | ======
 3 | 
 4 | Github bot for reviewing/testing pull requests with the help of [Hydra](http://nixos.org/hydra/)
 5 | 
 6 | Getting Started
 7 | ---------------
 8 | 
 9 | ```
10 | $ nix-shell
11 | $ $CELERY
12 | ```
13 | ```
14 | $ nix-shell
15 | $ $APP
16 | ```
17 | 
18 | Workflow
19 | --------
20 | 
21 | - new PR appears
22 | - maintainer tells bot to test the PR with `@nixborg build`
23 | - PR branches is rebased upon its base (which can also be nixpkgs-channels branch)
24 | - rebased branch is pushed to nixpkgs/pr-XXX branch
25 | - HTTP request to nixbot-receiver script running on hydra
26 | - receiver adds jobset definition to hydra via `hydra-update-jobset`
27 | - TODO: once built, report is commented on the PR
28 | - TODO: optionally, merge the PR if there are no new failures, disable the jobset
29 | 


--------------------------------------------------------------------------------
/default.nix:
--------------------------------------------------------------------------------
 1 | { nixborgSrc ? { outPath = ./.; revCount = 1234; rev = "abcdef"; }
 2 | , nixpkgs ? <nixpkgs>
 3 | }:
 4 | 
 5 | let
 6 |   pkgs = import nixpkgs{};
 7 | in pkgs.callPackages ({ stdenv, python3, fetchFromGitHub, git }:
 8 | let
 9 |   server = python3.pkgs.buildPythonApplication rec {
10 |     name = "nixborg-${version}";
11 |     version = builtins.readFile ./version + "." + toString nixborgSrc.revCount + "." + nixborgSrc.rev;
12 | 
13 |     src = nixborgSrc.outPath;
14 | 
15 |     shellHook = ''
16 |       export LC_ALL=en_US.utf8
17 |       export PATH=${git}/bin:$PATH
18 |       export NIXBORG_SETTINGS=$(pwd)/development.cfg
19 |       export FLASK_DEBUG=1
20 |       export FLASK_APP=nixborg
21 |       export APP='python -m flask run --reload'
22 |       export CELERY='celery -A nixborg.celery worker -E -l INFO'
23 |       export FLOWER='celery -A nixborg.celery flower'
24 |       echo 'Run $APP for the flask app, $CELERY for the celery worker'
25 |       echo 'Run $FLOWER for the celery flower webmonitoring'
26 |     '';
27 | 
28 |     propagatedBuildInputs = with python3.pkgs; [
29 |       PyGithub flask flask_migrate flask_sqlalchemy celery redis requests
30 |     ];
31 | 
32 |     doCheck = false;
33 | 
34 |     meta = with stdenv.lib; {
35 |       description = "Github bot for reviewing/testing pull requests with the help of Hydra";
36 |       maintainers = with maintainers; [ domenkozar fpletz globin ];
37 |       license = licenses.asl20;
38 |       homepage = https://github.com/mayflower/nixborg;
39 |     };
40 |   };
41 |   receiver = stdenv.mkDerivation rec {
42 |     name = "nixborg-receiver-${version}";
43 |     inherit (server) src version meta;
44 | 
45 |     buildInputs = [ python3 ];
46 |     dontBuild = true;
47 | 
48 |     installPhase = ''
49 |       install -vD nixborg/receiver.py $out/bin/nixborg-receiver
50 |     '';
51 |   };
52 | in { inherit server receiver; }) {}
53 | 


--------------------------------------------------------------------------------
/nixborg-module.nix:
--------------------------------------------------------------------------------
  1 | { config, lib, pkgs, ... }:
  2 | 
  3 | with lib;
  4 | 
  5 | let
  6 |   cfg = config.services.nixborg;
  7 |   pkg = import ./default.nix {};
  8 |   productionCfg = pkgs.writeText "production.cfg" ''
  9 |     NIXBORG_BOT_NAME = '${cfg.botName}'
 10 |     NIXBORG_REPO = '${cfg.repo}'
 11 |     NIXBORG_PR_REPO = '${cfg.prRepo}'
 12 |     NIXBORG_PUBLIC_URL = '${cfg.publicUrl}'
 13 |     NIXBORG_REPO_DIR = '${cfg.dataDir}/repositories'
 14 |     NIXBORG_NIXEXPR_PATH = '${cfg.nixexprPath}'
 15 |     NIXBORG_HYDRA_PROJECT = '${cfg.hydraProject}'
 16 |     NIXBORG_GITHUB_TOKEN = '${cfg.githubToken}'
 17 |     NIXBORG_GITHUB_SECRET = '${cfg.githubSecret}'
 18 |     NIXBORG_GITHUB_WRITE_COMMENTS = True
 19 |     NIXBORG_RECEIVER_KEY = '${cfg.receiver.key}'
 20 |     NIXBORG_RECEIVER_URL = '${cfg.receiverURL}'
 21 |   '';
 22 | in {
 23 |   options = {
 24 |     services.nixborg = {
 25 |       enable = mkEnableOption "nixborg";
 26 | 
 27 |       botName = mkOption {
 28 |         type = types.str;
 29 |         description = "The bot's github user account name.";
 30 |         default = "nixborg";
 31 |       };
 32 | 
 33 |       githubToken = mkOption {
 34 |         type = types.str;
 35 |         description = "The bot's github user account token.";
 36 |         example = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
 37 |       };
 38 | 
 39 |       githubSecret = mkOption {
 40 |         type = types.str;
 41 |         description = "The github webhook's secret key used for autheticating webhooks";
 42 |         example = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
 43 |       };
 44 | 
 45 |       repo = mkOption {
 46 |         type = types.str;
 47 |         description = "The github repository to check for PRs.";
 48 |         example = "nixos/nixpkgs";
 49 |       };
 50 | 
 51 |       prRepo = mkOption {
 52 |         type = types.str;
 53 |         description = "The github repository to push the PRs to.";
 54 |         example = "nixos/nixpkgs-pr";
 55 |       };
 56 | 
 57 |       publicUrl = mkOption {
 58 |         type = types.str;
 59 |         description = "The public URL the bot is reachable at (Github hook endpoint).";
 60 |         example = "https://nixborg.nixos.org";
 61 |       };
 62 | 
 63 |       dataDir = mkOption {
 64 |         type = types.path;
 65 |         description = "The directory the repositories are stored in.";
 66 |         default = "/var/lib/nixborg";
 67 |       };
 68 | 
 69 |       nixexprPath = mkOption {
 70 |         type = types.str;
 71 |         description = "Path to the nix expression to test.";
 72 |         default = "nixos/release-combined.nix";
 73 |       };
 74 | 
 75 |       hydraProject = mkOption {
 76 |         type = types.str;
 77 |         description = "Hydra project to create the jobsets in.";
 78 |         default = "nixos";
 79 |       };
 80 | 
 81 |       receiverURL = mkOption {
 82 |         type = types.str;
 83 |         description = "URL to the Hydra server running the receiver.";
 84 |         default = "https://hydra.example.com";
 85 |       };
 86 | 
 87 |       receiver = {
 88 |         enable = mkEnableOption "nixborg receiver for jobset creation on hydra";
 89 |         listenAddr = mkOption {
 90 |           type = types.str;
 91 |           description = "Address for the receiver to listen on.";
 92 |           default = "127.0.0.1";
 93 |         };
 94 |         port = mkOption {
 95 |           type = types.int;
 96 |           description = "Port for the receiver to listen on.";
 97 |           default = 7000;
 98 |         };
 99 |         key = mkOption {
100 |           type = types.str;
101 |           description = "PSK to be used for HMAC authentication of the bot with the receiver";
102 |         };
103 |       };
104 |     };
105 |   };
106 | 
107 |   config = let
108 |     env = pkgs.python3.buildEnv.override {
109 |       extraLibs = [ pkg.server ];
110 |     };
111 |     uwsgi = pkgs.uwsgi.override { plugins = [ "python3" ]; };
112 |     uwsgienv = uwsgi.python3.buildEnv.override {
113 |       extraLibs = [ pkg.server uwsgi ];
114 |     };
115 |     nixborgUwsgi = pkgs.writeText "uwsgi.json" (builtins.toJSON {
116 |       uwsgi = {
117 |         plugins = [ "python3" ];
118 |         pythonpath = "${uwsgienv}/${uwsgi.python3.sitePackages}";
119 |         uid = "nixborg";
120 |         socket = "/run/nixborg/uwsgi.socket";
121 |         chown-socket = "nixborg:nginx";
122 |         chmod-socket = 770;
123 |         chdir = "${cfg.dataDir}";
124 |         mount = "/=nixborg:app";
125 |         env = "NIXBORG_SETTINGS=${productionCfg}";
126 |         manage-script-name = true;
127 |         master = true;
128 |         processes = 4;
129 |         stats = "/run/nixborg/stats.socket";
130 |         no-orphans = true;
131 |         vacuum = true;
132 |         logger = "syslog";
133 |       };
134 |     });
135 |   in mkMerge [
136 |     (mkIf cfg.enable {
137 |       users.extraUsers.nixborg = {
138 |         createHome = true;
139 |         home = cfg.dataDir;
140 |       };
141 | 
142 |       services.redis.enable = true;
143 | 
144 |       systemd.tmpfiles.rules = [
145 |         "d /run/nixborg 0755 nixborg nogroup -"
146 |       ];
147 | 
148 |       systemd.services.nixborg = {
149 |         after = [ "network.target" ];
150 |         wantedBy = [ "multi-user.target" ];
151 |         serviceConfig = {
152 |           Type = "notify";
153 |           Restart = "on-failure";
154 |           KillSignal = "SIGQUIT";
155 |           StandardError = "syslog";
156 |           NotifyAccess = "all";
157 |           ExecStart = "${uwsgi}/bin/uwsgi --json ${nixborgUwsgi}";
158 |           PrivateDevices = "yes";
159 |           PrivateTmp = "yes";
160 |           ProtectSystem = "full";
161 |           ProtectHome = "yes";
162 |           NoNewPrivileges = "yes";
163 |           ReadWritePaths = "/run/nixborg /var/lib/nixborg";
164 |         };
165 |       };
166 | 
167 |       systemd.services.nixborg-workers = {
168 |         after = [ "network.target" ];
169 |         wantedBy = [ "multi-user.target" ];
170 |         script = ''
171 |           ${env}/bin/celery -A nixborg.celery worker -E -l INFO
172 |         '';
173 |         path = [ pkgs.git ];
174 |         environment = {
175 |           FLASK_APP = "nixborg";
176 |           NIXBORG_SETTINGS = productionCfg;
177 |         };
178 | 
179 |         serviceConfig = {
180 |           User = "nixborg";
181 |           Group = "nogroup";
182 |         };
183 |       };
184 |     })
185 |     (mkIf cfg.receiver.enable {
186 |       users.extraUsers.nixborg-receiver = { extraGroups = [ "hydra" ]; };
187 |       services.postgresql.identMap = ''
188 |         hydra-users nixborg-receiver hydra
189 |       '';
190 | 
191 |       systemd.services.nixborg-receiver = {
192 |         after = [ "network.target" ];
193 |         wantedBy = [ "multi-user.target" ];
194 |         path = [
195 |           # FIXME remove override after https://github.com/NixOS/hydra/pull/516
196 |           (pkgs.hydra.overrideAttrs (old: {
197 |             src = pkgs.fetchFromGitHub {
198 |               owner = "mayflower";
199 |               repo = "hydra";
200 |               rev = "2979a6b4a824e98ec3567bb9725cb24a18f85242";
201 |               sha256 = "0fxyn44bq8h8jnjjrsy4ji7ybq9k77xhp14l9gqnq216nl4fl086";
202 |             };
203 |           }))
204 |         ];
205 |         environment = {
206 |           NIXBORG_RECEIVER_PORT = toString cfg.receiver.port;
207 |           NIXBORG_RECEIVER_ADDRESS = cfg.receiver.listenAddr;
208 |           NIXBORG_RECEIVER_KEY = cfg.receiver.key;
209 |         } // config.systemd.services.hydra-evaluator.environment;
210 | 
211 |         serviceConfig = {
212 |           User = "nixborg-receiver";
213 |           Group = "hydra";
214 |           ExecStart = "${pkg.receiver}/bin/nixborg-receiver";
215 |           Restart = "always";
216 |         };
217 |       };
218 |     })
219 |   ];
220 | }
221 | 
222 | 


--------------------------------------------------------------------------------
/nixborg/__init__.py:
--------------------------------------------------------------------------------
 1 | import logging
 2 | 
 3 | from flask import Flask
 4 | from flask_sqlalchemy import SQLAlchemy
 5 | from flask_migrate import Migrate
 6 | 
 7 | from .celery import make_celery
 8 | from .views import github_hook
 9 | 
10 | 
11 | app = Flask(__name__)
12 | app.config.from_object('nixborg.default_settings')
13 | app.config.from_envvar('NIXBORG_SETTINGS')
14 | app.logger.setLevel(logging.INFO)
15 | formatter = logging.Formatter(
16 |     '{%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
17 | )
18 | handler = logging.StreamHandler()
19 | handler.setFormatter(formatter)
20 | handler.setLevel(logging.INFO)
21 | app.logger.addHandler(handler)
22 | 
23 | db = SQLAlchemy(app)
24 | migrate = Migrate(app, db)
25 | celery = make_celery(app)
26 | 
27 | import nixborg.tasks
28 | import nixborg.models
29 | 
30 | app.register_blueprint(github_hook)
31 | 
32 | 
33 | @app.route('/')
34 | def root():
35 |     return ''
36 | 
37 | if __name__ == '__main__':
38 |     app.run()
39 | 
40 | 
41 | # def main(global_config, **settings):
42 | #     callback = settings['nixborg.public_url'] + hook
43 | #     print("Subscribing to repository {} at {}".format(repo.html_url, callback))
44 | #     hooks = [h.config['url'] for h in repo.hooks()]
45 | #     if not any(filter(lambda url: url == callback, hooks)):
46 | #         repo.create_hook(
47 | #             "web",
48 | #             {
49 | #                 "url": callback,
50 | #                 "content_type": "json",
51 | #             },
52 | #             ["pull_request", "issue_comment"],
53 | #         )
54 | 
55 | 
56 | # def generate_github_token():
57 | #     user = ""
58 | #     password = ""
59 | #     scopes = ['user', 'repo', 'write:repo_hook']
60 | 
61 | #     while not user:
62 | #         user = input('User: ')
63 | #     while not password:
64 | #         password = getpass('Password: ')
65 | 
66 | #     auth = authorize(user, password, scopes, "testing", "http://example.com")
67 | #     print("Token: {}".format(auth.token))
68 | 


--------------------------------------------------------------------------------
/nixborg/celery.py:
--------------------------------------------------------------------------------
 1 | from celery import Celery
 2 | 
 3 | def make_celery(app):
 4 |     celery = Celery(
 5 |         app.import_name,
 6 |         backend=app.config['NIXBORG_CELERY_RESULT_BACKEND'],
 7 |         broker=app.config['NIXBORG_CELERY_BROKER_URL'],
 8 |     )
 9 |     celery.conf.update(app.config)
10 |     TaskBase = celery.Task
11 | 
12 |     class ContextTask(TaskBase):
13 |         abstract = True
14 |         def __call__(self, *args, **kwargs):
15 |             with app.app_context():
16 |                 return TaskBase.__call__(self, *args, **kwargs)
17 | 
18 |     celery.Task = ContextTask
19 | 
20 |     return celery
21 | 


--------------------------------------------------------------------------------
/nixborg/default_settings.py:
--------------------------------------------------------------------------------
 1 | NIXBORG_BOT_NAME = 'nixborg'
 2 | NIXBORG_MAIL = 'nixborg@nixos.community'
 3 | NIXBORG_REPO = 'nixos/nixpkgs'
 4 | NIXBORG_PR_REPO = 'mayflower/nixpkgs-pr'
 5 | NIXBORG_PUBLIC_URL = 'https://nixborg.nixos.community'
 6 | NIXBORG_REPO_DIR = '/var/lib/nixborg/repositories'
 7 | NIXBORG_HYDRA_PROJECT = 'nixos'
 8 | NIXBORG_NIXEXPR_PATH = 'nixos/release-small.nix'
 9 | NIXBORG_GITHUB_TOKEN = '<insert github token>'
10 | NIXBORG_GITHUB_SECRET = 'justnotsorandom'
11 | NIXBORG_GITHUB_WRITE_COMMENTS = True
12 | NIXBORG_CELERY_BROKER_URL = 'redis://localhost:6379'
13 | NIXBORG_CELERY_RESULT_BACKEND = 'redis://localhost:6379'
14 | CELERYD_LOG_FORMAT = "[%(asctime)s: %(levelname)s/%(processName)s/%(name)s] %(message)s"
15 | SQLALCHEMY_TRACK_MODIFICATIONS = False
16 | SQLALCHEMY_DATABASE_URI = 'sqlite:///var/lib/nixborg/db.sqlite'
17 | 


--------------------------------------------------------------------------------
/nixborg/helper.py:
--------------------------------------------------------------------------------
1 | from github import Github
2 | 
3 | def gh_login(token):
4 |     return Github(token)
5 | 


--------------------------------------------------------------------------------
/nixborg/hydra_jobsets.py:
--------------------------------------------------------------------------------
 1 | import hmac
 2 | import json
 3 | import logging
 4 | import subprocess
 5 | 
 6 | import requests
 7 | 
 8 | log = logging.getLogger(__name__)
 9 | 
10 | class HydraJobsets(object):
11 |     def __init__(self, config):
12 |         self.hydra_project = config.get('NIXBORG_HYDRA_PROJECT')
13 |         self.repo = 'https://github.com/' + config.get('NIXBORG_PR_REPO')
14 |         self.receiver = config.get('NIXBORG_RECEIVER_URL')
15 |         self.key = config.get('NIXBORG_RECEIVER_KEY')
16 |         self.nixexpr_path = config.get('NIXBORG_NIXEXPR_PATH')
17 | 
18 |     def _sendAuthenticatedRequest(self, url, data):
19 |         headers = {
20 |           'X-Nixborg-HMAC': hmac.new(self.key.encode('utf-8'), json.dumps(data).encode('utf-8')).hexdigest()
21 |         }
22 | 
23 |         result = requests.post(url, json=data, headers=headers);
24 |         result.raise_for_status()
25 |         return result.json()
26 | 
27 |     def add(self, pr_id):
28 |         return self._sendAuthenticatedRequest(f'{self.receiver}/jobset', {
29 |             'project': self.hydra_project,
30 |             'repo': self.repo,
31 |             'jobset': f'pr-{pr_id}',
32 |             'ref': f'pr-{pr_id}',
33 |             'nixexpr_path': self.nixexpr_path,
34 |         })['jobset_url']
35 | 
36 |     def remove(self, pr_id):
37 |         return self._sendAuthenticatedRequest(f'{self.receiver}/jobset', {
38 |             'project': self.hydra_project,
39 |             'repo': self.repo,
40 |             'jobset': f'pr-{pr_id}',
41 |             'ref': f'pr-{pr_id}',
42 |             'nixexpr_path': self.nixexpr_path,
43 |             'hidden': True,
44 |             'disabled': True,
45 |         })['jobset_url']
46 | 


--------------------------------------------------------------------------------
/nixborg/models.py:
--------------------------------------------------------------------------------
 1 | from . import db
 2 | 
 3 | class PullRequest(db.Model):
 4 |     id = db.Column(db.Integer, primary_key=True, autoincrement=False)
 5 |     mergeable = db.Column(db.Boolean)
 6 |     author = db.Column(db.String(128))
 7 |     title = db.Column(db.String(1024))
 8 |     state = db.Column(db.String(128))
 9 |     head = db.Column(db.String(1024))
10 |     assignee = db.Column(db.String(128))
11 |     priority = db.Column(db.Integer)
12 |     approved_by = db.Column(db.String(128))
13 | 


--------------------------------------------------------------------------------
/nixborg/pr_merge.py:
--------------------------------------------------------------------------------
 1 | from pathlib import Path
 2 | import logging
 3 | import subprocess
 4 | from billiard import current_process
 5 | 
 6 | log = logging.getLogger(__name__)
 7 | 
 8 | 
 9 | def merge_push(pr, ref, base, config):
10 |     USER = config.get('NIXBORG_BOT_NAME')
11 |     TOKEN = config.get('NIXBORG_GITHUB_TOKEN')
12 |     REPO = f"https://github.com/" + config.get('NIXBORG_REPO')
13 |     PR_REPO = f"https://{TOKEN}@github.com/" + config.get('NIXBORG_PR_REPO')
14 |     REPO_PATH = config.get('NIXBORG_REPO_DIR') + str(current_process().index)
15 | 
16 |     path = Path(REPO_PATH, "nixpkgs.git")
17 | 
18 |     if not path.exists():
19 |         log.info('Cloning {} to {}'.format(REPO, path))
20 |         path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
21 |         logged_call(f'git clone {REPO} {path}')
22 | 
23 |     GIT = f'git -C {path}'
24 |     logged_call(f'git config --global user.email "nixborg@nixos.community"')
25 |     logged_call(f'git config --global user.name "{USER}"')
26 |     logged_call(f'{GIT} remote add origin {REPO} || true')
27 |     logged_call(f'{GIT} remote set-url origin {REPO}')
28 |     logged_call(f'{GIT} remote add pr {PR_REPO} || true')
29 |     logged_call(f'{GIT} remote set-url pr {PR_REPO}')
30 | 
31 |     log.info('Fetching base repository including PRs')
32 |     logged_call(f'{GIT} config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"')
33 |     logged_call(f'{GIT} fetch origin')
34 | 
35 |     log.info(f'Checking out PR {pr} at ref {ref}')
36 |     logged_call(f'{GIT} branch -f pr-{pr} {ref}')
37 |     logged_call(f'{GIT} reset --hard {ref}')
38 |     log.info(f'Rebasing PR on top of {base}')
39 |     logged_call(f'{GIT} rebase --abort || true')
40 |     logged_call(f'{GIT} rebase origin/{base}')
41 | 
42 |     log.info(f'Pushing PR branch to PR repository')
43 |     logged_call(f'{GIT} push -f pr HEAD:pr-{pr}')
44 | 
45 | 
46 | def logged_call(args):
47 |     try:
48 |         subprocess.run(args, check=True, shell=True)
49 |     except Exception as e:
50 |         log.error(f'Failed to execute command: {args}')
51 |         log.error(e)
52 |         raise e
53 | 


--------------------------------------------------------------------------------
/nixborg/receiver.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | 
 3 | from http.server import BaseHTTPRequestHandler, HTTPServer
 4 | from shlex import quote
 5 | from urllib.parse import urlparse
 6 | import hmac
 7 | import json
 8 | import logging
 9 | import os
10 | import subprocess
11 | 
12 | 
13 | logging.basicConfig(
14 |     level=logging.INFO,
15 |     format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s',
16 |     datefmt='%H:%M:%S'
17 | )
18 | logger = logging.getLogger(__name__)
19 | 
20 | 
21 | def add_jobset(project, jobset, repo, ref, nixexpr_path, disabled=False, hidden=False):
22 |     disabled_flag = '--disabled ' if disabled else '--trigger'
23 |     hidden_flag = '--hidden' if hidden else ''
24 |     result = subprocess.run(
25 |         f'hydra-update-jobset {quote(project)} {quote(jobset)} --url {quote(repo)} --ref {quote(ref)} ' +
26 |         f'--nixexpr-path {quote(nixexpr_path)} {disabled_flag} {hidden_flag}',
27 |         shell=True,
28 |         stdout=subprocess.PIPE,
29 |         stderr=subprocess.PIPE,
30 |     )
31 |     if result.stderr:
32 |         logger.error(result.stderr)
33 |     result.check_returncode()
34 | 
35 |     return result.stdout
36 | 
37 | 
38 | class HTTPForbidden(Exception):
39 |     pass
40 | 
41 | 
42 | class HydraJobsetManagerHandler(BaseHTTPRequestHandler):
43 |     def request_setup(self):
44 |         self.body = self.rfile.read(int(self.headers['Content-Length']))
45 |         logger.info(self.body)
46 |         self.json_body = self.parse_json_body()
47 |         if not self.check_token():
48 |             raise HTTPForbidden
49 | 
50 |     def check_token(self):
51 |         return hmac.compare_digest(
52 |             hmac.new(os.environ['NIXBORG_RECEIVER_KEY'].encode('utf-8'), self.body).hexdigest(),
53 |             self.headers.get('X-Nixborg-HMAC')
54 |         )
55 | 
56 |     def parse_json_body(self):
57 |         return json.loads(self.body)
58 | 
59 |     def handle_jobset_creation(self):
60 |         return add_jobset(**self.json_body)
61 | 
62 |     def do_POST(self):
63 |         response = None
64 |         try:
65 |             self.request_setup()
66 |             if urlparse(self.path).path == '/jobset':
67 |                 jobset_url = self.handle_jobset_creation()
68 |                 self.send_response(200)
69 |                 response = json.dumps({ 'jobset_url': jobset_url.decode('utf-8') }).encode('utf-8')
70 |             else:
71 |                 self.send_response(204)
72 |         except HTTPForbidden as e:
73 |             logger.error(e)
74 |             self.send_response(403)
75 |         except Exception as e:
76 |             logger.error(e)
77 |             self.send_response(500)
78 |         self.end_headers()
79 |         self.wfile.write(response)
80 | 
81 | def main():
82 |     server_address = (
83 |         os.environ.get('NIXBORG_RECEIVER_ADDRESS', '127.0.0.1'),
84 |         int(os.environ.get('NIXBORG_RECEIVER_PORT', 7000))
85 |     )
86 |     httpd = HTTPServer(server_address, HydraJobsetManagerHandler)
87 |     httpd.serve_forever()
88 | 
89 | if __name__ == "__main__":
90 |     main()
91 | 


--------------------------------------------------------------------------------
/nixborg/tasks.py:
--------------------------------------------------------------------------------
 1 | from datetime import datetime
 2 | import logging
 3 | 
 4 | from . import app, celery
 5 | from .helper import gh_login
 6 | from .hydra_jobsets import HydraJobsets
 7 | from .pr_merge import merge_push
 8 | 
 9 | log = logging.getLogger(__name__)
10 | 
11 | 
12 | @celery.task()
13 | def github_comment(args, number, body):
14 |     gh = gh_login(app.config.get('NIXBORG_GITHUB_TOKEN'))
15 |     repo = app.config.get('NIXBORG_REPO')
16 |     pr = gh.get_repo(repo).get_pull(number)
17 |     log.info(f'Comment on PR {pr.number}: ' + body.format(*args))
18 |     if app.config.get('NIXBORG_GITHUB_WRITE_COMMENTS'):
19 |         pr.create_issue_comment(body.format(*args))
20 | 
21 | 
22 | @celery.task()
23 | def test_github_pr(number, comment_time):
24 |     gh = gh_login(app.config.get('NIXBORG_GITHUB_TOKEN'))
25 |     repo = app.config.get('NIXBORG_REPO')
26 |     pr = gh.get_repo(repo).get_pull(number)
27 | 
28 |     if datetime.strptime(comment_time, '%Y-%m-%dT%H:%M:%SZ') < pr.head.repo.pushed_at:
29 |         raise(Exception('Comment is older than PR'))
30 | 
31 |     merge_push_task.delay(number, pr.head.sha, pr.base.ref)
32 | 
33 | 
34 | @celery.task
35 | def add_hydra_jobset(number):
36 |     jobsets = HydraJobsets(app.config)
37 |     return [jobsets.add(number)]
38 | 
39 | 
40 | @celery.task
41 | def merge_push_task(number, ref, base):
42 |     merge_push(number, ref, base, app.config)
43 |     (add_hydra_jobset.s(number) |
44 |         github_comment.s(number, 'Jobset created at {}'))()
45 | 
46 | 
47 | @celery.task
48 | def issue_commented(payload):
49 |     bot_name = app.config.get('NIXBORG_BOT_NAME')
50 |     if payload.get("action") not in ["created", "edited"]:
51 |         return
52 | 
53 |     comment = payload['comment']['body'].strip()
54 |     if comment == (f'@{bot_name} build'):
55 |         # TODO: this should ignore issues
56 |         number = payload["issue"]["number"]
57 |         gh = gh_login(app.config.get('NIXBORG_GITHUB_TOKEN'))
58 |         repo = gh.get_repo(app.config.get('NIXBORG_REPO'))
59 |         reviewer = payload["comment"]["user"]["login"]
60 |         if repo.has_in_collaborators(reviewer) or reviewer in [ "globin", "fpletz", "grahamc" ]:
61 |             test_github_pr.delay(number, payload['comment']['updated_at'])
62 |         else:
63 |             github_comment.delay((), number, f'@{payload["comment"]["user"]["login"]} is not a committer')
64 | 


--------------------------------------------------------------------------------
/nixborg/views.py:
--------------------------------------------------------------------------------
 1 | import hashlib
 2 | import hmac
 3 | 
 4 | from flask import abort, request, Blueprint
 5 | 
 6 | from .helper import gh_login
 7 | from .hydra_jobsets import HydraJobsets
 8 | 
 9 | HELP = """
10 | Hi! I'm a bot that helps with reviewing and testing Nix code.
11 | 
12 | Commands:
13 | 
14 | - `@{bot_name} build` creates a new Hydra jobset and reports results
15 | """
16 | 
17 | github_hook = Blueprint('github_hook', __name__)
18 | 
19 | 
20 | @github_hook.route('/github-webhook', methods=['POST'])
21 | def github_webhook():
22 |     from . import app
23 |     from .tasks import github_comment, test_github_pr, issue_commented
24 | 
25 |     signature = request.headers['X-Hub-Signature']
26 |     key = app.config.get('NIXBORG_GITHUB_SECRET').encode('utf-8')
27 |     comp_signature = "sha1=" + hmac.new(key, request.get_data(), hashlib.sha1).hexdigest()
28 |     if not hmac.compare_digest(signature.encode('utf-8'), comp_signature.encode('utf-8')):
29 |         app.logger.error(f'HMAC of github webhook is incorrect')
30 |         abort(403)
31 | 
32 |     event = request.headers['X-GitHub-Event']
33 |     payload = request.get_json()
34 |     bot_name = app.config.get('NIXBORG_BOT_NAME')
35 | 
36 |     repo = payload['repository']['full_name']
37 |     configured_repo = app.config.get('NIXBORG_REPO')
38 |     if repo != configured_repo:
39 |         app.logger.error(f'Repository `{repo}` does not match configured `{configured_repo}`')
40 |         abort(400)
41 | 
42 |     if event == "pull_request":
43 |         pr_id = payload["pull_request"]["number"]
44 |         if payload.get("action") in ["opened", "reopened", "edited"]:
45 |             pr_info = (
46 |                 payload["pull_request"]["base"]["repo"]["owner"]["login"],
47 |                 payload["pull_request"]["base"]["repo"]["name"],
48 |                 pr_id
49 |             )
50 |             # TODO: evaluate and report statistics
51 |             # TODO: merge next line with mention-bot
52 |             # github_comment.delay(pr_info, HELP.format(bot_name=bot_name))
53 |         if payload.get("action") in ["closed", "merged"]:
54 |             jobsets = HydraJobsets(app.config)
55 |             jobsets.remove(pr_id)
56 |     elif event == "issue_comment":
57 |         issue_commented.delay(payload)
58 | 
59 |     return "Ok"
60 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | 
 3 | from setuptools import setup, find_packages
 4 | 
 5 | here = os.path.abspath(os.path.dirname(__file__))
 6 | with open(os.path.join(here, 'README.md')) as f:
 7 |     README = f.read()
 8 | with open(os.path.join(here, 'CHANGES.txt')) as f:
 9 |     CHANGES = f.read()
10 | 
11 | requires = [
12 |     'flask',
13 |     'flask_migrate',
14 |     'flask_sqlalchemy',
15 |     'PyGithub',
16 |     'celery',
17 |     #'flower',
18 |     'redis',
19 |     'requests',
20 | ]
21 | 
22 | tests_require = [
23 |   'pylint', 'mypy', 'pycodestyle',
24 | 
25 |   'setuptools_scm', 'pytest-runner',  # FIXME needed for pypi2nix
26 | ]
27 | 
28 | setup(name='nixborg',
29 |       version='0.0',
30 |       description='nixborg',
31 |       long_description=README + '\n\n' + CHANGES,
32 |       classifiers=[
33 |           "Programming Language :: Python",
34 |           "Topic :: Internet :: WWW/HTTP",
35 |           "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
36 |       ],
37 |       entry_points={
38 |           'console_scripts': [
39 |               'nixborg-receiver = nixborg.receiver:main',
40 |           ]
41 |       },
42 |       author='',
43 |       author_email='',
44 |       url='',
45 |       keywords='web',
46 |       packages=find_packages(),
47 |       include_package_data=True,
48 |       zip_safe=False,
49 |       extras_require={
50 |           'testing': tests_require,
51 |       },
52 |       install_requires=requires,
53 |       )
54 | 


--------------------------------------------------------------------------------
/version:
--------------------------------------------------------------------------------
1 | 0.0.1


--------------------------------------------------------------------------------