├── tests ├── __init__.py └── test_git_uri_parser.py ├── gitlab_tools ├── bin │ └── __init__.py ├── enums │ ├── __init__.py │ ├── ProtocolEnum.py │ ├── VcsEnum.py │ └── InvokedByEnum.py ├── forms │ ├── __init__.py │ ├── custom_fields.py │ ├── fingerprint.py │ ├── sign.py │ ├── push_mirror.py │ └── pull_mirror.py ├── models │ ├── __init__.py │ ├── gitlab_tools.py │ └── celery.py ├── tasks │ └── __init__.py ├── tools │ ├── __init__.py │ ├── Svn.py │ ├── Validators.py │ ├── GitRemote.py │ ├── celery.py │ ├── gitlab.py │ ├── fingerprint.py │ ├── formaters.py │ ├── crypto.py │ ├── jsonifier.py │ ├── GitUri.py │ ├── Git.py │ ├── GitSubprocess.py │ └── helpers.py ├── views │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ └── index.py │ ├── home │ │ ├── __init__.py │ │ ├── templates │ │ │ └── home.index.home.html │ │ └── index.py │ ├── sign │ │ ├── __init__.py │ │ ├── templates │ │ │ └── sign.index.login.html │ │ └── index.py │ ├── fingerprint │ │ ├── __init__.py │ │ └── templates │ │ │ ├── fingerprint.index.fingerprint.html │ │ │ └── fingerprint.index.new.html │ ├── pull_mirror │ │ ├── __init__.py │ │ └── templates │ │ │ ├── url_info.html │ │ │ ├── pull_mirror.index.pull_mirror.html │ │ │ └── pull_mirror.index.log.html │ └── push_mirror │ │ ├── __init__.py │ │ └── templates │ │ ├── url_info.html │ │ ├── push_mirror.index.push_mirror.html │ │ ├── push_mirror.index.log.html │ │ ├── push_mirror.index.new.html │ │ └── push_mirror.index.edit.html ├── celery_beat │ └── __init__.py ├── migrations │ ├── __init__.py │ ├── versions │ │ ├── __init__.py │ │ ├── README.md │ │ ├── 0bf6832fd1f2_.py │ │ ├── 946211522c2f_.py │ │ ├── 3b2efd6da478_.py │ │ ├── cad4d8aacceb_.py │ │ ├── 29579044e36c_.py │ │ ├── 56189bfb2c5f_.py │ │ ├── 6f456354bea1_.py │ │ ├── 82696705e6df_.py │ │ ├── 20bcb4b2673c_.py │ │ ├── d4841aeeb072_.py │ │ ├── 6ea24c49b7a1_.py │ │ ├── 56c59adcfe10_.py │ │ └── 19e8725e0581_.py │ ├── README │ ├── script.py.mako │ ├── alembic.ini │ └── env.py ├── __init__.py ├── static │ ├── img │ │ ├── favicon.png │ │ └── no_group_avatar.png │ ├── package.json │ ├── style.css │ └── package-lock.json ├── wsgi.py ├── templates │ ├── 403.html │ ├── 404.html │ ├── 400.html │ ├── 500.html │ ├── _formhelpers.html │ ├── macros.html │ └── base.html ├── __main__.py ├── extensions.py ├── blueprints.py ├── config.py ├── middleware.py └── application.py ├── asu.sh ├── code-check.sh ├── translation_compile.sh ├── translation_generate.sh ├── debian ├── py3dist-overrides └── python3-gitlab-tools.postinst ├── doc ├── img │ ├── home.png │ ├── home_thumb.png │ ├── fingerprints.png │ ├── pull_mirrors.png │ ├── fingerprints_thumb.png │ └── pull_mirrors_thumb.png └── PostgreSQL.md ├── pre-commit.sh ├── translation_update.sh ├── babel.cfg ├── stdeb.cfg ├── etc ├── gitlab-tools │ └── config.yml └── systemd │ └── system │ ├── gitlab-tools.service │ ├── gitlab-tools_celeryworker.service │ └── gitlab-tools_celerybeat.service ├── manage.py ├── MANIFEST.in ├── setup.cfg ├── .version.yml ├── CHANGELOG.md ├── archlinux ├── PKGBUILD └── gitlab-tools.install ├── .gitignore ├── tox.ini ├── .gitlab-ci.yml ├── setup.py └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/bin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/enums/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/forms/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/views/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/celery_beat/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/views/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/views/home/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/views/sign/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/views/fingerprint/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/views/pull_mirror/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/views/push_mirror/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gitlab_tools/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.5.3' 2 | -------------------------------------------------------------------------------- /asu.sh: -------------------------------------------------------------------------------- 1 | sudo -u gitlab-tools -H python3 manage.py $1 2 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /code-check.sh: -------------------------------------------------------------------------------- 1 | flake8 gitlab-tools tests --max-line-length=160 --builtins=_ -------------------------------------------------------------------------------- /translation_compile.sh: -------------------------------------------------------------------------------- 1 | pybabel compile -d gitlab-tools/translations 2 | -------------------------------------------------------------------------------- /translation_generate.sh: -------------------------------------------------------------------------------- 1 | pybabel extract -F babel.cfg -o messages.pot . 2 | -------------------------------------------------------------------------------- /debian/py3dist-overrides: -------------------------------------------------------------------------------- 1 | psycopg2-binary python3-psycopg2 2 | raven python3-raven 3 | -------------------------------------------------------------------------------- /doc/img/home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/doc/img/home.png -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/README.md: -------------------------------------------------------------------------------- 1 | # Directory where all migrations are stored -------------------------------------------------------------------------------- /pre-commit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | ./code-check.sh 4 | python3 setup.py test 5 | -------------------------------------------------------------------------------- /translation_update.sh: -------------------------------------------------------------------------------- 1 | pybabel update -i messages.pot -d gitlab-tools/translations 2 | -------------------------------------------------------------------------------- /doc/img/home_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/doc/img/home_thumb.png -------------------------------------------------------------------------------- /doc/img/fingerprints.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/doc/img/fingerprints.png -------------------------------------------------------------------------------- /doc/img/pull_mirrors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/doc/img/pull_mirrors.png -------------------------------------------------------------------------------- /babel.cfg: -------------------------------------------------------------------------------- 1 | [python: **.py] 2 | [jinja2: **/templates/**.html] 3 | extensions=jinja2.ext.autoescape,jinja2.ext.with_ 4 | -------------------------------------------------------------------------------- /doc/img/fingerprints_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/doc/img/fingerprints_thumb.png -------------------------------------------------------------------------------- /doc/img/pull_mirrors_thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/doc/img/pull_mirrors_thumb.png -------------------------------------------------------------------------------- /gitlab_tools/enums/ProtocolEnum.py: -------------------------------------------------------------------------------- 1 | 2 | class ProtocolEnum: 3 | SSH = 1 4 | HTTP = 2 5 | HTTPS = 3 6 | -------------------------------------------------------------------------------- /gitlab_tools/enums/VcsEnum.py: -------------------------------------------------------------------------------- 1 | 2 | class VcsEnum: 3 | GIT = 1 4 | BAZAAR = 2 5 | SVN = 3 6 | MERCURIAL = 4 7 | -------------------------------------------------------------------------------- /gitlab_tools/static/img/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/gitlab_tools/static/img/favicon.png -------------------------------------------------------------------------------- /stdeb.cfg: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | Package3: gitlab-tools 3 | Depends3: redis-server, git, rabbitmq-server 4 | Replaces3: python3-gitlab-tools -------------------------------------------------------------------------------- /gitlab_tools/enums/InvokedByEnum.py: -------------------------------------------------------------------------------- 1 | 2 | class InvokedByEnum: 3 | UNKNOWN = 1 4 | SCHEDULER = 2 5 | HOOK = 3 6 | MANUAL = 4 7 | -------------------------------------------------------------------------------- /gitlab_tools/static/img/no_group_avatar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Salamek/gitlab-tools/HEAD/gitlab_tools/static/img/no_group_avatar.png -------------------------------------------------------------------------------- /etc/gitlab-tools/config.yml: -------------------------------------------------------------------------------- 1 | PORT: 80 2 | HOST: 0.0.0.0 3 | SECRET_KEY: # Change to something secure 4 | SQLALCHEMY_DATABASE_URI: # Change to your server configuration -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from gitlab_tools.bin.gitlab_tools import main 5 | 6 | if __name__ == '__main__': 7 | main() 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include README.md 3 | recursive-include etc * 4 | recursive-include tests * 5 | recursive-include debian * 6 | recursive-inclide gitlab_tools/static * 7 | -------------------------------------------------------------------------------- /gitlab_tools/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | Bootstrap for use in uwsgi and so 3 | """ 4 | 5 | from gitlab_tools.application import create_app, get_config 6 | 7 | config = get_config('gitlab_tools.config.Production') 8 | app = create_app(config) 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | 4 | [mypy] 5 | python_version = 3.4 6 | strict = true 7 | show_error_codes = true 8 | follow_imports = silent 9 | ignore_missing_imports = true 10 | allow_any_generics = true 11 | warn_return_any = false 12 | warn_unreachable = true 13 | exclude = venv/* 14 | -------------------------------------------------------------------------------- /gitlab_tools/static/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Gitlab Tools npm", 3 | "license": "GPL-3.0", 4 | "repository": "https://github.com/Salamek/gitlab-tools", 5 | "dependencies": { 6 | "bootstrap": "^4.6.2", 7 | "jquery": "~3.5.1", 8 | "font-awesome": "^4.7.0", 9 | "select2": "^4.0.13" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /gitlab_tools/templates/403.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 403{% endblock %} 4 | 5 | {% block body %} 6 |
7 |

403 Forbidden

8 |

Access denied.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /gitlab_tools/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 404{% endblock %} 4 | 5 | {% block body %} 6 |
7 |

404 Not Found

8 |

Page not found.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /gitlab_tools/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | def main() -> None: 5 | """Entrypoint to the ``celery`` umbrella command.""" 6 | from gitlab_tools.bin.gitlab_tools import main as _main # pylint: disable=import-outside-toplevel 7 | _main() 8 | 9 | 10 | if __name__ == '__main__': # pragma: no cover 11 | main() 12 | -------------------------------------------------------------------------------- /etc/systemd/system/gitlab-tools.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=GitLab Tools list Web Service 3 | Requires=network.target 4 | 5 | [Service] 6 | User=gitlab-tools 7 | Type=simple 8 | Restart=on-failure 9 | RemainAfterExit=yes 10 | ExecStart=/usr/bin/gitlab-tools server --config_prod --log_dir ${HOME} 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /etc/systemd/system/gitlab-tools_celeryworker.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=GitLab Tools Service Celery worker 3 | Requires=network.target 4 | 5 | [Service] 6 | User=gitlab-tools 7 | Type=simple 8 | Restart=on-failure 9 | RemainAfterExit=yes 10 | ExecStart=/usr/bin/gitlab-tools celeryworker --config_prod 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /gitlab_tools/templates/400.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 400{% endblock %} 4 | 5 | {% block body %} 6 |
7 |

400 Bad Request

8 |

Page found but the POST/GET data sent was invalid.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} -------------------------------------------------------------------------------- /gitlab_tools/forms/custom_fields.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from wtforms import SelectField 3 | 4 | 5 | class NonValidatingSelectField(SelectField): # type: ignore 6 | """ 7 | Attempt to make an open ended select field that can accept dynamic 8 | choices added by the browser. 9 | """ 10 | def pre_validate(self, form: Any) -> None: 11 | pass 12 | -------------------------------------------------------------------------------- /gitlab_tools/templates/500.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block append_title %} - HTTP 500{% endblock %} 4 | 5 | {% block body %} 6 |
7 |

500 Internal Server Error

8 |

Uh-oh you found a bug! An email has been sent to the ADMINs list.

9 |

URL: {{ request.url }}

10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /gitlab_tools/tools/Svn.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class Svn: 4 | 5 | @staticmethod 6 | def fix_url(url: str) -> str: 7 | # We use unsupported URL formats for SVN, we need to convert that 8 | if 'svn+http://' in url: 9 | return url.replace('svn+http://', 'http://') 10 | 11 | if 'svn+https://' in url: 12 | return url.replace('svn+https://', 'https://') 13 | 14 | return url 15 | -------------------------------------------------------------------------------- /gitlab_tools/forms/fingerprint.py: -------------------------------------------------------------------------------- 1 | from wtforms import Form, StringField, validators 2 | 3 | __author__ = "Adam Schubert" 4 | __date__ = "$26.7.2017 19:33:05$" 5 | 6 | 7 | class NewForm(Form): # type: ignore 8 | hostname = StringField(None, [validators.Length(min=1, max=255)]) 9 | 10 | def validate(self) -> bool: 11 | rv = Form.validate(self) 12 | if not rv: 13 | return False 14 | 15 | return True 16 | -------------------------------------------------------------------------------- /etc/systemd/system/gitlab-tools_celerybeat.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=GitLab Tools Service Celery beat 3 | Requires=network.target 4 | 5 | [Service] 6 | User=gitlab-tools 7 | Type=simple 8 | RuntimeDirectory=gitlab-tools 9 | RuntimeDirectoryMode=0775 10 | Restart=on-failure 11 | RemainAfterExit=yes 12 | ExecStart=/usr/bin/gitlab-tools celerybeat --config_prod --pid=/run/gitlab-tools/celerybeat.pid 13 | PIDFile=/run/gitlab-tools/celerybeat.pid 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | -------------------------------------------------------------------------------- /gitlab_tools/tools/Validators.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | class ParamCheckException(Exception): 5 | pass 6 | 7 | 8 | class Validators: 9 | 10 | @staticmethod 11 | def is_valid_hostname(hostname: str) -> bool: 12 | if len(hostname) > 255: 13 | return False 14 | if hostname[-1] == ".": 15 | hostname = hostname[:-1] # strip exactly one dot from the right, if present 16 | allowed = re.compile(r'(?!-)[A-Z\d-]{1,63}(?\d+)\.(?P\d+)\.(?P\d+)\' 9 | 'setup.py': version\s*=\s*\'(?P\d+)\.(?P\d+)\.(?P\d+)\' 10 | 'PKGBUILD': pkgver\s*=\s*(?P.*) 11 | 12 | VERSION_FILES: 13 | 'gitlab_tools/__init__.py': 'python' 14 | 'setup.py': 'setup.py' 15 | 'archlinux/PKGBUILD': 'PKGBUILD' -------------------------------------------------------------------------------- /gitlab_tools/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /gitlab_tools/views/sign/templates/sign.index.login.html: -------------------------------------------------------------------------------- 1 | {% set layout = 'login' %} 2 | {% extends "base.html" %} 3 | {% block body %} 4 | {% from "_formhelpers.html" import render_field %} 5 |
6 |
7 |
8 |
9 | {{ "Sign In" }} 10 |
11 |
12 | {{ "Sign In" }} 13 |
14 |
15 |
16 |
17 | {% endblock %} -------------------------------------------------------------------------------- /gitlab_tools/templates/_formhelpers.html: -------------------------------------------------------------------------------- 1 | {% macro render_field(field, class, title) %} 2 |
3 | 4 | {{field(class_="form-control input-sm", **{'placeholder': title, 'data-toggle':"popover", 'data-trigger':"focus ", 'data-placement': "bottom", 'data-content': '
'.join(field.errors)})}} 5 |
6 | {% endmacro %} 7 | {% macro render_checkbox(field, title, class) %} 8 |
9 | 13 |
14 | {% endmacro %} -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/0bf6832fd1f2_.py: -------------------------------------------------------------------------------- 1 | """Create correct sequence 2 | 3 | Revision ID: 0bf6832fd1f2 4 | Revises: 3b2efd6da478 5 | Create Date: 2018-06-22 17:56:53.940104 6 | 7 | """ 8 | from alembic import op 9 | from sqlalchemy.schema import Sequence, CreateSequence, DropSequence 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '0bf6832fd1f2' 14 | down_revision = '3b2efd6da478' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.execute(CreateSequence(Sequence("task_id_sequence"))) 21 | op.execute(CreateSequence(Sequence("taskset_id_sequence"))) 22 | 23 | 24 | def downgrade(): 25 | op.execute(DropSequence(Sequence("task_id_sequence"))) 26 | op.execute(DropSequence(Sequence("taskset_id_sequence"))) 27 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/946211522c2f_.py: -------------------------------------------------------------------------------- 1 | """Add periodic_sync column to support periodic syncs 2 | 3 | Revision ID: 946211522c2f 4 | Revises: cad4d8aacceb 5 | Create Date: 2018-06-08 12:44:28.966293 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '946211522c2f' 14 | down_revision = 'cad4d8aacceb' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('pull_mirror', sa.Column('periodic_sync', sa.String(length=255), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('push_mirror', 'periodic_sync') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /gitlab_tools/extensions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Flask and other extensions instantiated here. 4 | 5 | To avoid circular imports with views and create_app(), extensions are instantiated here. They will be initialized 6 | (calling init_app()) in application.py. 7 | """ 8 | import os 9 | from logging import getLogger 10 | from flask_sqlalchemy import SQLAlchemy 11 | from flask_babel import Babel 12 | from flask_celery import Celery 13 | from flask_login import LoginManager 14 | from flask_migrate import Migrate 15 | import gitlab_tools as app_root 16 | 17 | LOG = getLogger(__name__) 18 | APP_ROOT_FOLDER = os.path.abspath(os.path.dirname(app_root.__file__)) 19 | MIGRATE_ROOT_FOLDER = os.path.abspath(os.path.join(APP_ROOT_FOLDER, 'migrations')) 20 | 21 | 22 | db = SQLAlchemy() 23 | babel = Babel() 24 | login_manager = LoginManager() 25 | celery = Celery() 26 | migrate = Migrate(directory=MIGRATE_ROOT_FOLDER) 27 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## [1.1.3] - 2018-06-23 10 | ### Changed 11 | - Fixes bugs in celery backend 12 | 13 | ## [1.1.2] - 2018-06-22 14 | ### Changed 15 | - Fix debian package build 16 | 17 | ## [1.1.1] - 2018-06-22 18 | ### Changed 19 | - Fix debian package build 20 | 21 | ## [1.1.0] - 2018-06-22 22 | ### Added 23 | - This changelog. 24 | - Mirror task logs, now you can view status of each mirror task and check what went wrong. 25 | 26 | 27 | ### Changed 28 | - Note in mirror list is now only title for row. 29 | - Webhook URL is inside ro input to preserve space. 30 | 31 | 32 | ### Removed 33 | - Redis backend was replaced by RabbitMQ. 34 | - Flask-Caching was unused. 35 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # template used to generate migration files 5 | # file_template = %%(rev)s_%%(slug)s 6 | 7 | # set to 'true' to run the environment during 8 | # the 'revision' command, regardless of autogenerate 9 | # revision_environment = false 10 | 11 | 12 | # Logging configuration 13 | [loggers] 14 | keys = root,sqlalchemy,alembic 15 | 16 | [handlers] 17 | keys = console 18 | 19 | [formatters] 20 | keys = generic 21 | 22 | [logger_root] 23 | level = WARN 24 | handlers = console 25 | qualname = 26 | 27 | [logger_sqlalchemy] 28 | level = WARN 29 | handlers = 30 | qualname = sqlalchemy.engine 31 | 32 | [logger_alembic] 33 | level = INFO 34 | handlers = 35 | qualname = alembic 36 | 37 | [handler_console] 38 | class = StreamHandler 39 | args = (sys.stderr,) 40 | level = NOTSET 41 | formatter = generic 42 | 43 | [formatter_generic] 44 | format = %(levelname)-5.5s [%(name)s] %(message)s 45 | datefmt = %H:%M:%S 46 | -------------------------------------------------------------------------------- /gitlab_tools/tools/GitRemote.py: -------------------------------------------------------------------------------- 1 | from gitlab_tools.enums.VcsEnum import VcsEnum 2 | from gitlab_tools.tools.GitUri import GitUri 3 | 4 | 5 | class GitRemote(GitUri): 6 | def __init__(self, url: str, is_force_update: bool = False, is_prune_mirrors: bool = False): 7 | super().__init__(url) 8 | self.is_force_update = is_force_update 9 | self.is_prune_mirrors = is_prune_mirrors 10 | 11 | @property 12 | def vcs_type(self) -> int: 13 | """ 14 | Detects VCS type by its URL protocol 15 | :return: VcsEnum int 16 | """ 17 | 18 | scheme_to_vcs_mapping = { 19 | 'bzr': VcsEnum.BAZAAR, 20 | 'hg': VcsEnum.MERCURIAL, 21 | 'svn': VcsEnum.SVN, 22 | 'ssh': VcsEnum.GIT, 23 | 'http': VcsEnum.GIT, 24 | 'git': VcsEnum.GIT, 25 | } 26 | 27 | for scheme, vcs in scheme_to_vcs_mapping.items(): 28 | if scheme in self.scheme: 29 | return vcs 30 | 31 | return VcsEnum.GIT 32 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/3b2efd6da478_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 3b2efd6da478 4 | Revises: 6f456354bea1 5 | Create Date: 2018-06-21 23:00:53.266721 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '3b2efd6da478' 14 | down_revision = '6f456354bea1' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('task_result', sa.Column('parent_id', sa.Integer(), nullable=True)) 22 | op.create_index(op.f('ix_task_result_parent_id'), 'task_result', ['parent_id'], unique=False) 23 | op.create_foreign_key(None, 'task_result', 'task_result', ['parent_id'], ['id']) 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.drop_constraint(None, 'task_result', type_='foreignkey') 30 | op.drop_index(op.f('ix_task_result_parent_id'), table_name='task_result') 31 | op.drop_column('task_result', 'parent_id') 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/cad4d8aacceb_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: cad4d8aacceb 4 | Revises: 6ea24c49b7a1 5 | Create Date: 2018-06-08 01:36:29.014684 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'cad4d8aacceb' 14 | down_revision = '6ea24c49b7a1' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('pull_mirror', sa.Column('periodic_task_id', sa.Integer(), nullable=True)) 22 | op.create_index(op.f('ix_pull_mirror_periodic_task_id'), 'pull_mirror', ['periodic_task_id'], unique=False) 23 | op.create_foreign_key(None, 'pull_mirror', 'periodic_task', ['periodic_task_id'], ['id']) 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.drop_constraint(None, 'pull_mirror', type_='foreignkey') 30 | op.drop_index(op.f('ix_pull_mirror_periodic_task_id'), table_name='pull_mirror') 31 | op.drop_column('pull_mirror', 'periodic_task_id') 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /gitlab_tools/views/pull_mirror/templates/url_info.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | Project mirror is using standard URL syntax to specify protocol and mirrored repository VCS type: 4 |
    5 |
  • [nothing] - Git SSH SCP like syntax (git@github.com:Salamek/gitlab-tools.git)
    • 6 |
  • https:// - Git over SSL (https://github.com/Salamek/gitlab-tools.git)
    • 7 |
  • http:// - Git over HTTP (http://github.com/Salamek/gitlab-tools.git)
    • 8 |
  • git:// - Git native (git://github.com/Salamek/gitlab-tools.git)
    • 9 |
  • ssh:// - Git over SSH (ssh://git@github.com/Salamek/gitlab-tools.git)
    • 10 |
  • svn+http:// - SVN over HTTP
    • 11 |
  • svn+https:// - SVN over SSL
    • 12 |
  • svn+ssh:// - SVN over SSH
    • 13 |
  • bzr::bzr:// - Bazaar
    • 14 |
  • hg::https:// - Mercurial
    • 15 |
16 | Other combinations may work, consult GIT documentation. 17 |
18 |
-------------------------------------------------------------------------------- /gitlab_tools/views/push_mirror/templates/url_info.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | Project mirror is using standard URL syntax to specify protocol and mirrored repository VCS type: 4 |
    5 |
  • [nothing] - Git SSH SCP like syntax (git@github.com:Salamek/gitlab-tools.git)
    • 6 |
  • https:// - Git over SSL (https://github.com/Salamek/gitlab-tools.git)
    • 7 |
  • http:// - Git over HTTP (http://github.com/Salamek/gitlab-tools.git)
    • 8 |
  • git:// - Git native (git://github.com/Salamek/gitlab-tools.git)
    • 9 |
  • ssh:// - Git over SSH (ssh://git@github.com/Salamek/gitlab-tools.git)
    • 10 |
  • svn+http:// - SVN over HTTP
    • 11 |
  • svn+https:// - SVN over SSL
    • 12 |
  • svn+ssh:// - SVN over SSH
    • 13 |
  • bzr::bzr:// - Bazaar
    • 14 |
  • hg::https:// - Mercurial
    • 15 |
16 | Other combinations may work, consult GIT documentation. 17 |
18 |
-------------------------------------------------------------------------------- /gitlab_tools/forms/sign.py: -------------------------------------------------------------------------------- 1 | from flask_babel import gettext 2 | from wtforms import Form, StringField, PasswordField, validators 3 | from gitlab_tools.extensions import db 4 | from gitlab_tools.models.gitlab_tools import User 5 | 6 | __author__ = "Adam Schubert" 7 | __date__ = "$26.7.2017 19:33:05$" 8 | 9 | 10 | class InForm(Form): # type: ignore 11 | username = StringField(None, [validators.Length(min=5, max=35)]) 12 | password = PasswordField(None, [validators.Length(min=5, max=35)]) 13 | 14 | def __init__(self, *args, **kwargs): 15 | Form.__init__(self, *args, **kwargs) 16 | self.user = None 17 | 18 | def validate(self) -> bool: 19 | rv = Form.validate(self) 20 | if not rv: 21 | return False 22 | user = db.session.query(User).filter(User.username == self.username.data).first() 23 | # email and password found and match 24 | if user is None: 25 | self.username.errors.append(gettext('Username was not found.')) 26 | return False 27 | 28 | if user.check_password(self.password.data) is False: 29 | self.password.errors.append(gettext('Wrong password.')) 30 | return False 31 | 32 | self.user = user 33 | return True 34 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/29579044e36c_.py: -------------------------------------------------------------------------------- 1 | """Add is_jobs_enabled option 2 | 3 | Revision ID: 29579044e36c 4 | Revises: 946211522c2f 5 | Create Date: 2018-06-11 13:53:19.920827 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.ext.declarative import declarative_base 11 | from sqlalchemy.orm import sessionmaker 12 | 13 | Session = sessionmaker() 14 | 15 | Base = declarative_base() 16 | 17 | 18 | class PullMirror(Base): 19 | __tablename__ = 'pull_mirror' 20 | 21 | id = sa.Column(sa.Integer, primary_key=True) 22 | is_jobs_enabled = sa.Column(sa.Boolean()) 23 | 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '29579044e36c' 27 | down_revision = '946211522c2f' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | bind = op.get_bind() 34 | session = Session(bind=bind) 35 | 36 | op.add_column('pull_mirror', sa.Column('is_jobs_enabled', sa.Boolean(), nullable=True)) 37 | 38 | # Set is_jobs_enabled to true 39 | for pull_mirror in session.query(PullMirror): 40 | pull_mirror.is_jobs_enabled = True 41 | session.add(pull_mirror) 42 | session.commit() 43 | 44 | op.alter_column('pull_mirror', 'is_jobs_enabled', nullable=False) 45 | 46 | 47 | def downgrade(): 48 | # ### commands auto generated by Alembic - please adjust! ### 49 | op.drop_column('pull_mirror', 'is_jobs_enabled') 50 | # ### end Alembic commands ### 51 | -------------------------------------------------------------------------------- /debian/python3-gitlab-tools.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | case "$1" in 5 | abort-upgrade|abort-remove|abort-deconfigure) 6 | ;; 7 | 8 | configure) 9 | if [ -z "$2" ]; then 10 | useradd -m gitlab-tools 11 | systemctl daemon-reload 12 | gitlab-tools post_install --config_prod --user=gitlab-tools 13 | systemctl start gitlab-tools 14 | systemctl start gitlab-tools_celeryworker 15 | systemctl start gitlab-tools_celerybeat 16 | systemctl enable gitlab-tools_celeryworker 17 | systemctl enable gitlab-tools_celerybeat 18 | systemctl enable gitlab-tools 19 | else 20 | gitlab-tools db upgrade 21 | systemctl daemon-reload 22 | 23 | # Restart service only when is active or enabled 24 | if systemctl is-active --quiet gitlab-tools || systemctl is-enabled --quiet gitlab-tools; then 25 | systemctl restart gitlab-tools 26 | fi 27 | 28 | if systemctl is-active --quiet gitlab-tools_celeryworker || systemctl is-enabled --quiet gitlab-tools_celeryworker; then 29 | systemctl restart gitlab-tools_celeryworker 30 | fi 31 | 32 | if systemctl is-active --quiet gitlab-tools_celerybeat || systemctl is-enabled --quiet gitlab-tools_celerybeat; then 33 | systemctl restart gitlab-tools_celerybeat 34 | fi 35 | fi 36 | ;; 37 | 38 | *) 39 | echo "postinst called with unknown argument \`$1'" >&2 40 | exit 1 41 | ;; 42 | esac 43 | 44 | exit 0 45 | -------------------------------------------------------------------------------- /archlinux/PKGBUILD: -------------------------------------------------------------------------------- 1 | pkgname=gitlab-tools 2 | pkgver=1.5.3 3 | pkgdesc="GitLab Tools" 4 | pkgrel=1 5 | arch=('any') 6 | backup=('etc/gitlab-tools/config.yml') 7 | license=('GPL-3.0') 8 | url='https://github.com/Salamek/gitlab-tools' 9 | install=gitlab-tools.install 10 | makedepends=('python-setuptools' 'npm') 11 | depends=( 12 | 'python' 13 | 'systemd' 14 | 'redis' 15 | 'rabbitmq' 16 | 'python-flask' 17 | 'python-psycopg2' 18 | 'python-babel' 19 | 'python-yaml' 20 | 'python-flask-login' 21 | 'python-flask-migrate' 22 | 'python-flask-wtf' 23 | 'python-flask-script' 24 | 'python-raven' 25 | 'python-docopt' 26 | 'python-crypto' 27 | 'python-flask-babel' 28 | 'python-flask-navigation' 29 | 'python-celery' 30 | 'python-flask-celery-helper' 31 | 'python-gitlab' 32 | 'python-gitpython' 33 | 'python-paramiko' 34 | 'python-cron-descriptor' 35 | 'git' 36 | ) 37 | 38 | prepare() { 39 | mkdir -p "${srcdir}/${pkgname}" 40 | cp -R "${srcdir}/../../etc" "${srcdir}/${pkgname}" 41 | cp -R "${srcdir}/../../gitlab_tools" "${srcdir}/${pkgname}" 42 | cp -R "${srcdir}/../../tests" "${srcdir}/${pkgname}" 43 | cp -R "${srcdir}/../../setup.py" "${srcdir}/${pkgname}" 44 | cp -R "${srcdir}/../../requirements.txt" "${srcdir}/${pkgname}" 45 | cp -R "${srcdir}/../../README.md" "${srcdir}/${pkgname}" 46 | } 47 | 48 | build() { 49 | cd "${srcdir}/${pkgname}/gitlab_tools/static" 50 | npm install 51 | } 52 | 53 | package() { 54 | cd "${srcdir}/${pkgname}" 55 | python setup.py install --root="$pkgdir/" --optimize=1 56 | } -------------------------------------------------------------------------------- /gitlab_tools/tools/celery.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional 2 | from sqlalchemy.exc import IntegrityError 3 | from celery.result import AsyncResult 4 | from gitlab_tools.models.gitlab_tools import Mirror, PullMirror, PushMirror, TaskResult 5 | from gitlab_tools.extensions import db 6 | from gitlab_tools.models.celery import TaskMeta 7 | from gitlab_tools.enums.InvokedByEnum import InvokedByEnum 8 | 9 | 10 | def log_task_pending( 11 | task: AsyncResult, 12 | mirror: Mirror, 13 | task_callable: Optional[Callable] = None, 14 | invoked_by: int = InvokedByEnum.UNKNOWN, 15 | parent: Optional[TaskResult] = None 16 | ) -> TaskResult: 17 | 18 | try: 19 | task_meta = TaskMeta() 20 | task_meta.task_id = task.id 21 | db.session.add(task_meta) 22 | db.session.commit() 23 | except IntegrityError: 24 | db.session.rollback() 25 | # Wow it already exists, celery was so fast ?! 26 | # Lets make sure that result info have a correct data 27 | task_meta = TaskMeta.query.filter_by(task_id=task.id).first() 28 | db.session.add(task_meta) 29 | db.session.commit() 30 | 31 | task_result = TaskResult() 32 | task_result.taskmeta = task_meta 33 | task_result.parent = parent 34 | task_result.task_name = task_callable.__name__ if task_callable else None 35 | task_result.invoked_by = invoked_by 36 | if isinstance(mirror, PullMirror): 37 | task_result.pull_mirror = mirror 38 | elif isinstance(mirror, PushMirror): 39 | task_result.push_mirror = mirror 40 | 41 | db.session.add(task_result) 42 | db.session.commit() 43 | 44 | return task_result 45 | -------------------------------------------------------------------------------- /gitlab_tools/blueprints.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """All Flask blueprints for the entire application. 4 | 5 | All blueprints for all views go here. They shall be imported by the views themselves and by application.py. Blueprint 6 | URL paths are defined here as well. 7 | """ 8 | 9 | from flask import Blueprint 10 | 11 | 12 | def _factory(name: str, partial_module_string: str, url_prefix: str = None) -> Blueprint: 13 | """Generates blueprint objects for view modules. 14 | 15 | Positional arguments: 16 | partial_module_string -- string representing a view module without the absolute path (e.g. 'home.index' for 17 | gitlab-tools.views.home.index). 18 | url_prefix -- URL prefix passed to the blueprint. 19 | 20 | Returns: 21 | Blueprint instance for a view module. 22 | """ 23 | import_name = 'gitlab_tools.views.{}'.format(partial_module_string) 24 | template_folder = 'templates' 25 | blueprint = Blueprint(name, import_name, template_folder=template_folder, url_prefix=url_prefix) 26 | return blueprint 27 | 28 | 29 | home_index = _factory('home_index', 'home.index') 30 | sign_index = _factory('sign_index', 'sign.index', '/sign') 31 | api_index = _factory('api_index', 'api.index', '/api') 32 | 33 | pull_mirror_index = _factory('pull_mirror_index', 'pull_mirror.index', '/pull-mirror') 34 | push_mirror_index = _factory('push_mirror_index', 'push_mirror.index', '/push-mirror') 35 | fingerprint_index = _factory('fingerprint_index', 'fingerprint.index', '/fingerprint') 36 | 37 | 38 | all_blueprints = ( 39 | home_index, 40 | sign_index, 41 | api_index, 42 | pull_mirror_index, 43 | fingerprint_index, 44 | push_mirror_index 45 | ) 46 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/56189bfb2c5f_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 56189bfb2c5f 4 | Revises: 5 | Create Date: 2018-04-04 05:16:24.055254 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '56189bfb2c5f' 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('fingerprint', 22 | sa.Column('updated', sa.DateTime(), nullable=True), 23 | sa.Column('created', sa.DateTime(), nullable=True), 24 | sa.Column('id', sa.Integer(), nullable=False), 25 | sa.Column('user_id', sa.Integer(), nullable=True), 26 | sa.Column('hostname', sa.String(length=255), nullable=True), 27 | sa.Column('sha256_fingerprint', sa.String(length=255), nullable=True), 28 | sa.Column('hashed_hostname', sa.String(length=255), nullable=True), 29 | sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), 30 | sa.PrimaryKeyConstraint('id'), 31 | sa.UniqueConstraint('user_id', 'hashed_hostname', name='_user_id_hashed_hostname_uc') 32 | ) 33 | op.create_index(op.f('ix_fingerprint_hashed_hostname'), 'fingerprint', ['hashed_hostname'], unique=False) 34 | op.create_index(op.f('ix_fingerprint_user_id'), 'fingerprint', ['user_id'], unique=False) 35 | # ### end Alembic commands ### 36 | 37 | 38 | def downgrade(): 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | op.drop_index(op.f('ix_fingerprint_user_id'), table_name='fingerprint') 41 | op.drop_index(op.f('ix_fingerprint_hashed_hostname'), table_name='fingerprint') 42 | op.drop_table('fingerprint') 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/6f456354bea1_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 6f456354bea1 4 | Revises: d4841aeeb072 5 | Create Date: 2018-06-21 22:16:03.577289 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '6f456354bea1' 14 | down_revision = 'd4841aeeb072' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('task_result', sa.Column('taskmeta_id', sa.Integer(), nullable=False)) 22 | op.create_index(op.f('ix_task_result_taskmeta_id'), 'task_result', ['taskmeta_id'], unique=True) 23 | op.drop_index('ix_task_result_celery_taskmeta_id', table_name='task_result') 24 | op.drop_constraint('task_result_celery_taskmeta_id_fkey', 'task_result', type_='foreignkey') 25 | op.create_foreign_key(None, 'task_result', 'celery_taskmeta', ['taskmeta_id'], ['id']) 26 | op.drop_column('task_result', 'celery_taskmeta_id') 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade(): 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | op.add_column('task_result', sa.Column('celery_taskmeta_id', sa.INTEGER(), autoincrement=False, nullable=False)) 33 | op.drop_constraint(None, 'task_result', type_='foreignkey') 34 | op.create_foreign_key('task_result_celery_taskmeta_id_fkey', 'task_result', 'celery_taskmeta', ['celery_taskmeta_id'], ['id']) 35 | op.create_index('ix_task_result_celery_taskmeta_id', 'task_result', ['celery_taskmeta_id'], unique=False) 36 | op.drop_index(op.f('ix_task_result_taskmeta_id'), table_name='task_result') 37 | op.drop_column('task_result', 'taskmeta_id') 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /gitlab_tools/tools/gitlab.py: -------------------------------------------------------------------------------- 1 | import gitlab 2 | from flask import current_app 3 | from flask_login import current_user 4 | 5 | 6 | class VisibilityError(Exception): 7 | pass 8 | 9 | 10 | def check_project_visibility_in_group(project_visibility: str, group_id: int) -> None: 11 | visibility_to_int = { 12 | 'private': 1, 13 | 'internal': 2, 14 | 'public': 3 15 | } 16 | group = get_group(group_id) 17 | if visibility_to_int.get(project_visibility) > visibility_to_int.get(group.visibility): 18 | raise VisibilityError( 19 | 'Project visibility is less restrictive than its group {} > {}'.format( 20 | project_visibility, 21 | group.visibility 22 | ) 23 | ) 24 | 25 | 26 | def check_project_exists(project_name: str, group_id: int, ignore_project_id: int=None): 27 | gl = get_gitlab_instance() 28 | found_projects = gl.projects.list(search=project_name) 29 | for found_project in found_projects: 30 | if ignore_project_id and ignore_project_id == found_project.id: 31 | continue 32 | 33 | if project_name in [found_project.name, found_project.path] and group_id == int(found_project.namespace['id']): 34 | return True 35 | return False 36 | 37 | 38 | def get_gitlab_instance() -> gitlab.Gitlab: 39 | gl = gitlab.Gitlab( 40 | current_app.config['GITLAB_URL'], 41 | oauth_token=current_user.access_token, 42 | api_version=current_app.config['GITLAB_API_VERSION'] 43 | ) 44 | 45 | gl.auth() 46 | 47 | return gl 48 | 49 | 50 | def get_group(group_id: int): 51 | gl = get_gitlab_instance() 52 | return gl.groups.get(group_id) 53 | 54 | 55 | def get_project(project_id: int): 56 | gl = get_gitlab_instance() 57 | return gl.projects.get(project_id) 58 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/82696705e6df_.py: -------------------------------------------------------------------------------- 1 | """Add visibility field to PullMirror and drops is_public options 2 | 3 | Revision ID: 82696705e6df 4 | Revises: 29579044e36c 5 | Create Date: 2018-06-11 17:42:00.647650 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.ext.declarative import declarative_base 11 | from sqlalchemy.orm import sessionmaker 12 | 13 | Session = sessionmaker() 14 | 15 | Base = declarative_base() 16 | 17 | 18 | class PullMirror(Base): 19 | __tablename__ = 'pull_mirror' 20 | 21 | id = sa.Column(sa.Integer, primary_key=True) 22 | is_public = sa.Column(sa.Boolean()) 23 | visibility = sa.Column(sa.String(255)) 24 | 25 | # revision identifiers, used by Alembic. 26 | revision = '82696705e6df' 27 | down_revision = '29579044e36c' 28 | branch_labels = None 29 | depends_on = None 30 | 31 | 32 | def upgrade(): 33 | bind = op.get_bind() 34 | session = Session(bind=bind) 35 | op.add_column('pull_mirror', sa.Column('visibility', sa.String(length=255), nullable=True)) 36 | 37 | # Set visibility correctly 38 | for pull_mirror in session.query(PullMirror): 39 | pull_mirror.visibility = 'public' if pull_mirror.is_public else 'private' 40 | session.add(pull_mirror) 41 | session.commit() 42 | 43 | op.alter_column('pull_mirror', 'visibility', nullable=False) 44 | 45 | op.drop_column('pull_mirror', 'is_public') 46 | 47 | 48 | def downgrade(): 49 | bind = op.get_bind() 50 | session = Session(bind=bind) 51 | op.add_column('pull_mirror', sa.Column('is_public', sa.BOOLEAN(), autoincrement=False, nullable=True)) 52 | # Set is_public correctly 53 | for pull_mirror in session.query(PullMirror): 54 | pull_mirror.is_public = pull_mirror.visibility == 'public' 55 | session.add(pull_mirror) 56 | session.commit() 57 | op.drop_column('pull_mirror', 'visibility') 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | gitlab_tools/static/node_modules 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | .idea/workspace.xml 108 | reveng 109 | 110 | 111 | .idea 112 | config.yml 113 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [general] 2 | install_requires = 3 | lxml 4 | name = gitlab_tools 5 | 6 | [tox] 7 | envlist = lint, py36, py37, py38, py39, py310 8 | 9 | [gh-actions] 10 | python = 11 | 3.6: py36 12 | 3.7: py37 13 | 3.8: py38 14 | 3.9: py39 15 | 3.10: py310 16 | 17 | 18 | [testenv] 19 | commands = 20 | py.test --cov-report term-missing --cov-report xml --cov {[general]name} --cov-config tox.ini {posargs:tests} 21 | deps = 22 | {[general]install_requires} 23 | pytest-cov==2.12.1 24 | usedevelop = True 25 | 26 | [testenv:lint] 27 | commands = 28 | python setup.py check --strict 29 | python setup.py check --strict -m 30 | python setup.py check --strict -s 31 | #flake8 --application-import-names={[general]name},tests 32 | pylint --rcfile=tox.ini setup.py {[general]name} --ignored-classes=SQLAlchemy,Registrant,scoped_session --ignored-modules=alembic.op,alembic.context 33 | bandit -r {[general]name} -s B410 34 | #mypy -p gitlab_tools -p tests 35 | deps = 36 | {[general]install_requires} 37 | flake8-docstrings==1.6.0 38 | flake8-import-order==0.18.1 39 | flake8==3.9.2 40 | mypy==0.812 41 | pycodestyle==2.7.0 42 | pep8-naming==0.11.1 43 | pylint==2.8.3 44 | bandit==1.7.0 45 | 46 | [flake8] 47 | exclude = .tox/*,build/*,docs/*,venv/*,get-pip.py 48 | import-order-style = smarkets 49 | max-line-length = 120 50 | statistics = True 51 | 52 | [pylint] 53 | disable = 54 | locally-disabled, 55 | missing-docstring, 56 | protected-access, 57 | too-many-instance-attributes, 58 | bad-whitespace, 59 | invalid-name, 60 | too-few-public-methods, 61 | too-many-public-methods, 62 | duplicate-code 63 | ignore = .tox/*,build/*,docs/*,venv/*,get-pip.py 64 | max-args = 7 65 | max-line-length = 130 66 | max-branches = 20 67 | reports = no 68 | ignore-imports=yes 69 | extension-pkg-whitelist=lxml 70 | 71 | [run] 72 | branch = True -------------------------------------------------------------------------------- /gitlab_tools/tools/fingerprint.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Tuple 3 | import paramiko 4 | from gitlab_tools.tools.crypto import get_remote_server_key 5 | 6 | 7 | def get_remote_server_key_for_hostname(hostname: str) -> paramiko.pkey.PKey: 8 | if ':' in hostname: 9 | host, port = hostname.split(':') 10 | return get_remote_server_key(host.replace('[', '').replace(']', ''), int(port)) 11 | 12 | return get_remote_server_key(hostname) 13 | 14 | 15 | def check_hostname(hostname: str, known_hosts_path: str) -> Tuple[bool, paramiko.pkey.PKey]: 16 | """ 17 | Get remote_key 18 | :param hostname: Hostname to check 19 | :param known_hosts_path: Path to known_hosts 20 | :return: 21 | """ 22 | host_keys = paramiko.hostkeys.HostKeys(known_hosts_path if os.path.isfile(known_hosts_path) else None) 23 | remote_server_key_lookup = host_keys.lookup(hostname) 24 | 25 | if remote_server_key_lookup: 26 | key_name, = remote_server_key_lookup 27 | remote_server_key = remote_server_key_lookup[key_name] 28 | found = True 29 | else: 30 | # Not found, request validation 31 | remote_server_key = get_remote_server_key_for_hostname(hostname) 32 | found = False 33 | 34 | return found, remote_server_key 35 | 36 | 37 | def add_hostname(hostname: str, remote_server_key: paramiko.pkey.PKey, known_hosts_path: str) -> str: 38 | """ 39 | Add hostname to known_hosts 40 | :param hostname: str 41 | :param remote_server_key: paramiko.pkey.PKey 42 | :param known_hosts_path: str 43 | :return: str 44 | """ 45 | host_keys = paramiko.hostkeys.HostKeys(known_hosts_path if os.path.isfile(known_hosts_path) else None) 46 | hashed_hostname = host_keys.hash_host(hostname) 47 | host_keys.add(hashed_hostname, remote_server_key.get_name(), remote_server_key) 48 | #host_keys.add(hostname, remote_server_key.get_name(), remote_server_key) #!FIXME remove me 49 | host_keys.save(known_hosts_path) 50 | 51 | return hashed_hostname 52 | -------------------------------------------------------------------------------- /gitlab_tools/tools/formaters.py: -------------------------------------------------------------------------------- 1 | import base64 2 | from gitlab_tools.enums.VcsEnum import VcsEnum 3 | 4 | 5 | def format_bytes(num: int, suffix: str = 'B') -> str: 6 | """ 7 | Format bytes to human readable string 8 | @param num: 9 | @param suffix: 10 | @return: 11 | """ 12 | temp_num = float(num) 13 | for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: 14 | if abs(temp_num) < 1024: 15 | return "%3.1f %s%s" % (temp_num, unit, suffix) 16 | temp_num /= 1024 17 | return "%.1f%s%s" % (temp_num, 'Yi', suffix) 18 | 19 | 20 | def fix_url(url: str) -> str: 21 | """ 22 | Fixes url 23 | :param url: 24 | :return: 25 | """ 26 | if not url.startswith('http'): 27 | url = 'http://{}'.format(url) 28 | return url 29 | 30 | 31 | def format_boolean(bool_to_format: bool) -> str: 32 | """ 33 | Formats boolean 34 | :param bool_to_format: 35 | :return: 36 | """ 37 | if bool_to_format: 38 | return '
Yes
' 39 | 40 | return '
No
' 41 | 42 | 43 | def format_vcs(vcs_id: int) -> str: 44 | """ 45 | Formats vcs enum to string representation 46 | :param vcs_id: VcsEnum 47 | :return: str 48 | """ 49 | return { 50 | VcsEnum.GIT: 'Git', 51 | VcsEnum.SVN: 'SVN', 52 | VcsEnum.MERCURIAL: 'Mercurial', 53 | VcsEnum.BAZAAR: 'Bazaar', 54 | }.get(vcs_id, '?') 55 | 56 | 57 | def format_md5_fingerprint(fingerprint: bytes) -> str: 58 | """ 59 | Formats md5 fingerprint 60 | :param fingerprint: fingerprint to format 61 | :return: formated fingerprint 62 | """ 63 | fingerprint_hex = fingerprint.hex() 64 | return ':'.join(a + b for a, b in zip(fingerprint_hex[::2], fingerprint_hex[1::2])) 65 | 66 | 67 | def format_sha256_fingerprint(fingerprint: bytes) -> str: 68 | """ 69 | Formats sha256 fingerprint 70 | :param fingerprint: fingerprint to format 71 | :return: formated fingerprint 72 | """ 73 | return base64.b64encode(fingerprint).decode().rstrip('=') 74 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/20bcb4b2673c_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 20bcb4b2673c 4 | Revises: 82696705e6df 5 | Create Date: 2018-06-21 17:12:31.330775 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '20bcb4b2673c' 14 | down_revision = '82696705e6df' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('task_result', 22 | sa.Column('updated', sa.DateTime(), nullable=True), 23 | sa.Column('created', sa.DateTime(), nullable=True), 24 | sa.Column('id', sa.Integer(), nullable=False), 25 | sa.Column('pull_mirror_id', sa.Integer(), nullable=True), 26 | sa.Column('push_mirror_id', sa.Integer(), nullable=True), 27 | sa.Column('task_id', sa.String(length=155), nullable=True), 28 | sa.Column('status', sa.String(length=50), nullable=True), 29 | sa.Column('task_name', sa.String(length=255), nullable=True), 30 | sa.Column('invoked_by', sa.Integer(), nullable=True), 31 | sa.Column('result', sa.PickleType(), nullable=True), 32 | sa.Column('date_done', sa.DateTime(), nullable=True), 33 | sa.Column('traceback', sa.Text(), nullable=True), 34 | sa.ForeignKeyConstraint(['pull_mirror_id'], ['pull_mirror.id'], ), 35 | sa.ForeignKeyConstraint(['push_mirror_id'], ['push_mirror.id'], ), 36 | sa.PrimaryKeyConstraint('id'), 37 | sa.UniqueConstraint('task_id') 38 | ) 39 | op.create_index(op.f('ix_task_result_pull_mirror_id'), 'task_result', ['pull_mirror_id'], unique=False) 40 | op.create_index(op.f('ix_task_result_push_mirror_id'), 'task_result', ['push_mirror_id'], unique=False) 41 | # ### end Alembic commands ### 42 | 43 | 44 | def downgrade(): 45 | # ### commands auto generated by Alembic - please adjust! ### 46 | op.drop_index(op.f('ix_task_result_push_mirror_id'), table_name='task_result') 47 | op.drop_index(op.f('ix_task_result_pull_mirror_id'), table_name='task_result') 48 | op.drop_table('task_result') 49 | # ### end Alembic commands ### 50 | -------------------------------------------------------------------------------- /gitlab_tools/templates/macros.html: -------------------------------------------------------------------------------- 1 | {% macro render_pagination(pagination) %} 2 | 52 | 53 | {% endmacro %} -------------------------------------------------------------------------------- /gitlab_tools/views/home/templates/home.index.home.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% from 'macros.html' import render_pagination %} 3 | {% block body %} 4 | 5 | 6 |
7 |
8 | {% if private_key %} 9 |

{{ _('Your public key') }}

10 | 

{{private_key.get_name()}} {{private_key.get_base64()}}
11 |

{{_('MD5 Fingerprint')}} 12 | 
{{fingerprint_md5}}
13 |

{{_('SHA256 Fingerprint')}} 14 | 
{{fingerprint_sha256}}
15 |

16 |

{{_("Request new key")}}

17 | {% else %} 18 |
Not yet generated... is celeryworker runnung ?
19 | {% endif %} 20 |

21 |
22 | 23 |
24 |
25 |

{{_('Pull mirrors: %(pull_mirrors_count)s', pull_mirrors_count=pull_mirrors_count)}}

26 |

Pull mirrors allows you to pull code from other repositories to your gitlab installation.

27 |

View details »

28 |
29 |
30 |

{{_('Push mirrors: %(push_mirrors_count)s', push_mirrors_count=push_mirrors_count)}}

31 |

Push mirrors allows you to push your code from gitlab to other VCS services.

32 |

View details »

33 |
34 |
35 |

{{_('Fingerprints: %(fingerprint_count)s', fingerprint_count=fingerprint_count)}}

36 |

Fingerprints keeps your code/traffic safe.

37 |

View details »

38 |
39 |
40 | 41 | 42 | 43 | {% endblock %} -------------------------------------------------------------------------------- /archlinux/gitlab-tools.install: -------------------------------------------------------------------------------- 1 | ## arg 1: the new package version 2 | pre_install() { 3 | : 4 | } 5 | 6 | ## arg 1: the new package version 7 | post_install() { 8 | useradd -m gitlab-tools 9 | if ! systemctl is-enabled --quiet redis; then 10 | systemctl enable redis 11 | fi 12 | 13 | if ! systemctl is-active --quiet redis; then 14 | systemctl restart redis 15 | fi 16 | 17 | if ! systemctl is-enabled --quiet rabbitmq; then 18 | systemctl enable rabbitmq 19 | fi 20 | 21 | if ! systemctl is-active --quiet rabbitmq; then 22 | systemctl restart rabbitmq 23 | fi 24 | 25 | gitlab-tools post_install --config_prod --user=gitlab-tools 26 | systemctl daemon-reload 27 | systemctl start gitlab-tools 28 | systemctl enable gitlab-tools 29 | systemctl start gitlab-tools_celeryworker 30 | systemctl enable gitlab-tools_celeryworker 31 | systemctl start gitlab-tools_celerybeat 32 | systemctl enable gitlab-tools_celerybeat 33 | } 34 | 35 | ## arg 1: the new package version 36 | ## arg 2: the old package version 37 | pre_upgrade() { 38 | : 39 | } 40 | 41 | ## arg 1: the new package version 42 | ## arg 2: the old package version 43 | post_upgrade() { 44 | gitlab-tools db upgrade 45 | systemctl daemon-reload 46 | 47 | # Restart service only when is active or enabled 48 | if systemctl is-active --quiet gitlab-tools || systemctl is-enabled --quiet gitlab-tools; then 49 | systemctl restart gitlab-tools 50 | fi 51 | 52 | if systemctl is-active --quiet gitlab-tools_celeryworker || systemctl is-enabled --quiet gitlab-tools_celeryworker; then 53 | systemctl restart gitlab-tools_celeryworker 54 | fi 55 | 56 | if systemctl is-active --quiet gitlab-tools_celerybeat || systemctl is-enabled --quiet gitlab-tools_celerybeat; then 57 | systemctl restart gitlab-tools_celerybeat 58 | fi 59 | } 60 | 61 | ## arg 1: the old package version 62 | pre_remove() { 63 | : 64 | } 65 | 66 | ## arg 1: the old package version 67 | post_remove() { 68 | : 69 | } 70 | -------------------------------------------------------------------------------- /gitlab_tools/forms/push_mirror.py: -------------------------------------------------------------------------------- 1 | from flask_babel import gettext 2 | from flask_login import current_user 3 | from wtforms import Form, StringField, validators, HiddenField, TextAreaField, BooleanField 4 | from gitlab_tools.forms.custom_fields import NonValidatingSelectField 5 | from gitlab_tools.models.gitlab_tools import PushMirror 6 | from gitlab_tools.tools.GitRemote import GitRemote 7 | 8 | __author__ = "Adam Schubert" 9 | __date__ = "$26.7.2017 19:33:05$" 10 | 11 | 12 | class NewForm(Form): 13 | project_mirror = StringField(None, [validators.Length(min=1, max=255)]) 14 | note = TextAreaField(None, [validators.Optional()]) 15 | project = NonValidatingSelectField(None, [validators.Optional()], coerce=int, choices=[]) 16 | 17 | is_force_update = BooleanField() 18 | is_prune_mirrors = BooleanField() 19 | 20 | def validate(self) -> bool: 21 | rv = Form.validate(self) 22 | if not rv: 23 | return False 24 | 25 | project_mirror_exists = PushMirror.query.filter_by(project_mirror=self.project_mirror.data, user=current_user).first() 26 | if project_mirror_exists: 27 | self.project_mirror.errors.append( 28 | gettext('Project mirror %(project_mirror)s already exists.', project_mirror=self.project_mirror.data) 29 | ) 30 | return False 31 | 32 | if not GitRemote(self.project_mirror.data).vcs_type: 33 | self.project_mirror.errors.append( 34 | gettext('Unknown VCS type or detection failed.') 35 | ) 36 | return False 37 | 38 | return True 39 | 40 | 41 | class EditForm(NewForm): 42 | id = HiddenField() 43 | 44 | def validate(self) -> bool: 45 | rv = Form.validate(self) 46 | if not rv: 47 | return False 48 | 49 | project_mirror_exists = PushMirror.query.filter( 50 | PushMirror.project_mirror == self.project_mirror.data, 51 | PushMirror.id != self.id.data, 52 | PushMirror.user == current_user 53 | ).first() 54 | if project_mirror_exists: 55 | self.project_mirror.errors.append( 56 | gettext('Project mirror %(project_mirror)s already exists.', project_mirror=self.project_mirror.data) 57 | ) 58 | return False 59 | 60 | return True 61 | -------------------------------------------------------------------------------- /doc/PostgreSQL.md: -------------------------------------------------------------------------------- 1 | # Guide how to use/migreate to PostgreSQL (Debian and derivates) 2 | 3 | ## Install PostgreSQL 4 | ```bash 5 | sudo apt update 6 | sudo apt install postgresql 7 | ``` 8 | 9 | ## Configure PostgreSQL database and user 10 | 11 | Log in as postgres user and start psql console 12 | 13 | ```bash 14 | su postgres 15 | psql 16 | ``` 17 | 18 | Create user 19 | 20 | ```psql 21 | CREATE USER gitlab_tools WITH PASSWORD ; 22 | ``` 23 | 24 | Create database 25 | 26 | ```psql 27 | CREATE DATABASE gitlab_tools; 28 | ``` 29 | 30 | Grant privileges 31 | 32 | ```psql 33 | GRANT ALL PRIVILEGES ON DATABASE gitlab_tools TO gitlab_tools; 34 | ``` 35 | 36 | Grant usage on schema public to gitlab_tools user 37 | ```psql 38 | GRANT USAGE ON SCHEMA public TO gitlab_tools; 39 | ``` 40 | 41 | Set owner of gitlab_tools database to gitlab_tools user 42 | 43 | ```psql 44 | ALTER DATABASE gitlab_tools OWNER TO gitlab_tools; 45 | ``` 46 | 47 | ## Set gitlab-tools configuration to use PostgreSQL database 48 | 49 | Edit `/etc/gitlab-tools/config.yml`: 50 | 51 | ```bash 52 | nano /etc/gitlab-tools/config.yml 53 | ``` 54 | 55 | and replace value of `SQLALCHEMY_DATABASE_URI` to look ~like this: 56 | 57 | ```yml 58 | SQLALCHEMY_DATABASE_URI: 'postgresql://gitlab_tools:@127.0.0.1/gitlab_tools' 59 | ``` 60 | and save your changes using: Ctrl + o 61 | 62 | 63 | ## Create empty database schema 64 | 65 | ```bash 66 | gitlab-tools create_all 67 | ``` 68 | 69 | 70 | ## Restart services for gitlab-tools to use new configuration 71 | 72 | ```bash 73 | systemctl restart gitlab-tools 74 | systemctl restart gitlab-tools_celeryworker 75 | systemctl restart gitlab-tools_celerybeat 76 | ``` 77 | 78 | ## Migrate data from old database to PostgreSQL 79 | 80 | ### SqLite 81 | 82 | #### Install sqlite3 83 | 84 | ```bash 85 | apt install sqlite3 86 | ``` 87 | 88 | #### Dump data from database 89 | 90 | ```bash 91 | sqlite3 gitlab_tools.db .schema > schema.sql 92 | sqlite3 gitlab_tools.db .dump > dump.sql 93 | grep -vx -f schema.sql dump.sql > data.sql 94 | ``` 95 | 96 | #### Insert data into PostgreSQL 97 | 98 | ``` 99 | su postgres 100 | psql gitlab_tools < /path/to/data.sql 101 | ``` 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /gitlab_tools/views/fingerprint/templates/fingerprint.index.fingerprint.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from 'macros.html' import render_pagination %} 4 | 5 |

{{ _('Fingerprints') }}

6 | 7 | {{ _('Add') }} 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | {% for item in host_keys_proccessed %} 19 | 20 | 21 | 53 | 56 | 57 | {% endfor %} 58 | 59 |
{{_("Hostname")}}{{_("Keys")}}
{{item['hostname']}} 22 | 23 | 24 | 25 | 28 | 31 | 34 | 35 | 36 | 37 | {% for key in item['keys'] %} 38 | 39 | 42 | 45 | 48 | 49 | {% endfor %} 50 | 51 |
26 | {{_('Type')}} 27 | 29 | {{_('MD5 Fingerprint')}} 30 | 32 | {{_('SHA256 Fingerprint')}} 33 |
40 | {{key['type']}} 41 | 43 | {{key['md5_fingerprint']}} 44 | 46 | {{key['sha256_fingerprint']}} 47 |
52 | 		54 | 55 |
60 | 61 | {% endblock %} 62 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | # - test 3 | - package 4 | - deploy 5 | 6 | #test:debian: 7 | # stage: test 8 | # script: 9 | # - apt-get update -qy 10 | # - apt-get install -y python3-dev python3-pip tox 11 | # - pip3 install . 12 | # - tox -e py39 13 | # tags: 14 | # - debian 15 | 16 | 17 | package:debian: 18 | stage: package 19 | script: 20 | - apt-get update -qy 21 | - apt-get install -y curl gnupg apt-transport-https wget 22 | - wget -O- https://repository.salamek.cz/deb/salamek.gpg | tee /usr/share/keyrings/salamek-archive-keyring.gpg 23 | - echo "deb [signed-by=/usr/share/keyrings/salamek-archive-keyring.gpg] https://repository.salamek.cz/deb/pub all main" | tee /etc/apt/sources.list.d/salamek.cz.list 24 | - apt-get update -qy 25 | - apt-get install -y nodejs dh-python npm python3-python-gitlab python3-paramiko python3-git python3-flask-celery-tools python3-dev redis-server python3-pip python3-stdeb python3-celery nodejs git python3-markupsafe python3-psycopg2 python3-dateutil python3-docopt python3-yaml python3-wtforms python3-raven python3-flask-migrate python3-flask-babel python3-flask-navigation python3-cron-descriptor python3-flask-login python3-kombu rabbitmq-server 26 | - rm -rf "./deb_dist" 27 | - cd gitlab_tools/static 28 | - npm install 29 | - cd - 30 | - export DEB_BUILD_OPTIONS=nocheck 31 | - python3 setup.py --command-packages=stdeb.command bdist_deb 32 | tags: 33 | - debian 34 | 35 | artifacts: 36 | paths: 37 | - deb_dist/*.deb 38 | expire_in: 1d 39 | 40 | 41 | repoupdate: 42 | stage: deploy 43 | variables: 44 | GIT_STRATEGY: none 45 | before_script: 46 | - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' 47 | - eval $(ssh-agent -s) 48 | - ssh-add <(echo "$SSH_PRIVATE_KEY") 49 | - mkdir -p ~/.ssh 50 | - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' 51 | script: 52 | - ssh www-data@repository 'rm -rf /var/www/repository.salamek.cz/cache/deb/pub/all/gitlab-tools*.deb' 53 | - scp deb_dist/*.deb www-data@repository:/var/www/repository.salamek.cz/cache/deb/pub/all 54 | - ssh www-data@repository '/var/www/repository.salamek.cz/deb-pub-update.sh' 55 | dependencies: 56 | - package:debian 57 | tags: 58 | - docker 59 | only: 60 | - tags 61 | -------------------------------------------------------------------------------- /gitlab_tools/views/home/index.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Tuple 3 | import flask 4 | import paramiko 5 | from flask_login import current_user, login_required 6 | from gitlab_tools.extensions import db 7 | from gitlab_tools.models.gitlab_tools import PullMirror, PushMirror, Fingerprint 8 | from gitlab_tools.tasks.gitlab_tools import create_rsa_pair 9 | from gitlab_tools.tools.helpers import get_user_private_key_path 10 | from gitlab_tools.tools.formaters import format_md5_fingerprint, format_sha256_fingerprint 11 | from gitlab_tools.tools.crypto import calculate_fingerprint 12 | from gitlab_tools.blueprints import home_index 13 | 14 | __author__ = "Adam Schubert" 15 | __date__ = "$26.7.2017 19:33:05$" 16 | 17 | 18 | @home_index.route('/', methods=['GET']) 19 | @login_required 20 | def get_home() -> Tuple[str, int]: 21 | pull_mirrors_count = PullMirror.query.filter_by(is_deleted=False, user=current_user).count() 22 | push_mirrors_count = PushMirror.query.filter_by(is_deleted=False, user=current_user).count() 23 | fingerprint_count = Fingerprint.query.filter_by(user=current_user).count() 24 | private_key_path = get_user_private_key_path(current_user, flask.current_app.config['USER']) 25 | if os.path.isfile(private_key_path): 26 | 27 | with open(private_key_path, 'r') as f: 28 | private_key = paramiko.RSAKey.from_private_key(f) 29 | fingerprint_md5 = format_md5_fingerprint(calculate_fingerprint(private_key, 'md5')) 30 | fingerprint_sha256 = format_sha256_fingerprint(calculate_fingerprint(private_key, 'sha256')) 31 | else: 32 | private_key = None 33 | fingerprint_md5 = None 34 | fingerprint_sha256 = None 35 | 36 | return flask.render_template( 37 | 'home.index.home.html', 38 | pull_mirrors_count=pull_mirrors_count, 39 | push_mirrors_count=push_mirrors_count, 40 | fingerprint_count=fingerprint_count, 41 | private_key=private_key, 42 | fingerprint_md5=fingerprint_md5, 43 | fingerprint_sha256=fingerprint_sha256 44 | ), 200 45 | 46 | 47 | @home_index.route('/new-rsa-key', methods=['GET']) 48 | def get_new_rsa_key() -> flask.Response: 49 | 50 | current_user.is_rsa_pair_set = False # pylint: disable=assigning-non-slot 51 | current_user.gitlab_deploy_key_id = None # pylint: disable=assigning-non-slot 52 | db.session.add(current_user) 53 | db.session.commit() 54 | 55 | create_rsa_pair.delay(current_user.id) 56 | flask.flash('New RSA pair key has been requested!', 'success') 57 | return flask.redirect(flask.url_for('home_index.get_home')) 58 | -------------------------------------------------------------------------------- /gitlab_tools/static/style.css: -------------------------------------------------------------------------------- 1 | 2 | html, body 3 | { 4 | height: 100%; 5 | } 6 | 7 | .container 8 | { 9 | width: 90%; 10 | } 11 | 12 | .container.login { 13 | height: calc(100% - 104px); 14 | } 15 | 16 | .table > tbody > tr > td { 17 | vertical-align: middle; 18 | } 19 | 20 | .navbar-default 21 | { 22 | border-radius: 0px; 23 | margin-bottom: 0px; 24 | } 25 | 26 | .alert 27 | { 28 | border-radius: 0px; 29 | margin-bottom: 0px; 30 | } 31 | 32 | .footer ul 33 | { 34 | list-style-type: none; 35 | } 36 | 37 | /*data option width is not working, dont know dont care*/ 38 | @media (min-width: 768px) 39 | { 40 | .modal-dialog { 41 | width: 1024px; 42 | } 43 | } 44 | 45 | 46 | .top-buffer { 47 | margin-top:20px; 48 | } 49 | 50 | .select2-result-group, .select2-result-project{ 51 | padding-top:4px; 52 | padding-bottom:3px; 53 | } 54 | 55 | .select2-result-group__avatar, .select2-result-project__avatar{ 56 | float:left; 57 | width:60px; 58 | margin-right:10px; 59 | } 60 | 61 | .select2-result-group__avatar img, .select2-result-project__avatar img{ 62 | width:100%; 63 | height:auto; 64 | border-radius:2px; 65 | } 66 | 67 | .select2-result-group__meta, .select2-result-project__meta{ 68 | margin-left:70px; 69 | } 70 | 71 | .select2-result-group__title, .select2-result-project__title{ 72 | color:black; 73 | font-weight:700; 74 | word-wrap:break-word; 75 | line-height:1.1; 76 | margin-bottom:4px; 77 | } 78 | 79 | .select2-result-group__description, .select2-result-project__description{ 80 | font-size:13px; 81 | color:#777; 82 | margin-top:4px; 83 | } 84 | 85 | .select2-results__option--highlighted .select2-result-group__title, 86 | .select2-results__option--highlighted .select2-result-project__title{ 87 | color:white; 88 | } 89 | 90 | .select2-results__option--highlighted .select2-result-group__description, 91 | .select2-results__option--highlighted .select2-result-project__forks, 92 | .select2-results__option--highlighted .select2-result-project__stars, 93 | .select2-results__option--highlighted .select2-result-project__description, 94 | .select2-results__option--highlighted .select2-result-project__issues{ 95 | color:#c6dcef; 96 | } 97 | 98 | 99 | .select2-result-project__forks,.select2-result-project__stars{ 100 | margin-right:1em; 101 | } 102 | 103 | .select2-result-project__forks,.select2-result-project__stars,.select2-result-project__issues{ 104 | display:inline-block; 105 | color:#aaa; 106 | font-size:11px; 107 | } -------------------------------------------------------------------------------- /gitlab_tools/views/fingerprint/templates/fingerprint.index.new.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from "_formhelpers.html" import render_field %} 4 |
5 |
6 |
7 |
8 | {{ _("New fingerprint") }} 9 |
10 |
11 | {% if form.errors %} 12 |
13 | {{_("There was some errors when proccesing a form.")}} 14 | 15 |
    16 | {% for field, errors in form.errors.items() %} 17 |
  • 18 | {{field}}: 19 |
      20 | {% for error in errors %} 21 |
    • {{error}}
      • 22 | {% endfor %} 23 |
    24 |
    • 25 | {% endfor %} 26 |
27 | 28 |
29 | {% endif %} 30 |
31 |
32 |
33 |
34 | 35 | {{form.hostname(class_="form-control input-sm check-signature", **{'placeholder': _('Domain'), 'data-toggle':"popover", 'data-trigger':"focus ", 'data-fingerprint-url': url_for('fingerprint_index.check_hostname_fingerprint'), 'data-fingerprint-add-url': url_for('fingerprint_index.add_hostname_fingerprint'), 'data-gitlab-url': config['GITLAB_SSH'], 'data-placement': "bottom", 'data-content': '
'.join(form.hostname.errors)})}} 36 |
37 |
38 |
39 |
40 |
41 | 42 |
43 |
44 | {{ form.csrf_token }} 45 |
46 |
47 |
48 |
49 |
50 | {% endblock %} -------------------------------------------------------------------------------- /gitlab_tools/views/push_mirror/templates/push_mirror.index.push_mirror.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from 'macros.html' import render_pagination %} 4 | 5 |

{{ _('Push mirrors') }}

6 | 7 | {{ _('Add') }} 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | {% for item in pagination.items %} 24 | {% set webhook_url = url_for('api_index.schedule_sync_push_mirror', mirror_id=item.id, token=item.hook_token, _external=True) %} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 41 | 42 | {% endfor %} 43 | 44 |
#{{ _('VCS') }}{{ _('Project name') }}{{ _('Mirror') }}{{ _('Webhook') }}{{ _('Last sync') }}{{ _('Created') }}
{{item.id}}{{item.foreign_vcs_type|format_vcs}}{{item.project.name_with_namespace}}{{item.project_mirror}}{{item.last_sync|format_datetime}}{{item.created|format_datetime}} 34 | {% if item.target %} 35 | {{ _('Trigger sync') }} 36 | {% endif %} 37 | 38 | 39 | 40 |
45 | 46 |
47 | {{render_pagination(pagination)}} 48 |
49 | {% endblock %} 50 | -------------------------------------------------------------------------------- /gitlab_tools/views/pull_mirror/templates/pull_mirror.index.pull_mirror.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from 'macros.html' import render_pagination %} 4 | 5 |

{{ _('Pull Mirrors') }}

6 | 7 | {{ _('Add') }} 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | {% for item in pagination.items %} 25 | {% set webhook_url = url_for('api_index.schedule_sync_pull_mirror', mirror_id=item.id, token=item.hook_token, _external=True) %} 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 43 | 44 | {% endfor %} 45 | 46 |
#{{ _('VCS') }}{{ _('Project name') }}{{ _('Mirror') }}{{ _('Webhook') }}{{ _('Periodic sync') }}{{ _('Last sync') }}{{ _('Created') }}
{{item.id}}{{item.foreign_vcs_type|format_vcs}}{{item.project_name}}{{item.project_mirror}}{{item.periodic_sync|format_cron_syntax}}{{item.last_sync|format_datetime}}{{item.created|format_datetime}} 36 | {% if item.project_id %} 37 | {{ _('Trigger sync') }} 38 | {% endif %} 39 | 40 | 41 | 42 |
47 | 48 |
49 | {{render_pagination(pagination)}} 50 |
51 | {% endblock %} 52 | -------------------------------------------------------------------------------- /gitlab_tools/tools/crypto.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | import sys 4 | import string 5 | import hashlib 6 | import socket 7 | import paramiko 8 | 9 | import Cryptodome.PublicKey.RSA as RSA 10 | from Cryptodome.Signature import PKCS1_v1_5 11 | from Cryptodome.Hash import SHA256 12 | 13 | 14 | def sign_data(data: bytes, private_key: RSA) -> bytes: 15 | """ 16 | Signs message 17 | :param data: message to sign 18 | :param private_key: private key to use 19 | :return: bytes 20 | """ 21 | signer = PKCS1_v1_5.new(private_key) 22 | digest = SHA256.new() 23 | digest.update(data) 24 | return signer.sign(digest) 25 | 26 | 27 | def verify_data(data: bytes, signature: bytes, public_key: RSA) -> bool: 28 | """ 29 | Verifies message 30 | :param data: Message to verify 31 | :param signature: signature of message to verify 32 | :param public_key: public key to use 33 | :return: bool 34 | """ 35 | signer = PKCS1_v1_5.new(public_key) 36 | digest = SHA256.new() 37 | digest.update(data) 38 | try: 39 | signer.verify(digest, signature) # pylint: disable=not-callable 40 | return True 41 | except ValueError: 42 | return False 43 | 44 | 45 | def import_key(key_path: str) -> RSA: 46 | with open(key_path, 'r') as f: 47 | return RSA.importKey(f.read()) 48 | 49 | 50 | def random_password() -> str: 51 | """ 52 | Generates random password with fixed len of 64 characters 53 | :return: 64 len string 54 | """ 55 | return hashlib.sha256('{}_{}_{}'.format( 56 | random.randint(0, sys.maxsize), # nosec: B311 57 | round(time.time() * 1000), 58 | ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(20)) 59 | ).encode('UTF-8')).hexdigest() 60 | 61 | 62 | def get_remote_server_key(ip: str, port: int=22) -> paramiko.pkey.PKey: 63 | """ 64 | Returns PKey for given server 65 | :param ip: IP or Hostname 66 | :param port: Port 67 | :return: Returns PKey 68 | """ 69 | 70 | my_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 71 | my_socket.settimeout(10) 72 | my_socket.connect((ip, port)) 73 | 74 | my_transport = paramiko.Transport(my_socket) 75 | my_transport.start_client() 76 | ssh_key = my_transport.get_remote_server_key() 77 | 78 | my_transport.close() 79 | my_socket.close() 80 | return ssh_key 81 | 82 | 83 | def calculate_fingerprint(pkey: paramiko.pkey.PKey, algorithm: str='md5') -> bytes: 84 | """ 85 | Calculates fingerprint for PKey 86 | :param pkey: pkey 87 | :param algorithm: algoright to use 88 | :return: fingerprint 89 | """ 90 | 91 | h = hashlib.new(algorithm) 92 | h.update(pkey.asbytes()) 93 | return h.digest() 94 | -------------------------------------------------------------------------------- /gitlab_tools/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import getpass 4 | 5 | 6 | class HardCoded: 7 | """Constants used throughout the application. 8 | 9 | All hard coded settings/data that are not actual/official configuration options for Flask, Celery, or their 10 | extensions goes here. 11 | """ 12 | ADMINS = ['adam.schubert@sg1-game.net'] 13 | DB_MODELS_IMPORTS = ('gitlab_tools', 'celery') # Like CELERY_IMPORTS in CeleryConfig. 14 | ENVIRONMENT = property(lambda self: self.__class__.__name__) 15 | SQLALCHEMY_TRACK_MODIFICATIONS = False 16 | SUPPORTED_LANGUAGES = {'cs': 'Čeština', 'en': 'English'} 17 | LANGUAGE = 'en' 18 | 19 | 20 | class CeleryConfig(HardCoded): 21 | """Configurations used by Celery only.""" 22 | CELERYD_PREFETCH_MULTIPLIER = 1 23 | CELERYD_TASK_SOFT_TIME_LIMIT = 20 * 60 # Raise exception if task takes too long. 24 | CELERYD_TASK_TIME_LIMIT = 30 * 60 # Kill worker if task takes way too long. 25 | CELERY_ACCEPT_CONTENT = ['json', 'pickle'] 26 | CELERY_ACKS_LATE = True 27 | CELERY_DISABLE_RATE_LIMITS = True 28 | CELERY_IMPORTS = ('gitlab_tools', ) 29 | CELERY_RESULT_SERIALIZER = 'json' 30 | CELERY_TASK_RESULT_EXPIRES = None 31 | CELERY_TASK_SERIALIZER = 'json' 32 | CELERY_TRACK_STARTED = True 33 | CELERY_DEFAULT_QUEUE = 'gitlab_tools' 34 | CELERY_BEAT_SCHEDULER = 'gitlab_tools.celery_beat.schedulers.DatabaseScheduler' 35 | 36 | 37 | class Config(CeleryConfig): 38 | """Default Flask configuration inherited by all environments. Use this for development environments.""" 39 | DEBUG = True 40 | TESTING = False 41 | SECRET_KEY = "i_don't_want_my_cookies_expiring_while_developing" # nosec: B105 42 | SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/gitlab-tools.db' 43 | CELERY_BROKER_URL = 'amqp://127.0.0.1:5672/' 44 | CELERY_TASK_LOCK_BACKEND = 'redis://127.0.0.1/0' 45 | PORT = 5000 46 | HOST = '0.0.0.0' # nosec: B104 47 | GITLAB_API_VERSION = 4 48 | USER = getpass.getuser() 49 | 50 | @property 51 | def CELERY_RESULT_BACKEND(self) -> str: 52 | return 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) 53 | 54 | 55 | class Testing(Config): 56 | TESTING = True 57 | CELERY_ALWAYS_EAGER = True 58 | CELERY_BROKER_URL = 'amqp://127.0.0.1:5672/' 59 | CELERY_TASK_LOCK_BACKEND = 'redis://127.0.0.1/0' 60 | 61 | 62 | class Production(Config): 63 | DEBUG = False 64 | SERVER_NAME = None 65 | SECRET_KEY = None # To be overwritten by a YAML file. 66 | SQLALCHEMY_DATABASE_URI = None 67 | PORT = None # To be overwritten by a YAML file. 68 | HOST = None # To be overwritten by a YAML file. 69 | GITLAB_URL = None # To be overwritten by a YAML file. 70 | GITLAB_APP_ID = None # To be overwritten by a YAML file. 71 | GITLAB_APP_SECRET = None # To be overwritten by a YAML file. 72 | GITLAB_SSH = None 73 | -------------------------------------------------------------------------------- /gitlab_tools/tools/jsonifier.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import functools 3 | import json 4 | import typing as t 5 | import uuid 6 | from decimal import Decimal 7 | 8 | 9 | def wrap_default(default_fn: t.Callable) -> t.Callable: 10 | """The Connexion defaults for JSON encoding. Handles extra types compared to the 11 | built-in :class:`json.JSONEncoder`. 12 | 13 | - :class:`datetime.datetime` and :class:`datetime.date` are 14 | serialized to :rfc:`822` strings. This is the same as the HTTP 15 | date format. 16 | - :class:`decimal.Decimal` is serialized to a float. 17 | - :class:`uuid.UUID` is serialized to a string. 18 | """ 19 | 20 | @functools.wraps(default_fn) 21 | def wrapped_default(self, o): 22 | if isinstance(o, datetime.datetime): 23 | if o.tzinfo: 24 | # eg: '2015-09-25T23:14:42.588601+00:00' 25 | return o.isoformat("T") 26 | else: 27 | # No timezone present - assume UTC. 28 | # eg: '2015-09-25T23:14:42.588601Z' 29 | return o.isoformat("T") + "Z" 30 | 31 | if isinstance(o, datetime.date): 32 | return o.isoformat() 33 | 34 | if isinstance(o, Decimal): 35 | return float(o) 36 | 37 | if isinstance(o, uuid.UUID): 38 | return str(o) 39 | 40 | return default_fn(o) 41 | 42 | return wrapped_default 43 | 44 | 45 | class JSONEncoder(json.JSONEncoder): 46 | """The default Connexion JSON encoder. Handles extra types compared to the 47 | built-in :class:`json.JSONEncoder`. 48 | """ 49 | 50 | @wrap_default 51 | def default(self, o): 52 | return super().default(o) 53 | 54 | 55 | class Jsonifier: 56 | """ 57 | Central point to serialize and deserialize to/from JSon in Connexion. 58 | """ 59 | 60 | def __init__(self, json_=json, **kwargs): 61 | """ 62 | :param json_: json library to use. Must have loads() and dumps() method # NOQA 63 | :param kwargs: default arguments to pass to json.dumps() 64 | """ 65 | self.json = json_ 66 | self.dumps_args = kwargs 67 | self.dumps_args.setdefault("cls", JSONEncoder) 68 | 69 | def dumps(self, data, **kwargs): 70 | """Central point where JSON serialization happens inside 71 | Connexion. 72 | """ 73 | for k, v in self.dumps_args.items(): 74 | kwargs.setdefault(k, v) 75 | return self.json.dumps(data, **kwargs) + "\n" 76 | 77 | def loads(self, data): 78 | """Central point where JSON deserialization happens inside 79 | Connexion. 80 | """ 81 | if isinstance(data, bytes): 82 | data = data.decode() 83 | 84 | try: 85 | return self.json.loads(data) 86 | except Exception: 87 | if isinstance(data, str): 88 | return data 89 | -------------------------------------------------------------------------------- /gitlab_tools/tools/GitUri.py: -------------------------------------------------------------------------------- 1 | import re 2 | import urllib.parse 3 | from gitlab_tools.enums.ProtocolEnum import ProtocolEnum 4 | 5 | 6 | class GitUri: 7 | 8 | # Mapping of scheme to (default_port, ProtocolEnum) 9 | scheme_to_info = { 10 | 'ssh': (22, ProtocolEnum.SSH), 11 | 'git': (22, ProtocolEnum.SSH), 12 | 'https': (443, ProtocolEnum.HTTPS), 13 | 'http': (80, ProtocolEnum.HTTP) 14 | } 15 | 16 | def __init__(self, uri: str, default_scheme: str = 'ssh'): 17 | self.default_scheme = default_scheme 18 | 19 | if not re.match(r'^\S+://', uri): 20 | # URI has no scheme, prepend one 21 | uri = '{}://{}'.format(self.default_scheme, uri) 22 | 23 | parsed = urllib.parse.urlparse(uri) 24 | default_port, protocol = self._detect_scheme_info(parsed.scheme) 25 | 26 | self.default_port = default_port 27 | self.scheme = parsed.scheme 28 | self.username = parsed.username 29 | self.password = parsed.password 30 | self.hostname = parsed.hostname 31 | self.path = parsed.path 32 | self.protocol = protocol 33 | 34 | try: 35 | self.port = parsed.port or self.default_port 36 | except ValueError: 37 | self.port = self.default_port 38 | 39 | # Check if we have single : in netloc, that means no port was provided but : was still there (GIT format) 40 | # Remove login info 41 | if '@' in parsed.netloc: 42 | _, netloc = parsed.netloc.split('@') 43 | else: 44 | netloc = parsed.netloc 45 | 46 | if ':' in netloc: 47 | hostname, path_prefix = netloc.split(':') 48 | if not path_prefix.isnumeric(): 49 | self.path = path_prefix + parsed.path 50 | 51 | # Check that path starts with / 52 | if not self.path.startswith('/'): 53 | self.path = '/{}'.format(self.path) 54 | 55 | def _detect_scheme_info(self, scheme: str): 56 | exact_match = self.scheme_to_info.get(scheme) 57 | if exact_match: 58 | return exact_match 59 | 60 | for key, value in self.scheme_to_info.items(): 61 | if key in scheme: 62 | return value 63 | 64 | @property 65 | def url(self) -> str: 66 | return self.build_url() 67 | 68 | def build_url(self, ignore_default_port: bool = False) -> str: 69 | hostname_parts = [] 70 | if self.username: 71 | hostname_parts.append(self.username) 72 | 73 | if self.password: 74 | hostname_parts.append(':') 75 | hostname_parts.append(self.password) 76 | 77 | if self.username: 78 | hostname_parts.append('@') 79 | 80 | hostname_parts.append(self.hostname) 81 | 82 | port = self.port 83 | if self.port == self.default_port and ignore_default_port: 84 | port = '' 85 | 86 | return '{}://{}{}{}{}'.format( 87 | self.scheme, 88 | ''.join(hostname_parts), 89 | ':' if self.protocol == ProtocolEnum.SSH or port else '', 90 | port, 91 | self.path 92 | ) 93 | 94 | def __str__(self) -> str: 95 | return self.url 96 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | from logging.config import fileConfig 3 | import logging 4 | from alembic import context 5 | from sqlalchemy import engine_from_config, pool 6 | from flask import current_app 7 | 8 | 9 | # this is the Alembic Config object, which provides 10 | # access to the values within the .ini file in use. 11 | config = context.config 12 | 13 | # Interpret the config file for Python logging. 14 | # This line sets up loggers basically. 15 | fileConfig(config.config_file_name) 16 | logger = logging.getLogger('alembic.env') 17 | 18 | # add your model's MetaData object here 19 | # for 'autogenerate' support 20 | # from myapp import mymodel 21 | # target_metadata = mymodel.Base.metadata 22 | 23 | config.set_main_option('sqlalchemy.url', 24 | current_app.config.get('SQLALCHEMY_DATABASE_URI')) 25 | target_metadata = current_app.extensions['migrate'].db.metadata 26 | # other values from the config, defined by the needs of env.py, 27 | # can be acquired: 28 | # my_important_option = config.get_main_option("my_important_option") 29 | # ... etc. 30 | 31 | 32 | def run_migrations_offline(): 33 | """Run migrations in 'offline' mode. 34 | 35 | This configures the context with just a URL 36 | and not an Engine, though an Engine is acceptable 37 | here as well. By skipping the Engine creation 38 | we don't even need a DBAPI to be available. 39 | 40 | Calls to context.execute() here emit the given string to the 41 | script output. 42 | 43 | """ 44 | url = config.get_main_option("sqlalchemy.url") 45 | context.configure(url=url, compare_type=True) 46 | 47 | with context.begin_transaction(): 48 | context.run_migrations() 49 | 50 | 51 | def run_migrations_online(): 52 | """Run migrations in 'online' mode. 53 | 54 | In this scenario we need to create an Engine 55 | and associate a connection with the context. 56 | 57 | """ 58 | 59 | # this callback is used to prevent an auto-migration from being generated 60 | # when there are no changes to the schema 61 | # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html 62 | def process_revision_directives(_context, _revision, directives): 63 | if getattr(config.cmd_opts, 'autogenerate', False): 64 | script = directives[0] 65 | if script.upgrade_ops.is_empty(): 66 | directives[:] = [] 67 | logger.info('No changes in schema detected.') 68 | 69 | engine = engine_from_config(config.get_section(config.config_ini_section), 70 | prefix='sqlalchemy.', 71 | poolclass=pool.NullPool) 72 | 73 | connection = engine.connect() 74 | context.configure(connection=connection, 75 | target_metadata=target_metadata, 76 | process_revision_directives=process_revision_directives, 77 | **current_app.extensions['migrate'].configure_args) 78 | 79 | try: 80 | with context.begin_transaction(): 81 | context.run_migrations() 82 | finally: 83 | connection.close() 84 | 85 | if context.is_offline_mode(): 86 | run_migrations_offline() 87 | else: 88 | run_migrations_online() 89 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/d4841aeeb072_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: d4841aeeb072 4 | Revises: 20bcb4b2673c 5 | Create Date: 2018-06-21 22:06:12.215774 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'd4841aeeb072' 14 | down_revision = '20bcb4b2673c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('celery_taskmeta', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('task_id', sa.String(length=155), nullable=True), 24 | sa.Column('status', sa.String(length=50), nullable=True), 25 | sa.Column('result', sa.PickleType(), nullable=True), 26 | sa.Column('date_done', sa.DateTime(), nullable=True), 27 | sa.Column('traceback', sa.Text(), nullable=True), 28 | sa.PrimaryKeyConstraint('id'), 29 | sa.UniqueConstraint('task_id'), 30 | sqlite_autoincrement=True 31 | ) 32 | op.create_table('celery_tasksetmeta', 33 | sa.Column('id', sa.Integer(), nullable=False), 34 | sa.Column('taskset_id', sa.String(length=155), nullable=True), 35 | sa.Column('result', sa.PickleType(), nullable=True), 36 | sa.Column('date_done', sa.DateTime(), nullable=True), 37 | sa.PrimaryKeyConstraint('id'), 38 | sa.UniqueConstraint('taskset_id'), 39 | sqlite_autoincrement=True 40 | ) 41 | op.add_column('task_result', sa.Column('celery_taskmeta_id', sa.Integer(), nullable=False)) 42 | op.create_index(op.f('ix_task_result_celery_taskmeta_id'), 'task_result', ['celery_taskmeta_id'], unique=False) 43 | op.drop_constraint('task_result_task_id_key', 'task_result', type_='unique') 44 | op.create_foreign_key(None, 'task_result', 'celery_taskmeta', ['celery_taskmeta_id'], ['id']) 45 | op.drop_column('task_result', 'result') 46 | op.drop_column('task_result', 'status') 47 | op.drop_column('task_result', 'date_done') 48 | op.drop_column('task_result', 'task_id') 49 | op.drop_column('task_result', 'traceback') 50 | # ### end Alembic commands ### 51 | 52 | 53 | def downgrade(): 54 | # ### commands auto generated by Alembic - please adjust! ### 55 | op.add_column('task_result', sa.Column('traceback', sa.TEXT(), autoincrement=False, nullable=True)) 56 | op.add_column('task_result', sa.Column('task_id', sa.VARCHAR(length=155), autoincrement=False, nullable=True)) 57 | op.add_column('task_result', sa.Column('date_done', postgresql.TIMESTAMP(), autoincrement=False, nullable=True)) 58 | op.add_column('task_result', sa.Column('status', sa.VARCHAR(length=50), autoincrement=False, nullable=True)) 59 | op.add_column('task_result', sa.Column('result', postgresql.BYTEA(), autoincrement=False, nullable=True)) 60 | op.drop_constraint(None, 'task_result', type_='foreignkey') 61 | op.create_unique_constraint('task_result_task_id_key', 'task_result', ['task_id']) 62 | op.drop_index(op.f('ix_task_result_celery_taskmeta_id'), table_name='task_result') 63 | op.drop_column('task_result', 'celery_taskmeta_id') 64 | op.drop_table('celery_tasksetmeta') 65 | op.drop_table('celery_taskmeta') 66 | # ### end Alembic commands ### 67 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/6ea24c49b7a1_.py: -------------------------------------------------------------------------------- 1 | """Add tables needed for celery beat DatabaseScheduler 2 | 3 | Revision ID: 6ea24c49b7a1 4 | Revises: 19e8725e0581 5 | Create Date: 2018-06-08 01:15:31.961824 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '6ea24c49b7a1' 14 | down_revision = '19e8725e0581' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('crontab_schedule', 22 | sa.Column('updated', sa.DateTime(), nullable=True), 23 | sa.Column('created', sa.DateTime(), nullable=True), 24 | sa.Column('id', sa.Integer(), nullable=False), 25 | sa.Column('minute', sa.String(length=120), nullable=True), 26 | sa.Column('hour', sa.String(length=120), nullable=True), 27 | sa.Column('day_of_week', sa.String(length=120), nullable=True), 28 | sa.Column('day_of_month', sa.String(length=120), nullable=True), 29 | sa.Column('month_of_year', sa.String(length=120), nullable=True), 30 | sa.PrimaryKeyConstraint('id') 31 | ) 32 | op.create_table('interval_schedule', 33 | sa.Column('updated', sa.DateTime(), nullable=True), 34 | sa.Column('created', sa.DateTime(), nullable=True), 35 | sa.Column('id', sa.Integer(), nullable=False), 36 | sa.Column('every', sa.Integer(), nullable=False), 37 | sa.Column('period', sa.Unicode(length=255), nullable=True), 38 | sa.PrimaryKeyConstraint('id') 39 | ) 40 | op.create_table('periodic_tasks', 41 | sa.Column('updated', sa.DateTime(), nullable=True), 42 | sa.Column('created', sa.DateTime(), nullable=True), 43 | sa.Column('id', sa.Integer(), nullable=False), 44 | sa.Column('ident', sa.Integer(), nullable=True), 45 | sa.Column('last_update', sa.DateTime(), nullable=True), 46 | sa.PrimaryKeyConstraint('id') 47 | ) 48 | op.create_index(op.f('ix_periodic_tasks_ident'), 'periodic_tasks', ['ident'], unique=False) 49 | op.create_table('periodic_task', 50 | sa.Column('updated', sa.DateTime(), nullable=True), 51 | sa.Column('created', sa.DateTime(), nullable=True), 52 | sa.Column('id', sa.Integer(), nullable=False), 53 | sa.Column('name', sa.String(length=120), nullable=True), 54 | sa.Column('task', sa.String(length=120), nullable=True), 55 | sa.Column('crontab_id', sa.Integer(), nullable=True), 56 | sa.Column('interval_id', sa.Integer(), nullable=True), 57 | sa.Column('args', sa.String(length=120), nullable=True), 58 | sa.Column('kwargs', sa.String(length=120), nullable=True), 59 | sa.Column('last_run_at', sa.DateTime(), nullable=True), 60 | sa.Column('total_run_count', sa.Integer(), nullable=True), 61 | sa.Column('enabled', sa.Boolean(), nullable=True), 62 | sa.ForeignKeyConstraint(['crontab_id'], ['crontab_schedule.id'], ), 63 | sa.ForeignKeyConstraint(['interval_id'], ['interval_schedule.id'], ), 64 | sa.PrimaryKeyConstraint('id'), 65 | sa.UniqueConstraint('name') 66 | ) 67 | # ### end Alembic commands ### 68 | 69 | 70 | def downgrade(): 71 | # ### commands auto generated by Alembic - please adjust! ### 72 | op.drop_table('periodic_task') 73 | op.drop_index(op.f('ix_periodic_tasks_ident'), table_name='periodic_tasks') 74 | op.drop_table('periodic_tasks') 75 | op.drop_table('interval_schedule') 76 | op.drop_table('crontab_schedule') 77 | # ### end Alembic commands ### 78 | -------------------------------------------------------------------------------- /gitlab_tools/tools/Git.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess # nosec: B404 3 | import logging 4 | from git import Repo 5 | from gitlab_tools.tools.Svn import Svn 6 | from gitlab_tools.enums.VcsEnum import VcsEnum 7 | from gitlab_tools.tools.GitRemote import GitRemote 8 | 9 | 10 | class Git: 11 | 12 | @staticmethod 13 | def sync_mirror(namespace_path: str, temp_name: str, source: GitRemote, target: GitRemote=None): 14 | 15 | # Check if repository storage group directory exists: 16 | if not os.path.isdir(namespace_path): 17 | raise Exception('Group storage {} not found, creation failed ?'.format(namespace_path)) 18 | 19 | # Check if project clone exists 20 | project_path = os.path.join(namespace_path, temp_name) 21 | if not os.path.isdir(project_path): 22 | raise Exception('Repository storage {} not found, creation failed ?'.format(project_path)) 23 | 24 | repo = Repo(project_path) 25 | # Special code for SVN repo mirror 26 | if source.vcs_type == VcsEnum.SVN: 27 | repo.git.reset('--hard') 28 | repo.git.svn('fetch') 29 | repo.git.svn('rebase') 30 | 31 | if target: 32 | #repo.git.config('--bool', 'core.bare', 'true') 33 | repo.remotes.gitlab.push(refspec='master') 34 | #repo.git.config('--bool', 'core.bare', 'false') 35 | 36 | else: 37 | # Everything else 38 | repo.remotes.origin.fetch(force=source.is_force_update, prune=source.is_prune_mirrors) 39 | repo.remotes.gitlab.push( 40 | mirror=True, 41 | force=target.is_force_update, 42 | prune=target.is_prune_mirrors 43 | ) 44 | 45 | logging.info('Mirror sync done') 46 | 47 | @staticmethod 48 | def create_mirror(namespace_path: str, temp_name: str, source: GitRemote, target: GitRemote=None) -> None: 49 | 50 | # 2. Create/pull local repository 51 | 52 | # Check if project clone exists 53 | project_path = os.path.join(namespace_path, temp_name) 54 | if os.path.isdir(project_path): 55 | repo = Repo(project_path) 56 | 57 | # SVN REPO has no origin 58 | if source.vcs_type not in [VcsEnum.SVN]: 59 | repo.remotes.origin.set_url(source.url) 60 | 61 | if target: 62 | repo.remotes.gitlab.set_url(target.url) 63 | else: 64 | # Project not found, we can clone 65 | logging.info('Creating mirror for %s', source.url) 66 | 67 | # 3. Pull 68 | # 4. Push 69 | 70 | if source.vcs_type == VcsEnum.SVN: 71 | subprocess.Popen( # nosec: B607, B603 72 | ['git', 'svn', 'clone', Svn.fix_url(source.url), project_path], 73 | cwd=namespace_path 74 | ).communicate() 75 | repo = Repo(project_path) 76 | else: 77 | repo = Repo.clone_from(source.url, project_path, mirror=True) 78 | 79 | if source.vcs_type in [VcsEnum.BAZAAR, VcsEnum.MERCURIAL]: 80 | repo.git.gc('--aggressive') 81 | 82 | if target: 83 | logging.info('Adding GitLab remote to project.') 84 | 85 | repo.create_remote('gitlab', target.url) 86 | 87 | Git.sync_mirror(namespace_path, temp_name, source, target) 88 | 89 | logging.info('All done!') 90 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/56c59adcfe10_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 56c59adcfe10 4 | Revises: 0bf6832fd1f2 5 | Create Date: 2023-01-03 09:45:16.743950 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '56c59adcfe10' 14 | down_revision = '0bf6832fd1f2' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('periodic_task', sa.Column('queue', sa.String(length=200), nullable=True)) 22 | op.add_column('periodic_task', sa.Column('exchange', sa.String(length=200), nullable=True)) 23 | op.add_column('periodic_task', sa.Column('routing_key', sa.String(length=200), nullable=True)) 24 | op.add_column('periodic_task', sa.Column('headers', sa.Text(), nullable=True)) 25 | op.add_column('periodic_task', sa.Column('priority', sa.Integer(), nullable=True)) 26 | op.add_column('periodic_task', sa.Column('expires', sa.DateTime(), nullable=True)) 27 | op.add_column('periodic_task', sa.Column('expire_seconds', sa.Integer(), nullable=True)) 28 | op.add_column('periodic_task', sa.Column('one_off', sa.Boolean(), nullable=True)) 29 | op.add_column('periodic_task', sa.Column('start_time', sa.DateTime(), nullable=True)) 30 | op.alter_column('periodic_task', 'name', 31 | existing_type=sa.VARCHAR(length=120), 32 | type_=sa.String(length=200), 33 | existing_nullable=True) 34 | op.alter_column('periodic_task', 'task', 35 | existing_type=sa.VARCHAR(length=120), 36 | type_=sa.String(length=200), 37 | existing_nullable=True) 38 | op.alter_column('periodic_task', 'args', 39 | existing_type=sa.VARCHAR(length=120), 40 | type_=sa.Text(), 41 | existing_nullable=True) 42 | op.alter_column('periodic_task', 'kwargs', 43 | existing_type=sa.VARCHAR(length=120), 44 | type_=sa.Text(), 45 | existing_nullable=True) 46 | # ### end Alembic commands ### 47 | 48 | 49 | def downgrade(): 50 | # ### commands auto generated by Alembic - please adjust! ### 51 | op.alter_column('periodic_task', 'kwargs', 52 | existing_type=sa.Text(), 53 | type_=sa.VARCHAR(length=120), 54 | existing_nullable=True) 55 | op.alter_column('periodic_task', 'args', 56 | existing_type=sa.Text(), 57 | type_=sa.VARCHAR(length=120), 58 | existing_nullable=True) 59 | op.alter_column('periodic_task', 'task', 60 | existing_type=sa.String(length=200), 61 | type_=sa.VARCHAR(length=120), 62 | existing_nullable=True) 63 | op.alter_column('periodic_task', 'name', 64 | existing_type=sa.String(length=200), 65 | type_=sa.VARCHAR(length=120), 66 | existing_nullable=True) 67 | op.drop_column('periodic_task', 'start_time') 68 | op.drop_column('periodic_task', 'one_off') 69 | op.drop_column('periodic_task', 'expire_seconds') 70 | op.drop_column('periodic_task', 'expires') 71 | op.drop_column('periodic_task', 'priority') 72 | op.drop_column('periodic_task', 'headers') 73 | op.drop_column('periodic_task', 'routing_key') 74 | op.drop_column('periodic_task', 'exchange') 75 | op.drop_column('periodic_task', 'queue') 76 | # ### end Alembic commands ### 77 | -------------------------------------------------------------------------------- /gitlab_tools/views/push_mirror/templates/push_mirror.index.log.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from 'macros.html' import render_pagination %} 4 | 5 |

{{ _('Push Mirror %(mirror_name)s logs', mirror_name=push_mirror.project_mirror) }}

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | {% for task_result in pagination.items %} 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 32 | 33 | {% for task_result_children in task_result.children %} 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 44 | 45 | {% endfor %} 46 | {% endfor %} 47 | 48 |
{{ _('Task ID') }}{{ _('Task name') }}{{ _('Invoked by') }}{{ _('Status') }}{{ _('Result') }}{{ _('Date done') }}{{ _('Traceback') }}
{{task_result.taskmeta.task_id}}{{task_result.task_name}}{{task_result.invoked_by}}{{task_result.taskmeta.status}}{{task_result.taskmeta.result}}{{task_result.taskmeta.date_done|format_datetime}} 30 | 31 |
{{task_result_children.taskmeta.task_id}}{{task_result_children.task_name}}{{task_result_children.invoked_by|format_task_invoked_by}}{{task_result_children.taskmeta.status}}{{task_result_children.taskmeta.result}}{{task_result_children.taskmeta.date_done|format_datetime}} 42 | 43 |
49 | 50 |
51 | {{render_pagination(pagination)}} 52 |
53 | 70 | {% endblock %} 71 | -------------------------------------------------------------------------------- /gitlab_tools/views/pull_mirror/templates/pull_mirror.index.log.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from 'macros.html' import render_pagination %} 4 | 5 |

{{ _('Pull Mirror %(mirror_name)s logs', mirror_name=pull_mirror.project_mirror) }}

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | {% for task_result in pagination.items %} 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 32 | 33 | {% for task_result_children in task_result.children %} 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 44 | 45 | {% endfor %} 46 | {% endfor %} 47 | 48 | 49 |
{{ _('Task ID') }}{{ _('Task name') }}{{ _('Invoked by') }}{{ _('Status') }}{{ _('Result') }}{{ _('Date done') }}{{ _('Traceback') }}
{{task_result.taskmeta.task_id}}{{task_result.task_name}}{{task_result.invoked_by|format_task_invoked_by}}{{task_result.taskmeta.status}}{{task_result.taskmeta.result}}{{task_result.taskmeta.date_done|format_datetime}} 30 | 31 |
{{task_result_children.taskmeta.task_id}}{{task_result_children.task_name}}{{task_result_children.invoked_by|format_task_invoked_by}}{{task_result_children.taskmeta.status}}{{task_result_children.taskmeta.result}}{{task_result_children.taskmeta.date_done|format_datetime}} 42 | 43 |
50 | 51 |
52 | {{render_pagination(pagination)}} 53 |
54 | 71 | {% endblock %} 72 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import pathlib 4 | 5 | from setuptools import setup, find_packages 6 | 7 | sys_conf_dir = os.getenv("SYSCONFDIR", "/etc") 8 | 9 | 10 | current_directory = os.path.dirname(os.path.abspath(__file__)) 11 | 12 | here = pathlib.Path(__file__).parent.resolve() 13 | long_description = (here / "README.md").read_text(encoding="utf-8") 14 | 15 | 16 | def package_files(directory: str) -> list: 17 | paths = [] 18 | for (path, _, filenames) in os.walk(directory): 19 | for filename in filenames: 20 | paths.append(os.path.join('..', path, filename)) 21 | return paths 22 | 23 | 24 | classes = """ 25 | Development Status :: 4 - Beta 26 | Intended Audience :: Developers 27 | Programming Language :: Python 28 | Programming Language :: Python :: 3 29 | Programming Language :: Python :: 3.3 30 | Programming Language :: Python :: 3.4 31 | Programming Language :: Python :: 3.5 32 | Programming Language :: Python :: 3.6 33 | Programming Language :: Python :: Implementation :: CPython 34 | Programming Language :: Python :: Implementation :: PyPy 35 | Operating System :: OS Independent 36 | """ 37 | classifiers = [s.strip() for s in classes.split('\n') if s] 38 | 39 | 40 | extra_files = [ 41 | 'templates/*', 42 | 'migrations/alembic.ini', 43 | 'views/*/templates/*', 44 | 'views/*/templates/*/*', 45 | 'static/*' 46 | ] 47 | 48 | extra_files.extend(package_files('gitlab_tools/translations')) 49 | 50 | extra_files.extend(package_files('gitlab_tools/static/img')) 51 | 52 | # Bower components 53 | extra_files.extend(package_files('gitlab_tools/static/node_modules/bootstrap/dist')) 54 | extra_files.extend(package_files('gitlab_tools/static/node_modules/select2/dist')) 55 | extra_files.extend(package_files('gitlab_tools/static/node_modules/font-awesome/css')) 56 | extra_files.extend(package_files('gitlab_tools/static/node_modules/font-awesome/fonts')) 57 | extra_files.extend(package_files('gitlab_tools/static/node_modules/jquery/dist')) 58 | 59 | 60 | setup( 61 | name='gitlab-tools', 62 | version='1.5.3', 63 | python_requires='>=3.7', 64 | description='GitLab Tools', 65 | long_description=long_description, 66 | long_description_content_type='text/markdown', 67 | author='Adam Schubert', 68 | author_email='adam.schubert@sg1-game.net', 69 | url='https://gitlab.salamek.cz/sadam/gitlab-tools.git', 70 | license='GPL-3.0', 71 | classifiers=classifiers, 72 | packages=find_packages(exclude=['tests', 'tests.*']), 73 | install_requires=[ 74 | 'redis>=3.5.0,<=4.3.*', 75 | 'WTForms>=2.2.*, ~=3.0.1', 76 | 'psycopg2-binary~=2.9.5', 77 | 'pycryptodomex>=3.11.7', 78 | 'python-gitlab>=0.18,<=1.5.*', 79 | 'GitPython>=3.1.14', 80 | 'paramiko>=2.9.2', 81 | 'cron-descriptor>=1.2.24', 82 | 'python-dateutil>=0.4.5,~=2.8.2', 83 | 'markupsafe>=2.0.1', 84 | 85 | 'Flask-Babel>=2.0.0', 86 | 'flask-sqlalchemy>=2.5.1,~=3.0.3', 87 | 'sqlalchemy~=1.4.46', 88 | 'flask-migrate>=2.6.0,~=4.0.0', 89 | 'pyyaml~=6.0.1', 90 | 'docopt~=0.6.2', 91 | 'celery~=5.2.0', 92 | 'blinker>=1.4.*,~=1.5', 93 | 'Flask-Celery-Tools', 94 | 'sentry-sdk>=1.4.3,~=1.9.10', 95 | 'flask>=2.0.0,~=2.2.2', 96 | 'requests~=2.28.1', 97 | 'Flask-Login>=0.5.*' 98 | ], 99 | test_suite="tests", 100 | tests_require=[ 101 | 'tox' 102 | ], 103 | package_data={'gitlab_tools': extra_files}, 104 | entry_points={ 105 | 'console_scripts': [ 106 | 'gitlab-tools = gitlab_tools.__main__:main', 107 | ], 108 | }, 109 | data_files=[ 110 | (os.path.join(sys_conf_dir, 'systemd', 'system'), [ 111 | 'etc/systemd/system/gitlab-tools.service', 112 | 'etc/systemd/system/gitlab-tools_celerybeat.service', 113 | 'etc/systemd/system/gitlab-tools_celeryworker.service' 114 | ]), 115 | (os.path.join(sys_conf_dir, 'gitlab-tools'), [ 116 | 'etc/gitlab-tools/config.yml' 117 | ]) 118 | ] 119 | ) 120 | -------------------------------------------------------------------------------- /gitlab_tools/static/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "static", 3 | "lockfileVersion": 2, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "license": "GPL-3.0", 8 | "dependencies": { 9 | "bootstrap": "^4.6.2", 10 | "font-awesome": "^4.7.0", 11 | "jquery": "~3.5.1", 12 | "select2": "^4.0.13", 13 | "select2-bootstrap-theme": "0.1.0-beta.10" 14 | } 15 | }, 16 | "node_modules/bootstrap": { 17 | "version": "4.6.2", 18 | "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-4.6.2.tgz", 19 | "integrity": "sha512-51Bbp/Uxr9aTuy6ca/8FbFloBUJZLHwnhTcnjIeRn2suQWsWzcuJhGjKDB5eppVte/8oCdOL3VuwxvZDUggwGQ==", 20 | "funding": [ 21 | { 22 | "type": "github", 23 | "url": "https://github.com/sponsors/twbs" 24 | }, 25 | { 26 | "type": "opencollective", 27 | "url": "https://opencollective.com/bootstrap" 28 | } 29 | ], 30 | "peerDependencies": { 31 | "jquery": "1.9.1 - 3", 32 | "popper.js": "^1.16.1" 33 | } 34 | }, 35 | "node_modules/font-awesome": { 36 | "version": "4.7.0", 37 | "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.7.0.tgz", 38 | "integrity": "sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM=", 39 | "engines": { 40 | "node": ">=0.10.3" 41 | } 42 | }, 43 | "node_modules/jquery": { 44 | "version": "3.5.1", 45 | "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.5.1.tgz", 46 | "integrity": "sha512-XwIBPqcMn57FxfT+Go5pzySnm4KWkT1Tv7gjrpT1srtf8Weynl6R273VJ5GjkRb51IzMp5nbaPjJXMWeju2MKg==" 47 | }, 48 | "node_modules/popper.js": { 49 | "version": "1.16.1", 50 | "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz", 51 | "integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==", 52 | "deprecated": "You can find the new Popper v2 at @popperjs/core, this package is dedicated to the legacy v1", 53 | "peer": true, 54 | "funding": { 55 | "type": "opencollective", 56 | "url": "https://opencollective.com/popperjs" 57 | } 58 | }, 59 | "node_modules/select2": { 60 | "version": "4.0.13", 61 | "resolved": "https://registry.npmjs.org/select2/-/select2-4.0.13.tgz", 62 | "integrity": "sha512-1JeB87s6oN/TDxQQYCvS5EFoQyvV6eYMZZ0AeA4tdFDYWN3BAGZ8npr17UBFddU0lgAt3H0yjX3X6/ekOj1yjw==" 63 | }, 64 | "node_modules/select2-bootstrap-theme": { 65 | "version": "0.1.0-beta.10", 66 | "resolved": "https://registry.npmjs.org/select2-bootstrap-theme/-/select2-bootstrap-theme-0.1.0-beta.10.tgz", 67 | "integrity": "sha1-uUJuz8A79KI152oTI3dXQxBGmsA=" 68 | } 69 | }, 70 | "dependencies": { 71 | "bootstrap": { 72 | "version": "4.6.2", 73 | "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-4.6.2.tgz", 74 | "integrity": "sha512-51Bbp/Uxr9aTuy6ca/8FbFloBUJZLHwnhTcnjIeRn2suQWsWzcuJhGjKDB5eppVte/8oCdOL3VuwxvZDUggwGQ==", 75 | "requires": {} 76 | }, 77 | "font-awesome": { 78 | "version": "4.7.0", 79 | "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.7.0.tgz", 80 | "integrity": "sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM=" 81 | }, 82 | "jquery": { 83 | "version": "3.5.1", 84 | "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.5.1.tgz", 85 | "integrity": "sha512-XwIBPqcMn57FxfT+Go5pzySnm4KWkT1Tv7gjrpT1srtf8Weynl6R273VJ5GjkRb51IzMp5nbaPjJXMWeju2MKg==" 86 | }, 87 | "popper.js": { 88 | "version": "1.16.1", 89 | "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz", 90 | "integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==", 91 | "peer": true 92 | }, 93 | "select2": { 94 | "version": "4.0.13", 95 | "resolved": "https://registry.npmjs.org/select2/-/select2-4.0.13.tgz", 96 | "integrity": "sha512-1JeB87s6oN/TDxQQYCvS5EFoQyvV6eYMZZ0AeA4tdFDYWN3BAGZ8npr17UBFddU0lgAt3H0yjX3X6/ekOj1yjw==" 97 | }, 98 | "select2-bootstrap-theme": { 99 | "version": "0.1.0-beta.10", 100 | "resolved": "https://registry.npmjs.org/select2-bootstrap-theme/-/select2-bootstrap-theme-0.1.0-beta.10.tgz", 101 | "integrity": "sha1-uUJuz8A79KI152oTI3dXQxBGmsA=" 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /gitlab_tools/middleware.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Flask middleware definitions. This is also where template filters are defined. 4 | 5 | To be imported by the application.current_app() factory. 6 | """ 7 | import datetime 8 | from typing import Optional 9 | from logging import getLogger 10 | from celery import states 11 | from cron_descriptor import ExpressionDescriptor 12 | from markupsafe import Markup 13 | from flask import current_app, render_template 14 | from flask_babel import format_datetime, format_date 15 | from gitlab_tools.extensions import login_manager 16 | from gitlab_tools.tools.formaters import format_bytes, fix_url, format_boolean, format_vcs 17 | from gitlab_tools.enums.InvokedByEnum import InvokedByEnum 18 | from gitlab_tools.models.gitlab_tools import User, TaskResult 19 | 20 | LOG = getLogger(__name__) 21 | 22 | 23 | # Setup default error templates. 24 | @current_app.errorhandler(400) 25 | @current_app.errorhandler(403) 26 | @current_app.errorhandler(404) 27 | @current_app.errorhandler(500) 28 | def error_handler(e): 29 | code = getattr(e, 'code', 500) # If 500, e == the exception. 30 | return render_template('{}.html'.format(code)), code 31 | 32 | 33 | @login_manager.user_loader 34 | def load_user(user_id) -> User: 35 | return User.query.get(user_id) 36 | 37 | 38 | @current_app.before_request 39 | def before_request(): 40 | pass 41 | 42 | 43 | @current_app.template_filter('format_bytes') 44 | def format_bytes_filter(num: int) -> str: 45 | return format_bytes(num) 46 | 47 | 48 | @current_app.template_filter('format_datetime') 49 | def format_datetime_filter(date_time: datetime.datetime) -> str: 50 | return format_datetime(date_time) 51 | 52 | 53 | @current_app.template_filter('format_date') 54 | def format_date_filter(date_time: datetime.datetime) -> str: 55 | return format_date(date_time) 56 | 57 | 58 | @current_app.template_filter('fix_url') 59 | def fix_url_filter(url: str) -> str: 60 | return fix_url(url) 61 | 62 | 63 | @current_app.template_filter('format_vcs') 64 | def format_vcs_filter(vcs_enum: int) -> str: 65 | return format_vcs(vcs_enum) 66 | 67 | 68 | @current_app.template_filter('format_boolean') 69 | def format_boolean_filter(bool_to_format: bool) -> Markup: 70 | return Markup(format_boolean(bool_to_format)) 71 | 72 | 73 | @current_app.template_filter('format_cron_syntax') 74 | def format_cron_syntax_filter(cron_syntax: Optional[str]) -> Optional[str]: 75 | if cron_syntax: 76 | expression_descriptor = ExpressionDescriptor(cron_syntax) 77 | return str(expression_descriptor) 78 | 79 | return None 80 | 81 | 82 | @current_app.template_filter('format_task_status_class') 83 | def format_task_status_class_filter(task_result: Optional[TaskResult] = None) -> str: 84 | if not task_result or not task_result.taskmeta: 85 | return 'warning' 86 | return { 87 | states.SUCCESS: 'success', 88 | states.FAILURE: 'danger', 89 | states.REVOKED: 'danger', 90 | states.REJECTED: 'danger', 91 | states.RETRY: 'warning', 92 | states.PENDING: 'info', 93 | states.RECEIVED: 'info', 94 | states.STARTED: 'info', 95 | }.get(task_result.taskmeta.status, 'warning') 96 | 97 | 98 | @current_app.template_filter('format_task_invoked_by') 99 | def format_task_invoked_by_filter(invoked_by: int) -> str: 100 | return { 101 | InvokedByEnum.MANUAL: 'Manual', 102 | InvokedByEnum.HOOK: 'Web hook', 103 | InvokedByEnum.SCHEDULER: 'Scheduler', 104 | InvokedByEnum.UNKNOWN: 'Unknown', 105 | }.get(invoked_by, 'Unknown') 106 | 107 | 108 | # Template filters. 109 | @current_app.template_filter() 110 | def whitelist(value: str) -> Markup: 111 | """Whitelist specific HTML tags and strings. 112 | Positional arguments: 113 | value -- the string to perform the operation on. 114 | Returns: 115 | Markup() instance, indicating the string is safe. 116 | """ 117 | translations = { 118 | '&quot;': '"', 119 | '&#39;': ''', 120 | '&lsquo;': '‘', 121 | '&nbsp;': ' ', 122 | '<br>': '
', 123 | } 124 | escaped = str(Markup.escape(value)) # Escapes everything. 125 | for k, v in translations.items(): 126 | escaped = escaped.replace(k, v) # Un-escape specific elements using str.replace. 127 | return Markup(escaped) # Return as 'safe'. 128 | -------------------------------------------------------------------------------- /gitlab_tools/views/sign/index.py: -------------------------------------------------------------------------------- 1 | 2 | # -*- coding: utf-8 -*- 3 | import urllib.parse 4 | import os 5 | import datetime 6 | import flask 7 | import gitlab 8 | import requests 9 | from flask_login import login_user, logout_user, login_required 10 | 11 | from gitlab_tools.blueprints import sign_index 12 | from gitlab_tools.tasks.gitlab_tools import create_rsa_pair 13 | from gitlab_tools.tools.crypto import random_password 14 | from gitlab_tools.extensions import db 15 | from gitlab_tools.models.gitlab_tools import User, OAuth2State 16 | 17 | 18 | __author__ = "Adam Schubert" 19 | __date__ = "$26.7.2017 19:33:05$" 20 | 21 | 22 | @sign_index.route('/in', methods=['GET']) 23 | def login() -> str: 24 | return flask.render_template('sign.index.login.html') 25 | 26 | 27 | @sign_index.route('/in/request', methods=['GET']) 28 | def request_login() -> flask.Response: 29 | state = random_password() 30 | 31 | new_oauth_state = OAuth2State() 32 | new_oauth_state.state = state 33 | 34 | db.session.add(new_oauth_state) 35 | 36 | redirect_url = '{}?{}'.format( 37 | os.path.join(flask.current_app.config['GITLAB_URL'], 'oauth', 'authorize'), 38 | urllib.parse.urlencode({ 39 | 'client_id': flask.current_app.config['GITLAB_APP_ID'], 40 | 'redirect_uri': flask.url_for('sign_index.do_login', _external=True), 41 | 'response_type': 'code', 42 | 'state': state 43 | }) 44 | ) 45 | db.session.commit() 46 | return flask.redirect(redirect_url) 47 | 48 | 49 | @sign_index.route('/in/do', methods=['GET']) 50 | def do_login(): 51 | state = flask.request.args.get('state') 52 | code = flask.request.args.get('code') 53 | 54 | if not state or not code: 55 | return 'Invalid arguments', 400 56 | 57 | # Lets find out if we issued that state, and if so, delete it to prevent replay 58 | 59 | found_oauth_state = OAuth2State.query.filter_by(state=state).first_or_404() 60 | db.session.delete(found_oauth_state) 61 | db.session.commit() # We need to commit right now 62 | try: 63 | # Lets get token info from API 64 | oauth_url = os.path.join(flask.current_app.config['GITLAB_URL'], 'oauth', 'token') 65 | r = requests.post( 66 | oauth_url, 67 | { 68 | 'client_id': flask.current_app.config['GITLAB_APP_ID'], 69 | 'client_secret': flask.current_app.config['GITLAB_APP_SECRET'], 70 | 'code': code, 71 | 'grant_type': 'authorization_code', 72 | 'redirect_uri': flask.url_for('sign_index.do_login', _external=True) 73 | } 74 | ) 75 | r.raise_for_status() 76 | response_data = r.json() 77 | 78 | # Lets fetch current user info 79 | gl = gitlab.Gitlab( 80 | flask.current_app.config['GITLAB_URL'], 81 | oauth_token=response_data['access_token'], 82 | api_version=flask.current_app.config['GITLAB_API_VERSION'] 83 | ) 84 | 85 | gl.auth() 86 | 87 | logged_user = gl.user 88 | 89 | found_user = User.query.filter_by(gitlab_id=logged_user.id).first() 90 | if not found_user: 91 | found_user = User() 92 | found_user.gitlab_id = logged_user.id 93 | found_user.is_rsa_pair_set = False 94 | found_user.name = logged_user.name 95 | found_user.avatar_url = logged_user.avatar_url 96 | found_user.access_token = response_data['access_token'] 97 | found_user.refresh_token = response_data['refresh_token'] 98 | found_user.created = datetime.datetime.fromtimestamp(response_data['created_at']) 99 | 100 | db.session.add(found_user) 101 | db.session.commit() 102 | 103 | if not found_user.is_rsa_pair_set: 104 | create_rsa_pair.delay(found_user.id) 105 | 106 | login_user(found_user, remember=True) 107 | flask.flash('You has been logged in successfully', 'success') 108 | return flask.redirect(flask.url_for('home_index.get_home')) 109 | except requests.exceptions.HTTPError as e: 110 | db.session.rollback() 111 | flask.flash('Login failed: {}'.format(str(e)), 'danger') 112 | return 'Login failed: {}'.format(str(e)), 500 113 | 114 | 115 | @sign_index.route("/out") 116 | @login_required 117 | def logout(): 118 | logout_user() 119 | return flask.redirect(flask.url_for('sign_index.login')) 120 | -------------------------------------------------------------------------------- /gitlab_tools/views/push_mirror/templates/push_mirror.index.new.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from "_formhelpers.html" import render_field %} 4 |
5 |
6 |
7 |
8 | {{ _("New push mirror") }} 9 |
10 |
11 | {% if form.errors %} 12 |
13 | {{_("There was some errors when proccesing a form.")}} 14 | 15 |
    16 | {% for field, errors in form.errors.items() %} 17 |
  • 18 | {{field}}: 19 |
      20 | {% for error in errors %} 21 |
    • {{error}}
      • 22 | {% endfor %} 23 |
    24 |
    • 25 | {% endfor %} 26 |
27 | 28 |
29 | {% endif %} 30 |
31 |
32 | {% include 'url_info.html' %} 33 |
34 |
35 | 36 | {{form.project_mirror(class_="form-control input-sm check-signature", **{'placeholder': _('Project mirror'), 'data-toggle':"popover", 'data-trigger':"focus ", 'data-placement': "bottom", 'data-fingerprint-url': url_for('fingerprint_index.check_hostname_fingerprint'), 'data-fingerprint-add-url': url_for('fingerprint_index.add_hostname_fingerprint'), 'data-gitlab-url': config['GITLAB_SSH'], 'data-content': '
'.join(form.project_mirror.errors)})}} 37 |
38 |
39 |
40 | {{render_field(form.note, 'form-group', _('Note'))}} 41 |
42 |
43 |
44 | 45 | {{form.project(class_="form-control input-sm project-search", **{'placeholder': _('Project'), 'data-toggle':"popover", 'data-trigger':"focus ", 'data-placement': "bottom", 'data-content': '
'.join(form.project.errors), 'data-source': url_for('api_index.search_project'), 'data-selected-url': url_for('api_index.get_gitlab_project', project_id=form.project.data) if form.project.data})}} 46 |
47 |
48 |
49 |
50 | 55 |
56 |
57 | 62 |
63 | 64 |
65 |
66 | 67 |
68 |
69 | {{ form.csrf_token }} 70 |
71 |
72 |
73 |
74 |
75 | {% endblock %} -------------------------------------------------------------------------------- /gitlab_tools/views/push_mirror/templates/push_mirror.index.edit.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block body %} 3 | {% from "_formhelpers.html" import render_field, render_checkbox %} 4 |
5 |
6 |
7 |
8 | {{ _("Edit mirror of %(project_name_with_namespace)s", project_name_with_namespace=mirror_detail.project.name_with_namespace) }} 9 |
10 |
11 | {% if form.errors %} 12 |
13 | {{_("There was some errors when proccesing a form.")}} 14 | 15 |
    16 | {% for field, errors in form.errors.items() %} 17 |
  • 18 | {{field}}: 19 |
      20 | {% for error in errors %} 21 |
    • {{error}}
      • 22 | {% endfor %} 23 |
    24 |
    • 25 | {% endfor %} 26 |
27 | 28 |
29 | {% endif %} 30 |
31 |
32 | {% include 'url_info.html' %} 33 |
34 |
35 | 36 | {{form.project_mirror(class_="form-control input-sm check-signature", **{'placeholder': _('Project mirror'), 'data-toggle':"popover", 'data-trigger':"focus ", 'data-placement': "bottom", 'data-fingerprint-url': url_for('fingerprint_index.check_hostname_fingerprint'), 'data-fingerprint-add-url': url_for('fingerprint_index.add_hostname_fingerprint'), 'data-gitlab-url': config['GITLAB_SSH'], 'data-content': '
'.join(form.project_mirror.errors)})}} 37 |
38 |
39 |
40 | {{render_field(form.note, 'form-group', _('Note'))}} 41 |
42 |
43 |
44 | 45 | {{form.project(class_="form-control input-sm project-search", **{'placeholder': _('Project'), 'data-toggle':"popover", 'data-trigger':"focus ", 'data-placement': "bottom", 'data-content': '
'.join(form.project.errors), 'data-source': url_for('api_index.search_project'), 'data-selected-url': url_for('api_index.get_gitlab_project', project_id=form.project.data) if form.project.data})}} 46 |
47 |
48 |
49 |
50 | 55 |
56 |
57 | 62 |
63 |
64 |
65 | 66 |
67 |
68 | {{ form.id() }} 69 | {{ form.csrf_token }} 70 |
71 |
72 |
73 |
74 |
75 | {% endblock %} -------------------------------------------------------------------------------- /gitlab_tools/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {{_('GitLab Tools')}} {% block append_title %}{% endblock %} 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 61 | 62 | 63 |
64 | 69 | {% for category, message in get_flashed_messages(with_categories=true) %} 70 |
{{ message |safe}}
71 | {% endfor %} 72 | {% block body %}{% endblock %} 73 |
74 | 75 | 76 | 92 | 93 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /gitlab_tools/migrations/versions/19e8725e0581_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 19e8725e0581 4 | Revises: 56189bfb2c5f 5 | Create Date: 2018-04-07 11:55:43.084495 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | from sqlalchemy.ext.declarative import declarative_base 12 | from sqlalchemy.orm import sessionmaker, relationship 13 | 14 | 15 | # revision identifiers, used by Alembic. 16 | revision = '19e8725e0581' 17 | down_revision = '56189bfb2c5f' 18 | branch_labels = None 19 | depends_on = None 20 | 21 | 22 | Session = sessionmaker() 23 | 24 | Base = declarative_base() 25 | 26 | 27 | class PullMirror(Base): 28 | __tablename__ = 'pull_mirror' 29 | 30 | id = sa.Column(sa.Integer, primary_key=True) 31 | project_name = sa.Column(sa.String(255)) 32 | gitlab_id = sa.Column(sa.Integer) 33 | project_id = sa.Column(sa.Integer, sa.ForeignKey('project.id'), nullable=False) 34 | 35 | project = relationship('Project', backref='pull_mirror') 36 | 37 | 38 | class Project(Base): 39 | __tablename__ = 'project' 40 | 41 | id = sa.Column(sa.Integer, primary_key=True) 42 | name = sa.Column(sa.String, nullable=False, unique=True) 43 | gitlab_id = sa.Column(sa.Integer) 44 | 45 | 46 | def upgrade(): 47 | bind = op.get_bind() 48 | session = Session(bind=bind) 49 | 50 | # ### commands auto generated by Alembic - please adjust! ### 51 | op.create_table('project', 52 | sa.Column('updated', sa.DateTime(), nullable=True), 53 | sa.Column('created', sa.DateTime(), nullable=True), 54 | sa.Column('id', sa.Integer(), nullable=False), 55 | sa.Column('gitlab_id', sa.Integer(), nullable=True), 56 | sa.Column('name', sa.String(length=255), nullable=True), 57 | sa.Column('name_with_namespace', sa.String(length=255), nullable=True), 58 | sa.Column('web_url', sa.String(length=255), nullable=True), 59 | sa.PrimaryKeyConstraint('id'), 60 | sa.UniqueConstraint('gitlab_id') 61 | ) 62 | op.create_table('push_mirror', 63 | sa.Column('updated', sa.DateTime(), nullable=True), 64 | sa.Column('created', sa.DateTime(), nullable=True), 65 | sa.Column('source', sa.String(length=255), nullable=True), 66 | sa.Column('target', sa.String(length=255), nullable=True), 67 | sa.Column('foreign_vcs_type', sa.Integer(), nullable=True), 68 | sa.Column('last_sync', sa.DateTime(), nullable=True), 69 | sa.Column('note', sa.String(length=255), nullable=True), 70 | sa.Column('hook_token', sa.String(length=255), nullable=True), 71 | sa.Column('is_force_update', sa.Boolean(), nullable=True), 72 | sa.Column('is_prune_mirrors', sa.Boolean(), nullable=True), 73 | sa.Column('is_deleted', sa.Boolean(), nullable=True), 74 | sa.Column('id', sa.Integer(), nullable=False), 75 | sa.Column('project_mirror', sa.String(length=255), nullable=True), 76 | sa.Column('user_id', sa.Integer(), nullable=True), 77 | sa.Column('project_id', sa.Integer(), nullable=True), 78 | sa.ForeignKeyConstraint(['project_id'], ['project.id'], ), 79 | sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), 80 | sa.PrimaryKeyConstraint('id') 81 | ) 82 | op.create_index(op.f('ix_push_mirror_project_id'), 'push_mirror', ['project_id'], unique=False) 83 | op.create_index(op.f('ix_push_mirror_user_id'), 'push_mirror', ['user_id'], unique=False) 84 | op.add_column('pull_mirror', sa.Column('project_id', sa.Integer(), nullable=True)) 85 | op.create_index(op.f('ix_pull_mirror_project_id'), 'pull_mirror', ['project_id'], unique=False) 86 | op.create_foreign_key(None, 'pull_mirror', 'project', ['project_id'], ['id']) 87 | 88 | # Migrate dropped gitlab_id to project_id 89 | 90 | for pull_mirror in session.query(PullMirror): 91 | found_project = session.query(Project).filter_by(gitlab_id=pull_mirror.gitlab_id).first() 92 | if not found_project: 93 | found_project = Project() 94 | found_project.gitlab_id = pull_mirror.gitlab_id 95 | found_project.name = pull_mirror.project_name 96 | session.add(found_project) 97 | session.commit() 98 | 99 | pull_mirror.project_id = found_project.id 100 | 101 | session.add(pull_mirror) 102 | session.commit() 103 | 104 | op.drop_column('pull_mirror', 'gitlab_id') 105 | # ### end Alembic commands ### 106 | 107 | 108 | def downgrade(): 109 | bind = op.get_bind() 110 | session = Session(bind=bind) 111 | # ### commands auto generated by Alembic - please adjust! ### 112 | op.add_column('pull_mirror', sa.Column('gitlab_id', sa.INTEGER(), nullable=True)) 113 | 114 | for pull_mirror in session.query(PullMirror): 115 | pull_mirror.gitlab_id = pull_mirror.project.gitlab_id 116 | session.add(pull_mirror) 117 | session.commit() 118 | 119 | op.drop_constraint(None, 'pull_mirror', type_='foreignkey') 120 | op.drop_index(op.f('ix_pull_mirror_project_id'), table_name='pull_mirror') 121 | op.drop_column('pull_mirror', 'project_id') 122 | op.drop_index(op.f('ix_push_mirror_user_id'), table_name='push_mirror') 123 | op.drop_index(op.f('ix_push_mirror_project_id'), table_name='push_mirror') 124 | op.drop_table('push_mirror') 125 | op.drop_table('project') 126 | # ### end Alembic commands ### 127 | -------------------------------------------------------------------------------- /gitlab_tools/tools/GitSubprocess.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess # nosec: B404 3 | import logging 4 | from typing import Optional 5 | from gitlab_tools.enums.VcsEnum import VcsEnum 6 | from gitlab_tools.tools.GitRemote import GitRemote 7 | 8 | 9 | class GitSubprocess: 10 | 11 | @staticmethod 12 | def sync_mirror(namespace_path: str, temp_name: str, source: GitRemote, target: Optional[GitRemote] = None) -> None: 13 | 14 | # Check if repository storage group directory exists: 15 | if not os.path.isdir(namespace_path): 16 | raise Exception('Group storage {} not found, creation failed ?'.format(namespace_path)) 17 | 18 | # Check if project clone exists 19 | project_path = os.path.join(namespace_path, temp_name) 20 | if not os.path.isdir(project_path): 21 | raise Exception('Repository storage {} not found, creation failed ?'.format(project_path)) 22 | 23 | # Special code for SVN repo mirror 24 | if source.vcs_type == VcsEnum.SVN: 25 | subprocess.Popen(['git', 'reset', 'hard'], cwd=project_path).communicate() # nosec: B607, B603 26 | subprocess.Popen(['git', 'svn', 'fetch'], cwd=project_path).communicate() # nosec: B607, B603 27 | subprocess.Popen(['git', 'svn', 'rebase'], cwd=project_path).communicate() # nosec: B607, B603 28 | 29 | if not target: 30 | # repo.git.config('--bool', 'core.bare', 'true') 31 | subprocess.Popen(['git', 'push', 'gitlab'], cwd=project_path).communicate() # nosec: B607, B603 32 | # repo.git.config('--bool', 'core.bare', 'false') 33 | 34 | else: 35 | # Everything else 36 | fetch_command = ['git', 'fetch'] 37 | if source.is_force_update: 38 | fetch_command.append('--force') 39 | if source.is_prune_mirrors: 40 | fetch_command.append('--prune') 41 | 42 | #fetch_command.append('origin') 43 | 44 | push_command = ['git', 'push'] 45 | if target: 46 | if target.is_force_update: 47 | push_command.append('--force') 48 | if target.is_prune_mirrors: 49 | push_command.append('--prune') 50 | 51 | push_command.append('gitlab') 52 | #push_command.append('+refs/heads/*:refs/heads/*') 53 | #push_command.append('+refs/tags/*:refs/tags/*') 54 | 55 | subprocess.Popen(fetch_command, cwd=project_path).communicate() # nosec: B607, B603 56 | subprocess.Popen(push_command, cwd=project_path).communicate() # nosec: B607, B603 57 | 58 | logging.info('Mirror sync done') 59 | 60 | @staticmethod 61 | def create_mirror(namespace_path: str, temp_name: str, source: GitRemote, target: Optional[GitRemote] = None) -> None: 62 | 63 | # 2. Create/pull local repository 64 | 65 | # Check if project clone exists 66 | project_path = os.path.join(namespace_path, temp_name) 67 | if os.path.isdir(project_path): 68 | subprocess.Popen( # nosec: B607, B603 69 | ['git', 'remote', 'set-url', 'origin', source.url] 70 | , cwd=project_path 71 | ).communicate() 72 | if target: 73 | subprocess.Popen( # nosec: B607, B603 74 | ['git', 'remote', 'set-url', 'gitlab', target.url], 75 | cwd=project_path 76 | ).communicate() 77 | else: 78 | # Project not found, we can clone 79 | logging.info('Creating mirror for %s', source.url) 80 | 81 | # 3. Pull 82 | # 4. Push 83 | 84 | if source.vcs_type == VcsEnum.SVN: 85 | subprocess.Popen( # nosec: B607, B603 86 | ['git', 'svn', 'clone', source.url, project_path], 87 | cwd=namespace_path 88 | ).communicate() 89 | else: 90 | subprocess.Popen( # nosec: B607, B603 91 | ['git', 'clone', '--mirror', source.url, project_path] 92 | ).communicate() 93 | 94 | if source.vcs_type in [VcsEnum.BAZAAR, VcsEnum.MERCURIAL]: 95 | subprocess.Popen(['git', 'gc', '--aggressive'], cwd=project_path).communicate() # nosec: B607, B603 96 | 97 | if target: 98 | logging.info('Adding GitLab remote to project.') 99 | subprocess.Popen( # nosec: B607, B603 100 | ['git', 'remote', 'add', 'gitlab', target.url], 101 | cwd=project_path 102 | ).communicate() 103 | 104 | subprocess.Popen( # nosec: B607, B603 105 | ['git', 'config', '--add', 'remote.gitlab.push', '+refs/heads/*:refs/heads/*'], 106 | cwd=project_path 107 | ).communicate() 108 | 109 | subprocess.Popen( # nosec: B607, B603 110 | ['git', 'config', '--add', 'remote.gitlab.push', '+refs/tags/*:refs/tags/*'], 111 | cwd=project_path 112 | ).communicate() 113 | 114 | subprocess.Popen( # nosec: B607, B603 115 | ['git', 'config', 'remote.gitlab.mirror', 'true'], 116 | cwd=project_path 117 | ).communicate() 118 | 119 | GitSubprocess.sync_mirror(namespace_path, temp_name, source, target) 120 | 121 | logging.info('All done!') 122 | -------------------------------------------------------------------------------- /gitlab_tools/application.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sentry_sdk 5 | from importlib import import_module 6 | from yaml import load, SafeLoader 7 | from flask import Flask, url_for, request, json, g 8 | from flask_babel import gettext 9 | from gitlab_tools.blueprints import all_blueprints 10 | from gitlab_tools.config import Config 11 | 12 | import gitlab_tools as app_root 13 | from gitlab_tools.tools import jsonifier 14 | from gitlab_tools.extensions import db, babel, login_manager, migrate, celery 15 | 16 | APP_ROOT_FOLDER = os.path.abspath(os.path.dirname(app_root.__file__)) 17 | TEMPLATE_FOLDER = os.path.join(APP_ROOT_FOLDER, 'templates') 18 | STATIC_FOLDER = os.path.join(APP_ROOT_FOLDER, 'static') 19 | REDIS_SCRIPTS_FOLDER = os.path.join(APP_ROOT_FOLDER, 'redis_scripts') 20 | 21 | 22 | def get_config(config_class_string: str, yaml_files: list=None) -> Config: 23 | """Load the Flask config from a class. 24 | Positional arguments: 25 | config_class_string -- string representation of a configuration class that will be loaded (e.g. 26 | 'gitlab-tools.config.Production'). 27 | yaml_files -- List of YAML files to load. This is for testing, leave None in dev/production. 28 | Returns: 29 | A class object to be fed into app.config.from_object(). 30 | """ 31 | config_module, config_class = config_class_string.rsplit('.', 1) 32 | config_obj = getattr(import_module(config_module), config_class)() 33 | 34 | # Expand some options. 35 | db_fmt = 'gitlab_tools.models.{}' 36 | celery_fmt = 'gitlab_tools.tasks.{}' 37 | if getattr(config_obj, 'CELERY_IMPORTS', False): 38 | config_obj.CELERY_IMPORTS = [celery_fmt.format(m) for m in config_obj.CELERY_IMPORTS] 39 | for definition in getattr(config_obj, 'CELERYBEAT_SCHEDULE', dict()).values(): 40 | definition.update(task=celery_fmt.format(definition['task'])) 41 | if getattr(config_obj, 'DB_MODELS_IMPORTS', False): 42 | config_obj.DB_MODELS_IMPORTS = [db_fmt.format(m) for m in config_obj.DB_MODELS_IMPORTS] 43 | 44 | # Load additional configuration settings. 45 | yaml_files = yaml_files or [f for f in [ 46 | os.path.join('/', 'etc', 'gitlab-tools', 'config.yml'), 47 | os.path.abspath(os.path.join(APP_ROOT_FOLDER, '..', 'config.yml')), 48 | os.path.join(APP_ROOT_FOLDER, 'config.yml'), 49 | ] if os.path.exists(f)] 50 | additional_dict = dict() 51 | for y in yaml_files: 52 | with open(y) as f: 53 | loaded_data = load(f.read(), Loader=SafeLoader) 54 | if isinstance(loaded_data, dict): 55 | additional_dict.update(loaded_data) 56 | else: 57 | raise Exception('Failed to parse configuration {}'.format(y)) 58 | 59 | # Merge the rest into the Flask app config. 60 | for key, value in additional_dict.items(): 61 | setattr(config_obj, key, value) 62 | 63 | return config_obj 64 | 65 | 66 | def create_app(config_obj: Config, no_sql: bool = False) -> Flask: 67 | 68 | """Create an application.""" 69 | app = Flask(__name__, template_folder=TEMPLATE_FOLDER, static_folder=STATIC_FOLDER) 70 | config_dict = {k: getattr(config_obj, k) for k in dir(config_obj) if not k.startswith('_')} 71 | app.config.update(config_dict) 72 | 73 | # Import DB models. Flask-SQLAlchemy doesn't do this automatically like Celery does. 74 | with app.app_context(): 75 | for model in app.config.get('DB_MODELS_IMPORTS', list()): 76 | import_module(model) 77 | 78 | for bp in all_blueprints: 79 | import_module(bp.import_name) 80 | app.register_blueprint(bp) 81 | 82 | class FlaskJSONProvider(json.provider.DefaultJSONProvider): 83 | """Custom JSONProvider which adds connexion defaults on top of Flask's""" 84 | 85 | @jsonifier.wrap_default 86 | def default(self, o): 87 | return super().default(o) 88 | 89 | app.json = FlaskJSONProvider(app) 90 | 91 | def url_for_other_page(page: int) -> str: 92 | args = request.view_args.copy() 93 | args['page'] = page 94 | return url_for(request.endpoint, **args) 95 | 96 | app.jinja_env.globals['url_for_other_page'] = url_for_other_page # pylint: disable=no-member 97 | 98 | if not no_sql: 99 | db.init_app(app) 100 | 101 | migrate.init_app(app, db) 102 | 103 | def get_locale() -> str: 104 | # if a user is logged in, use the locale from the user settings 105 | user = getattr(g, 'user', None) 106 | if user is not None: 107 | return user.locale 108 | # otherwise try to guess the language from the user accept 109 | # header the browser transmits. We support de/fr/en in this 110 | # example. The best match wins. 111 | if request: 112 | return request.accept_languages.best_match(['cs', 'en']) 113 | else: 114 | return app.config.get('BABEL_DEFAULT_LOCALE', 'cs') 115 | 116 | if hasattr(babel, "localeselector"): 117 | babel.init_app(app) 118 | babel.localeselector(get_locale) 119 | else: 120 | babel.init_app(app, locale_selector=get_locale) 121 | 122 | sentry_sdk.init( 123 | dsn=app.config.get('SENTRY_CONFIG', {}).get('dsn') 124 | ) 125 | 126 | celery.init_app(app) 127 | 128 | login_manager.init_app(app) 129 | login_manager.login_view = "sign_index.login" 130 | login_manager.login_message_category = "info" 131 | login_manager.localize_callback = gettext 132 | 133 | login_manager.refresh_view = "sign_index.login" 134 | login_manager.needs_refresh_message = ( 135 | u"To protect your account, please reauthenticate to access this page." 136 | ) 137 | login_manager.needs_refresh_message_category = "info" 138 | 139 | with app.app_context(): 140 | import_module('gitlab_tools.middleware') 141 | 142 | return app 143 | -------------------------------------------------------------------------------- /gitlab_tools/views/api/index.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from typing import Tuple 3 | import flask 4 | from flask import jsonify, request, url_for 5 | from flask_login import login_required 6 | from gitlab_tools.blueprints import api_index 7 | from gitlab_tools.tasks.gitlab_tools import sync_pull_mirror, sync_push_mirror 8 | from gitlab_tools.models.gitlab_tools import PullMirror, PushMirror 9 | from gitlab_tools.models.celery import TaskMeta 10 | from gitlab_tools.enums.InvokedByEnum import InvokedByEnum 11 | from gitlab_tools.tools.gitlab import get_group, get_project, get_gitlab_instance 12 | from gitlab_tools.tools.celery import log_task_pending 13 | 14 | 15 | __author__ = "Adam Schubert" 16 | __date__ = "$26.7.2017 19:33:05$" 17 | 18 | 19 | @api_index.route('/pull/sync/', methods=['POST', 'GET']) 20 | def schedule_sync_pull_mirror(mirror_id: int) -> Tuple[flask.Response, int]: 21 | hook_token = request.args.get('token') 22 | if not hook_token: 23 | return jsonify({'message': 'Token was not provided'}), 400 24 | 25 | found_mirror = PullMirror.query.filter_by(id=mirror_id).first() 26 | if not found_mirror: 27 | return jsonify({'message': 'Mirror not found'}), 404 28 | 29 | if found_mirror.hook_token != hook_token: 30 | return jsonify({'message': 'Supplied token was incorrect'}), 400 31 | 32 | if not found_mirror.project_id: 33 | return jsonify({'message': 'Project mirror is not created, cannot be synced'}), 400 34 | 35 | task = sync_pull_mirror.delay(found_mirror.id) 36 | log_task_pending(task, found_mirror, sync_pull_mirror, InvokedByEnum.HOOK) 37 | 38 | return jsonify({'message': 'Sync task started', 'uuid': task.id}), 200 39 | 40 | 41 | @api_index.route('/push/sync/', methods=['POST', 'GET']) 42 | def schedule_sync_push_mirror(mirror_id: int) -> Tuple[flask.Response, int]: 43 | hook_token = request.args.get('token') 44 | if not hook_token: 45 | return jsonify({'message': 'Token was not provided'}), 400 46 | 47 | found_mirror = PushMirror.query.filter_by(id=mirror_id).first() 48 | if not found_mirror: 49 | return jsonify({'message': 'Mirror not found'}), 404 50 | 51 | if found_mirror.hook_token != hook_token: 52 | return jsonify({'message': 'Supplied token was incorrect'}), 400 53 | 54 | if not found_mirror.project_id: 55 | return jsonify({'message': 'Project mirror is not created, cannot be synced'}), 400 56 | 57 | task = sync_push_mirror.delay(found_mirror.id) 58 | log_task_pending(task, found_mirror, sync_push_mirror, InvokedByEnum.HOOK) 59 | 60 | return jsonify({'message': 'Sync task started', 'uuid': task.id}), 200 61 | 62 | 63 | def group_fix_avatar(group: dict) -> dict: 64 | if not group['avatar_url']: 65 | group['avatar_url'] = url_for('static', filename='img/no_group_avatar.png', _external=True) 66 | return group 67 | 68 | 69 | @api_index.route('/groups/search', methods=['GET']) 70 | @login_required 71 | def search_group() -> Tuple[flask.Response, int]: 72 | q = request.args.get('q') 73 | per_page = request.args.get('per_page') 74 | page = request.args.get('page') 75 | if not q: 76 | return jsonify({'message': 'q was not provided'}), 400 77 | 78 | gl = get_gitlab_instance() 79 | 80 | params = { 81 | 'search': q, 82 | 'as_list': False 83 | } 84 | 85 | if per_page: 86 | params['per_page'] = per_page 87 | 88 | if page: 89 | params['page'] = page 90 | 91 | items_list = gl.groups.list(**params) 92 | 93 | return jsonify({ 94 | 'items': [group_fix_avatar(i.attributes) for i in items_list], 95 | 'current_page': items_list.current_page, 96 | 'prev_page': items_list.prev_page, 97 | 'next_page': items_list.next_page, 98 | 'per_page': items_list.per_page, 99 | 'total_pages': items_list.total_pages, 100 | 'total': items_list.total, 101 | }), 200 102 | 103 | 104 | @api_index.route('/groups/', methods=['GET']) 105 | @login_required 106 | def get_gitlab_group(group_id: int) -> Tuple[flask.Response, int]: 107 | group = get_group(group_id) 108 | 109 | return jsonify(group_fix_avatar(group.attributes)), 200 110 | 111 | 112 | @api_index.route('/projects/search', methods=['GET']) 113 | @login_required 114 | def search_project() -> Tuple[flask.Response, int]: 115 | q = request.args.get('q') 116 | per_page = request.args.get('per_page') 117 | page = request.args.get('page') 118 | if not q: 119 | return jsonify({'message': 'q was not provided'}), 400 120 | 121 | gl = get_gitlab_instance() 122 | 123 | params = { 124 | 'search': q, 125 | 'as_list': False 126 | } 127 | 128 | if per_page: 129 | params['per_page'] = per_page 130 | 131 | if page: 132 | params['page'] = page 133 | 134 | items_list = gl.projects.list(**params) 135 | 136 | return jsonify({ 137 | 'items': [i.attributes for i in items_list], 138 | 'current_page': items_list.current_page, 139 | 'prev_page': items_list.prev_page, 140 | 'next_page': items_list.next_page, 141 | 'per_page': items_list.per_page, 142 | 'total_pages': items_list.total_pages, 143 | 'total': items_list.total, 144 | }), 200 145 | 146 | 147 | @api_index.route('/projects/', methods=['GET']) 148 | @login_required 149 | def get_gitlab_project(project_id: int) -> Tuple[flask.Response, int]: 150 | project = get_project(project_id) 151 | 152 | return jsonify(project.attributes), 200 153 | 154 | 155 | @api_index.route('/task//traceback', methods=['GET']) 156 | @login_required 157 | def get_task_traceback(task_id: str) -> Tuple[flask.Response, int]: 158 | task_meta = TaskMeta.query.filter_by(task_id=task_id).first_or_404() 159 | return jsonify({'traceback': task_meta.traceback}), 200 160 | -------------------------------------------------------------------------------- /gitlab_tools/tools/helpers.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pwd 3 | import grp 4 | import errno 5 | import paramiko 6 | from gitlab_tools.models.gitlab_tools import User, PullMirror, Mirror 7 | from gitlab_tools.tools.GitUri import GitUri 8 | 9 | 10 | def get_home_dir(user_name: str) -> str: 11 | """ 12 | Get home dir by user name 13 | :param user_name: User name 14 | :return: Path to homedir 15 | """ 16 | return os.path.expanduser('~{}'.format(user_name)) 17 | 18 | 19 | def get_namespace_path(mirror: Mirror, user_name: str) -> str: 20 | """ 21 | Returns path to repository group 22 | :param mirror: Mirror 23 | :param user_name: system username 24 | :return: str 25 | """ 26 | repository_storage_path = get_repository_storage(user_name) 27 | 28 | if isinstance(mirror, PullMirror): 29 | return os.path.join(repository_storage_path, str(mirror.user.id), 'pull', str(mirror.group.id)) 30 | 31 | return os.path.join(repository_storage_path, str(mirror.user.id), 'push', str(mirror.id)) 32 | 33 | 34 | def get_repository_path(namespace: str, mirror: Mirror) -> str: 35 | """ 36 | Get repository path 37 | :param namespace: namespace path 38 | :param mirror: Mirror 39 | :return: str 40 | """ 41 | # Check if project clone exists 42 | return os.path.join(namespace, str(mirror.id)) 43 | 44 | 45 | def get_ssh_storage(user_name: str) -> str: 46 | """ 47 | Gets user default ssh storage 48 | :param user_name: User name 49 | :return: path to ssh storage 50 | """ 51 | return os.path.join(get_home_dir(user_name), '.ssh') 52 | 53 | 54 | def get_repository_storage(user_name: str) -> str: 55 | """ 56 | Gets user repository storage 57 | :param user_name: user name 58 | :return: path to repository storage 59 | """ 60 | return os.path.join(get_home_dir(user_name), 'repositories') 61 | 62 | 63 | def get_user_group_id(user_name: str) -> int: 64 | """ 65 | Returns Default user group id 66 | :param user_name: User name 67 | :return: group id 68 | """ 69 | return pwd.getpwnam(user_name).pw_gid 70 | 71 | 72 | def get_group_name(group_id: int) -> str: 73 | """ 74 | Returns group name by ID 75 | :param group_id: Group id 76 | :return: Group name 77 | """ 78 | return grp.getgrgid(group_id).gr_name 79 | 80 | 81 | def get_user_id(user_name: str) -> int: 82 | """ 83 | Returns user ID 84 | :param user_name: User name 85 | :return: User ID 86 | """ 87 | return pwd.getpwnam(user_name).pw_uid 88 | 89 | 90 | def get_user_public_key_path(user: User, user_name: str) -> str: 91 | """ 92 | Returns path for user public key 93 | :param user: gitlab tools user 94 | :param user_name: system user name 95 | :return: str 96 | """ 97 | ssh_storage = get_ssh_storage(user_name) 98 | return os.path.join(ssh_storage, 'id_rsa_{}.pub'.format(user.id)) 99 | 100 | 101 | def get_user_private_key_path(user: User, user_name: str) -> str: 102 | """ 103 | Returns path for user private key 104 | :param user: gitlab tools user 105 | :param user_name: system user name 106 | :return: str 107 | """ 108 | ssh_storage = get_ssh_storage(user_name) 109 | return os.path.join(ssh_storage, 'id_rsa_{}'.format(user.id)) 110 | 111 | 112 | def get_user_known_hosts_path(user: User, user_name: str) -> str: 113 | """ 114 | Returns path for user known_hosts 115 | :param user: gitlab tools user 116 | :param user_name: system user name 117 | :return: str 118 | """ 119 | ssh_storage = get_ssh_storage(user_name) 120 | return os.path.join(ssh_storage, 'known_hosts_{}'.format(user.id)) 121 | 122 | 123 | def get_known_hosts_path(user_name: str) -> str: 124 | """ 125 | Returns path for user known_hosts 126 | :param user_name: system user name 127 | :return: str 128 | """ 129 | ssh_storage = get_ssh_storage(user_name) 130 | return os.path.join(ssh_storage, 'known_hosts') 131 | 132 | 133 | def get_ssh_config_path(user_name: str) -> str: 134 | """ 135 | Returns path to ssh config 136 | :param user_name: str 137 | :return: str 138 | """ 139 | ssh_storage = get_ssh_storage(user_name) 140 | return os.path.join(ssh_storage, 'config') 141 | 142 | 143 | def convert_url_for_user(url: str, user: User) -> str: 144 | """ 145 | Converts url hostname of url to user identified type for SSH config 146 | :param url: Url to convert 147 | :param user: User to use 148 | :return: Returns modified URL 149 | """ 150 | git_remote = GitUri(url) 151 | 152 | return url.replace(git_remote.hostname, '{}_{}'.format(git_remote.hostname, user.id), 1) 153 | 154 | 155 | def mkdir_p(path: str) -> None: 156 | """ 157 | Create path recursive 158 | :param path: Path to create 159 | :return: None 160 | """ 161 | try: 162 | os.makedirs(path) 163 | except OSError as exc: # Python >2.5 164 | if exc.errno == errno.EEXIST and os.path.isdir(path): 165 | pass 166 | else: 167 | raise 168 | 169 | 170 | def add_ssh_config(user: User, user_name: str, identifier: str, host_info: GitUri) -> None: 171 | """ 172 | Adds new SSH config host 173 | :param user: User 174 | :param user_name: str 175 | :param identifier: str 176 | :param host_info: GitUri 177 | :return: None 178 | """ 179 | ssh_config_path = get_ssh_config_path(user_name) 180 | user_known_hosts_path = get_user_known_hosts_path(user, user_name) 181 | user_private_key_path = get_user_private_key_path(user, user_name) 182 | 183 | ssh_config = paramiko.config.SSHConfig() 184 | if os.path.isfile(ssh_config_path): 185 | with open(ssh_config_path, 'r') as f: 186 | ssh_config.parse(f) 187 | if identifier not in ssh_config.get_hostnames(): 188 | rows = [ 189 | "Host {}".format(identifier), 190 | " HostName {}".format(host_info.hostname), 191 | " Port {}".format(host_info.port), 192 | " UserKnownHostsFile {}".format(user_known_hosts_path), 193 | " IdentitiesOnly yes", 194 | " IdentityFile {}".format(user_private_key_path), 195 | "" 196 | ] 197 | 198 | with open(ssh_config_path, 'a') as f: 199 | f.write('\n'.join(rows)) 200 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitLab Tools 2 | 3 | This application provides functionality not avaiable in GitLab (CE) Mainly useful for pull mirroring from GitHub 4 | 5 | > Please consider sponsoring if you're using this package commercially, my time is not free :) You can sponsor me by clicking on "Sponsor" button in top button row. Thank You. 6 | 7 | 8 | [![Home](doc/img/home_thumb.png)](doc/img/home.png) 9 | 10 | - [Fingerprints screenshoot](doc/img/fingerprints.png) 11 | - [Pull mirrors screenshoot](doc/img/pull_mirrors.png) 12 | 13 | # Avaiable features 14 | 15 | | Feature name | Gitlab CE | Gitlab EE | Gitlab Tools | Description | 16 | | :-- | :-: | :-: | :-: | :-: | 17 | | Pull mirror [(doc)](https://github.com/Salamek/gitlab-tools/wiki/1.-Pull-mirror-configuration-guide) | No | Yes | Yes | Allows to automaticaly mirror your Git or SVN repositories to GitLab by hook trigger or periodicaly | 18 | | Push mirror [(doc)](https://github.com/Salamek/gitlab-tools/wiki/2.-Push-mirror-configuration-guide) | Yes(10.8) | Yes | Yes | Allows to automaticaly mirror your GitLab repository to any remote Git repository | 19 | 20 | # Installation 21 | 22 | ## Debian and derivates (Debian 11 and Ubuntu 20.04.4 LTS are supported, other versions may also work...) 23 | 24 | Add repository by running these commands 25 | 26 | ```bash 27 | $ wget -O - https://repository.salamek.cz/deb/salamek.gpg.key|sudo apt-key add - 28 | $ echo "deb https://repository.salamek.cz/deb/pub all main" | sudo tee /etc/apt/sources.list.d/salamek.cz.list 29 | ``` 30 | 31 | And then you can install a package gitlab-tools 32 | 33 | ```bash 34 | $ apt update && apt install gitlab-tools 35 | ``` 36 | 37 | ## Archlinux 38 | 39 | Add repository by adding this at end of file /etc/pacman.conf 40 | 41 | ``` 42 | [salamek] 43 | Server = https://repository.salamek.cz/arch/pub 44 | SigLevel = Optional 45 | ``` 46 | 47 | and then install by running 48 | 49 | ```bash 50 | $ pacman -Sy gitlab-tools 51 | ``` 52 | 53 | ## Docker 54 | 55 | You can create your docker-compose stack to run this application on docker. You can visit [cenk1cenk2/docker-gitlab-tools](https://github.com/cenk1cenk2/docker-gitlab-tools) for more information on the container. 56 | 57 | # Setup 58 | 59 | After successful installation you will need to run setup to configure your installation: 60 | 61 | ```bash 62 | $ gitlab-tools setup 63 | ``` 64 | 65 | This will start simple setup utility where you can/must configure 66 | 67 | 1. gitlab-tools user (default: gitlab-tools, should be created after install) 68 | 2. Database type to use, PostgreSQL is recommended database to use (you can use other database types only on your own risk) 69 | 1. Database host (PostgreSQL, MySQL) or path (Only SQLite) 70 | 2. Database name (PostgreSQL, MySQL) 71 | 3. Database user (PostgreSQL, MySQL) 72 | 4. Database password (PostgreSQL, MySQL) 73 | 3. Webserver configuration (Host and port makes sense only when using integrated web server controlled by gitlab-tools service) 74 | 4. Server name (eg.: https://gitlab-tools.example.com or IP:PORT when using integrated webserver) 75 | 5. GitLab API configuration (you can follow this guide till point 7. https://docs.gitlab.com/ee/integration/gitlab.html) redirect url is https://gitlab-tools.example.com/sign/in/do where example.com is your server domain 76 | 1. Gitlab APP ID: Application Id 77 | 2. Gitlab APP SECRET: Secret 78 | 3. Gitlab SSH: hostname and port where is your gitlab SSH port exposed in format gitlab.com:22 79 | 6. Save new configuration ? -> y (this will create config file in /etc/gitlab-tools/config.yml) 80 | 7. Recreate database ? -> y (this will create new empty database) 81 | 8. Restart services to load new configuration ? -> y this will restart all gitlab-tools services: 82 | 1. gitlab-tools: Controlling integrated webserver, disable this one if you want to use uwsgi or so. 83 | 2. gitlab-tools_celeryworker: Controlling backround workers, must be enabled and running to perform mirroring 84 | 3. gitlab-tools_celerybeat: Controlling celery scheduler 85 | 86 | This creates/updates config file in /etc/gitlab-tools/config.yml, you can modify this file manualy 87 | 88 | After this you should have gitlab-tools running on your "server name" 89 | 90 | # Running behind UWSGI 91 | 92 | Install uwsgi 93 | 94 | ``` 95 | $ apt install uwsgi uwsgi-plugin-python3 96 | ``` 97 | 98 | Create uwsgi application configuration file at `/etc/uwsgi/apps-available/gitlab-tools.example.com.ini`: 99 | 100 | ```ini 101 | [uwsgi] 102 | uid = gitlab-tools 103 | master = true 104 | chdir = /usr/lib/python3/dist-packages/gitlab_tools 105 | socket = /tmp/gitlab-tools.sock 106 | module = wsgi 107 | callable = app 108 | plugins = python3 109 | buffer-size = 32768 110 | ``` 111 | 112 | Link this config file to `/etc/uwsgi/apps-enabled` by running 113 | 114 | ```bash 115 | $ ln -s /etc/uwsgi/apps-available/gitlab-tools.example.com.ini /etc/uwsgi/apps-enabled/ 116 | ``` 117 | 118 | Restart uwsgi to load new configuration 119 | 120 | ```bash 121 | $ systemctl restart uwsgi 122 | ``` 123 | 124 | Now you should have `/tmp/gitlab-tools.sock` socket file created, check that by running 125 | 126 | ```bash 127 | $ file /tmp/gitlab-tools.sock 128 | ``` 129 | 130 | # Webserver NGINX 131 | 132 | Install nginx by running 133 | 134 | ``` 135 | apt install nginx 136 | ``` 137 | 138 | Create configuration file at `/etc/nginx/sites-available/gitlab-tools.example.com`: 139 | 140 | ``` 141 | # Uncomment this when using SSL 142 | #server { 143 | # listen 80; 144 | # listen [::]:80; 145 | # server_name gitlab-tools.example.com; 146 | # return 301 https://gitlab-tools.example.com$request_uri; 147 | #} 148 | 149 | server { 150 | # Uncomment these to use SSL 151 | #listen 443 ssl http2; 152 | #listen [::]:443 ssl http2; 153 | # Comment these to use SSL 154 | listen 80; 155 | listen [::]:80; 156 | server_name gitlab-tools.example.com; 157 | 158 | root /usr/lib/python3/dist-packages/gitlab_tools; 159 | 160 | # Uncomment these to use SSL 161 | #ssl on; 162 | #ssl_certificate letsencrypt/certs/example.com/fullchain.pem; 163 | #ssl_certificate_key letsencrypt/certs/example.com/privkey.pem; 164 | 165 | location '/.well-known/acme-challenge' { 166 | default_type "text/plain"; 167 | root /var/tmp/letsencrypt-auto; 168 | } 169 | 170 | location / { 171 | uwsgi_pass unix:///tmp/gitlab-tools.sock; 172 | include uwsgi_params; 173 | } 174 | } 175 | 176 | ``` 177 | 178 | Link this file to `/etc/nginx/sites-enabled` by running 179 | 180 | ```bash 181 | $ ln -s /etc/nginx/sites-available/gitlab-tools.example.com /etc/nginx/sites-enabled/ 182 | ``` 183 | 184 | And restart nginx 185 | 186 | ```bash 187 | $ systemctl restart nginx 188 | ``` 189 | 190 | Now you should have gitlab-tools accessible at `server_name` 191 | 192 | # Mirrors 193 | 194 | This project is also mirrored on GitLab https://gitlab.com/Salamek/gitlab-tools 195 | -------------------------------------------------------------------------------- /tests/test_git_uri_parser.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from gitlab_tools.tools.GitUri import GitUri 3 | 4 | 5 | def test_be_constructed_with() -> None: 6 | result = GitUri('ssh://git@github.com:22/asdf/asdf.git') 7 | assert result.scheme == 'ssh' 8 | assert result.username == 'git' 9 | assert result.password == None 10 | assert result.hostname == 'github.com' 11 | assert result.port == 22 12 | assert result.path == '/asdf/asdf.git' 13 | assert result.url == 'ssh://git@github.com:22/asdf/asdf.git' 14 | assert result.build_url(ignore_default_port=True) == 'ssh://git@github.com:/asdf/asdf.git' 15 | 16 | 17 | 18 | def test_git_uri_with_scheme_user_port_path() -> None: 19 | result = GitUri('ssh://git@github.com:22/asdf/asdf.git') 20 | assert result.scheme == 'ssh' 21 | assert result.username == 'git' 22 | assert result.password == None 23 | assert result.hostname == 'github.com' 24 | assert result.port == 22 25 | assert result.path == '/asdf/asdf.git' 26 | assert result.url == 'ssh://git@github.com:22/asdf/asdf.git' 27 | assert result.build_url(ignore_default_port=True) == 'ssh://git@github.com:/asdf/asdf.git' 28 | 29 | 30 | def test_git_uri_with_scheme_user_password_port_path() -> None: 31 | result = GitUri('ssh://git:password@github.com:22/asdf/asdf.git') 32 | assert result.scheme == 'ssh' 33 | assert result.username == 'git' 34 | assert result.password == 'password' 35 | assert result.hostname == 'github.com' 36 | assert result.port == 22 37 | assert result.path == '/asdf/asdf.git' 38 | assert result.url == 'ssh://git:password@github.com:22/asdf/asdf.git' 39 | assert result.build_url(ignore_default_port=True) == 'ssh://git:password@github.com:/asdf/asdf.git' 40 | 41 | 42 | def test_git_uri_with_scheme_user_path_with_leading_slash() -> None: 43 | result = GitUri('ssh://git@github.com:/asdf/asdf.git') 44 | assert result.scheme == 'ssh' 45 | assert result.username == 'git' 46 | assert result.password == None 47 | assert result.hostname == 'github.com' 48 | assert result.port == 22 49 | assert result.path == '/asdf/asdf.git' 50 | assert result.url == 'ssh://git@github.com:22/asdf/asdf.git' 51 | assert result.build_url(ignore_default_port=True) == 'ssh://git@github.com:/asdf/asdf.git' 52 | 53 | 54 | def test_git_uri_with_scheme_user_path() -> None: 55 | result = GitUri('ssh://git@github.com:asdf/asdf.git') 56 | assert result.scheme == 'ssh' 57 | assert result.username == 'git' 58 | assert result.password == None 59 | assert result.hostname == 'github.com' 60 | assert result.port == 22 61 | assert result.path == '/asdf/asdf.git' 62 | assert result.url == 'ssh://git@github.com:22/asdf/asdf.git' 63 | assert result.build_url(ignore_default_port=True) == 'ssh://git@github.com:/asdf/asdf.git' 64 | 65 | 66 | def test_git_uri_with_user_path() -> None: 67 | result = GitUri('git@github.com:asdf/asdf.git') 68 | assert result.scheme == 'ssh' 69 | assert result.username == 'git' 70 | assert result.password == None 71 | assert result.hostname == 'github.com' 72 | assert result.port == 22 73 | assert result.path == '/asdf/asdf.git' 74 | assert result.url == 'ssh://git@github.com:22/asdf/asdf.git' 75 | assert result.build_url(ignore_default_port=True) == 'ssh://git@github.com:/asdf/asdf.git' 76 | 77 | 78 | def test_uri_with_hostname_port_path() -> None: 79 | result = GitUri('github.com:2222/sdfsdf.git') 80 | assert result.scheme == 'ssh' 81 | assert result.username == None 82 | assert result.password == None 83 | assert result.hostname == 'github.com' 84 | assert result.port == 2222 85 | assert result.path == '/sdfsdf.git' 86 | assert result.url == 'ssh://github.com:2222/sdfsdf.git' 87 | assert result.build_url(ignore_default_port=True) == 'ssh://github.com:2222/sdfsdf.git' 88 | 89 | 90 | def test_uri_with_hostname_port() -> None: 91 | result = GitUri('github.com:2222') 92 | assert result.scheme == 'ssh' 93 | assert result.username == None 94 | assert result.password == None 95 | assert result.hostname == 'github.com' 96 | assert result.port == 2222 97 | assert result.path == '/' 98 | assert result.url == 'ssh://github.com:2222/' 99 | assert result.build_url(ignore_default_port=True) == 'ssh://github.com:2222/' 100 | 101 | 102 | def test_uri_with_hostname() -> None: 103 | result = GitUri('github.com') 104 | assert result.scheme == 'ssh' 105 | assert result.username == None 106 | assert result.password == None 107 | assert result.hostname == 'github.com' 108 | assert result.port == 22 109 | assert result.path == '/' 110 | assert result.url == 'ssh://github.com:22/' 111 | assert result.build_url(ignore_default_port=True) == 'ssh://github.com:/' 112 | 113 | 114 | def test_git_uri_with_scheme_user_path_with_leading_slash_no_coma() -> None: 115 | result = GitUri('ssh://git@github.com/asdf/asdf.git') 116 | assert result.scheme == 'ssh' 117 | assert result.username == 'git' 118 | assert result.password == None 119 | assert result.hostname == 'github.com' 120 | assert result.port == 22 121 | assert result.path == '/asdf/asdf.git' 122 | assert result.url == 'ssh://git@github.com:22/asdf/asdf.git' 123 | assert result.build_url(ignore_default_port=True) == 'ssh://git@github.com:/asdf/asdf.git' 124 | 125 | 126 | def test_https_uri_with_scheme_path() -> None: 127 | result = GitUri('https://github.com/Salamek/qiosk.git') 128 | assert result.scheme == 'https' 129 | assert result.username == None 130 | assert result.password == None 131 | assert result.hostname == 'github.com' 132 | assert result.port == 443 133 | assert result.path == '/Salamek/qiosk.git' 134 | assert result.url == 'https://github.com:443/Salamek/qiosk.git' 135 | assert result.build_url(ignore_default_port=True) == 'https://github.com/Salamek/qiosk.git' 136 | 137 | 138 | def test_http_uri_with_scheme_path() -> None: 139 | result = GitUri('http://github.com/Salamek/qiosk.git') 140 | assert result.scheme == 'http' 141 | assert result.username == None 142 | assert result.password == None 143 | assert result.hostname == 'github.com' 144 | assert result.port == 80 145 | assert result.path == '/Salamek/qiosk.git' 146 | assert result.url == 'http://github.com:80/Salamek/qiosk.git' 147 | assert result.build_url(ignore_default_port=True) == 'http://github.com/Salamek/qiosk.git' 148 | 149 | 150 | def test_http_uri_with_scheme_path_port() -> None: 151 | result = GitUri('http://github.com:8080/Salamek/qiosk.git') 152 | assert result.scheme == 'http' 153 | assert result.username == None 154 | assert result.password == None 155 | assert result.hostname == 'github.com' 156 | assert result.port == 8080 157 | assert result.path == '/Salamek/qiosk.git' 158 | assert result.url == 'http://github.com:8080/Salamek/qiosk.git' 159 | assert result.build_url(ignore_default_port=True) == 'http://github.com:8080/Salamek/qiosk.git' 160 | 161 | 162 | def test_https_uri_with_scheme_path_port() -> None: 163 | result = GitUri('https://github.com:8443/Salamek/qiosk.git') 164 | assert result.scheme == 'https' 165 | assert result.username == None 166 | assert result.password == None 167 | assert result.hostname == 'github.com' 168 | assert result.port == 8443 169 | assert result.path == '/Salamek/qiosk.git' 170 | assert result.url == 'https://github.com:8443/Salamek/qiosk.git' 171 | assert result.build_url(ignore_default_port=True) == 'https://github.com:8443/Salamek/qiosk.git' -------------------------------------------------------------------------------- /gitlab_tools/forms/pull_mirror.py: -------------------------------------------------------------------------------- 1 | 2 | from flask_babel import gettext 3 | from flask_login import current_user 4 | from wtforms import Form, StringField, validators, HiddenField, TextAreaField, BooleanField, SelectField 5 | from cron_descriptor import ExpressionDescriptor, MissingFieldException, FormatException 6 | from gitlab_tools.models.gitlab_tools import PullMirror 7 | from gitlab_tools.tools.GitRemote import GitRemote 8 | from gitlab_tools.forms.custom_fields import NonValidatingSelectField 9 | from gitlab_tools.tools.gitlab import check_project_visibility_in_group, VisibilityError, check_project_exists 10 | 11 | __author__ = "Adam Schubert" 12 | __date__ = "$26.7.2017 19:33:05$" 13 | 14 | 15 | class NewForm(Form): 16 | project_name = StringField( 17 | None, 18 | [ 19 | validators.Regexp( 20 | r'^[a-zA-Z0-9._-]+( \w+)*$', 21 | message='Project name cannot contain special characters' 22 | ) 23 | ] 24 | ) 25 | project_mirror = StringField(None, [validators.Length(min=1, max=255)]) 26 | periodic_sync = StringField(None, [validators.Optional()]) 27 | note = TextAreaField(None, [validators.Optional()]) 28 | group = NonValidatingSelectField(None, [validators.Optional()], coerce=int, choices=[]) 29 | 30 | is_no_create = BooleanField() 31 | is_force_create = BooleanField() 32 | is_no_remote = BooleanField() 33 | is_issues_enabled = BooleanField() 34 | is_wall_enabled = BooleanField() 35 | is_wiki_enabled = BooleanField() 36 | is_snippets_enabled = BooleanField() 37 | is_merge_requests_enabled = BooleanField() 38 | visibility = SelectField(None, [validators.DataRequired()], coerce=str, choices=[ 39 | (PullMirror.VISIBILITY_PRIVATE, gettext('Private')), 40 | (PullMirror.VISIBILITY_INTERNAL, gettext('Internal')), 41 | (PullMirror.VISIBILITY_PUBLIC, gettext('Public')) 42 | ]) 43 | is_force_update = BooleanField() 44 | is_prune_mirrors = BooleanField() 45 | is_jobs_enabled = BooleanField() 46 | 47 | def validate(self) -> bool: # pylint: disable=too-many-return-statements 48 | rv = Form.validate(self) 49 | if not rv: 50 | return False 51 | 52 | project_name_exists = PullMirror.query.filter_by(project_name=self.project_name.data, user=current_user).first() 53 | if project_name_exists: 54 | self.project_name.errors.append( 55 | gettext('Project name %(project_name)s already exists.', project_name=self.project_name.data) 56 | ) 57 | return False 58 | 59 | project_mirror_exists = PullMirror.query.filter_by(project_mirror=self.project_mirror.data, user=current_user).first() 60 | if project_mirror_exists: 61 | self.project_mirror.errors.append( 62 | gettext('Project mirror %(project_mirror)s already exists.', project_mirror=self.project_mirror.data) 63 | ) 64 | return False 65 | 66 | if not GitRemote(self.project_mirror.data).vcs_type: 67 | self.project_mirror.errors.append( 68 | gettext('Unknown VCS type or detection failed.') 69 | ) 70 | return False 71 | 72 | if not self.group.data: 73 | self.group.errors.append( 74 | gettext('You have to select GitLab group where mirrored project will be created.') 75 | ) 76 | return False 77 | 78 | try: 79 | check_project_visibility_in_group(self.visibility.data, self.group.data) 80 | except VisibilityError as e: 81 | self.visibility.errors.append( 82 | gettext(str(e)) 83 | ) 84 | return False 85 | 86 | if check_project_exists(self.project_name.data, self.group.data): 87 | if not self.is_force_create.data: 88 | self.project_name.errors.append( 89 | gettext( 90 | 'Project with name %(project_name)s already exists in selected group and ' 91 | '"Create project in GitLab even if it already exists." is not checked in "Advanced options"', 92 | project_name=self.project_name.data 93 | ) 94 | ) 95 | return False 96 | 97 | if self.periodic_sync.data: 98 | try: 99 | ExpressionDescriptor(self.periodic_sync.data) 100 | except (MissingFieldException, FormatException): 101 | self.periodic_sync.errors.append( 102 | gettext('Wrong cron expression.') 103 | ) 104 | 105 | return True 106 | 107 | 108 | class EditForm(NewForm): 109 | id = HiddenField() 110 | 111 | def validate(self) -> bool: # pylint: disable=too-many-return-statements 112 | rv = Form.validate(self) 113 | if not rv: 114 | return False 115 | 116 | current_pull_mirror = PullMirror.query.filter(PullMirror.id == self.id.data, PullMirror.user == current_user).first() 117 | if not current_pull_mirror: 118 | self.project_name.errors.append('Incorrect project ID') 119 | return False 120 | 121 | project_name_exists = PullMirror.query.filter( 122 | PullMirror.project_name == self.project_name.data, 123 | PullMirror.id != current_pull_mirror.id, 124 | PullMirror.user == current_user 125 | ).first() 126 | if project_name_exists: 127 | self.project_name.errors.append( 128 | gettext('Project name %(project_name)s already exists.', project_name=self.project_name.data) 129 | ) 130 | return False 131 | 132 | project_mirror_exists = PullMirror.query.filter( 133 | PullMirror.project_mirror == self.project_mirror.data, 134 | PullMirror.id != current_pull_mirror.id, 135 | PullMirror.user == current_user 136 | ).first() 137 | if project_mirror_exists: 138 | self.project_mirror.errors.append( 139 | gettext('Project mirror %(project_mirror)s already exists.', project_mirror=self.project_mirror.data) 140 | ) 141 | return False 142 | 143 | if not GitRemote(self.project_mirror.data).vcs_type: 144 | self.project_mirror.errors.append( 145 | gettext('Unknown VCS type or detection failed.') 146 | ) 147 | return False 148 | 149 | try: 150 | check_project_visibility_in_group(self.visibility.data, self.group.data) 151 | except VisibilityError as e: 152 | self.visibility.errors.append( 153 | gettext(str(e)) 154 | ) 155 | return False 156 | 157 | if check_project_exists(self.project_name.data, self.group.data, current_pull_mirror.project.gitlab_id): 158 | if not self.is_force_create.data: 159 | self.project_name.errors.append( 160 | gettext( 161 | 'Project with name %(project_name)s already exists in selected group and ' 162 | '"Create project in GitLab even if it already exists." is not checked in "Advanced options"', 163 | project_name=self.project_name.data 164 | ) 165 | ) 166 | return False 167 | 168 | if self.periodic_sync.data: 169 | try: 170 | ExpressionDescriptor(self.periodic_sync.data) 171 | except (MissingFieldException, FormatException): 172 | self.periodic_sync.errors.append( 173 | gettext('Wrong cron expression.') 174 | ) 175 | 176 | return True 177 | -------------------------------------------------------------------------------- /gitlab_tools/models/gitlab_tools.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from sqlalchemy.orm import relationship, backref 3 | from sqlalchemy.sql import func 4 | from sqlalchemy.ext.declarative import declared_attr 5 | from gitlab_tools.extensions import db 6 | from gitlab_tools.enums.InvokedByEnum import InvokedByEnum 7 | 8 | __author__ = "Adam Schubert" 9 | __date__ = "$26.7.2017 19:33:05$" 10 | 11 | 12 | class BaseTable(db.Model): # type: ignore 13 | __abstract__ = True 14 | updated = db.Column(db.DateTime, default=func.now(), onupdate=func.current_timestamp()) 15 | created = db.Column(db.DateTime, default=func.now()) 16 | 17 | 18 | class Mirror(BaseTable): 19 | __abstract__ = True 20 | 21 | @declared_attr 22 | def user_id(self) -> db.Column: 23 | return db.Column(db.Integer, db.ForeignKey('user.id'), index=True) 24 | 25 | @declared_attr 26 | def project_id(self) -> db.Column: 27 | return db.Column(db.Integer, db.ForeignKey('project.id'), index=True) 28 | 29 | source = db.Column(db.String(255), nullable=True) 30 | target = db.Column(db.String(255), nullable=True) 31 | foreign_vcs_type = db.Column(db.Integer) 32 | last_sync = db.Column(db.DateTime, nullable=True) 33 | note = db.Column(db.String(255)) 34 | hook_token = db.Column(db.String(255)) 35 | is_force_update = db.Column(db.Boolean) 36 | is_prune_mirrors = db.Column(db.Boolean) 37 | is_deleted = db.Column(db.Boolean) 38 | 39 | 40 | class OAuth2State(BaseTable): 41 | __tablename__ = 'o_auth2_state' 42 | id = db.Column(db.Integer, primary_key=True) 43 | state = db.Column(db.String(255), unique=True) 44 | 45 | 46 | class User(BaseTable): 47 | __tablename__ = 'user' 48 | id = db.Column(db.Integer, primary_key=True) 49 | gitlab_id = db.Column(db.Integer, unique=True, nullable=False) 50 | gitlab_deploy_key_id = db.Column(db.Integer, unique=True, nullable=True) 51 | is_rsa_pair_set = db.Column(db.Boolean) 52 | name = db.Column(db.String(255)) 53 | avatar_url = db.Column(db.String(255)) 54 | access_token = db.Column(db.String(255), unique=True, nullable=True) 55 | refresh_token = db.Column(db.String(255), unique=True, nullable=True) 56 | expires = db.Column(db.DateTime, default=func.now()) 57 | pull_mirrors = relationship("PullMirror", order_by="PullMirror.id", backref="user", lazy='dynamic') 58 | push_mirrors = relationship("PushMirror", order_by="PushMirror.id", backref="user", lazy='dynamic') 59 | fingerprints = relationship("Fingerprint", order_by="Fingerprint.id", backref="user", lazy='dynamic') 60 | 61 | @property 62 | def is_active(self) -> bool: 63 | return True 64 | 65 | @property 66 | def is_authenticated(self) -> bool: 67 | return True 68 | 69 | @property 70 | def is_anonymous(self) -> bool: 71 | return False 72 | 73 | def get_id(self) -> int: 74 | try: 75 | if isinstance(self.id, str): 76 | return int(self.id) 77 | if isinstance(self.id, int): 78 | return self.id 79 | 80 | return self.id 81 | except AttributeError as e: 82 | raise NotImplementedError('No `id` attribute - override `get_id`') from e 83 | 84 | def __eq__(self, other) -> bool: 85 | """ 86 | Checks the equality of two `UserMixin` objects using `get_id`. 87 | """ 88 | if isinstance(other, User): 89 | return self.get_id() == other.get_id() 90 | return NotImplemented 91 | 92 | def __ne__(self, other) -> bool: 93 | """ 94 | Checks the inequality of two `UserMixin` objects using `get_id`. 95 | """ 96 | equal = self.__eq__(other) 97 | if equal is NotImplemented: 98 | return NotImplemented 99 | return not equal 100 | 101 | if sys.version_info[0] != 2: # pragma: no cover 102 | # Python 3 implicitly set __hash__ to None if we override __eq__ 103 | # We set it back to its default implementation 104 | __hash__ = object.__hash__ 105 | 106 | 107 | class Fingerprint(BaseTable): 108 | __tablename__ = 'fingerprint' 109 | __table_args__ = ( 110 | db.UniqueConstraint('user_id', 'hashed_hostname', name='_user_id_hashed_hostname_uc'), 111 | ) 112 | id = db.Column(db.Integer, primary_key=True) 113 | user_id = db.Column(db.Integer, db.ForeignKey('user.id'), index=True) 114 | hostname = db.Column(db.String(255)) 115 | sha256_fingerprint = db.Column(db.String(255)) 116 | hashed_hostname = db.Column(db.String(255), index=True) 117 | 118 | 119 | class PullMirror(Mirror): 120 | VISIBILITY_PRIVATE = 'private' 121 | VISIBILITY_INTERNAL = 'internal' 122 | VISIBILITY_PUBLIC = 'public' 123 | 124 | __tablename__ = 'pull_mirror' 125 | id = db.Column(db.Integer, primary_key=True) 126 | group_id = db.Column(db.Integer, db.ForeignKey('group.id'), index=True) 127 | periodic_task_id = db.Column(db.Integer, db.ForeignKey('periodic_task.id'), index=True, nullable=True) 128 | periodic_sync = db.Column(db.String(255), nullable=True) 129 | project_name = db.Column(db.String(255)) 130 | project_mirror = db.Column(db.String(255)) 131 | is_no_create = db.Column(db.Boolean) 132 | is_force_create = db.Column(db.Boolean) 133 | is_no_remote = db.Column(db.Boolean) 134 | is_issues_enabled = db.Column(db.Boolean) 135 | is_wall_enabled = db.Column(db.Boolean) 136 | is_wiki_enabled = db.Column(db.Boolean) 137 | is_jobs_enabled = db.Column(db.Boolean, nullable=False) 138 | is_snippets_enabled = db.Column(db.Boolean) 139 | is_merge_requests_enabled = db.Column(db.Boolean) 140 | visibility = db.Column(db.String(255), nullable=False) 141 | task_results = relationship("TaskResult", backref="pull_mirror", lazy='dynamic', order_by='desc(TaskResult.updated)') 142 | 143 | 144 | class PushMirror(Mirror): 145 | __tablename__ = 'push_mirror' 146 | id = db.Column(db.Integer, primary_key=True) 147 | project_mirror = db.Column(db.String(255)) 148 | task_results = relationship("TaskResult", backref="push_mirror", lazy='dynamic', order_by='desc(TaskResult.updated)') 149 | 150 | 151 | class Project(BaseTable): 152 | __tablename__ = 'project' 153 | 154 | id = db.Column(db.Integer, primary_key=True) 155 | gitlab_id = db.Column(db.Integer, unique=True) 156 | name = db.Column(db.String(255)) 157 | name_with_namespace = db.Column(db.String(255)) 158 | web_url = db.Column(db.String(255)) 159 | push_mirrors = relationship("PushMirror", order_by="PushMirror.id", backref="project", lazy='dynamic') 160 | pull_mirrors = relationship("PullMirror", order_by="PullMirror.id", backref="project", lazy='dynamic') 161 | 162 | 163 | class Group(BaseTable): 164 | __tablename__ = 'group' 165 | 166 | id = db.Column(db.Integer, primary_key=True) 167 | gitlab_id = db.Column(db.Integer, unique=True) 168 | name = db.Column(db.String(255)) 169 | pull_mirrors = relationship("PullMirror", order_by="PullMirror.id", backref="group", lazy='dynamic') 170 | 171 | 172 | class TaskResult(BaseTable): 173 | __tablename__ = 'task_result' 174 | id = db.Column(db.Integer, primary_key=True) 175 | pull_mirror_id = db.Column(db.Integer, db.ForeignKey('pull_mirror.id'), index=True, nullable=True) 176 | push_mirror_id = db.Column(db.Integer, db.ForeignKey('push_mirror.id'), index=True, nullable=True) 177 | task_name = db.Column(db.String(255)) 178 | invoked_by = db.Column(db.Integer, default=InvokedByEnum.UNKNOWN) 179 | taskmeta_id = db.Column(db.Integer, db.ForeignKey('celery_taskmeta.id'), index=True, nullable=False, unique=True) 180 | parent_id = db.Column(db.Integer, db.ForeignKey('task_result.id'), index=True, nullable=True) 181 | children = relationship("TaskResult", backref=backref('parent', remote_side=[id])) 182 | -------------------------------------------------------------------------------- /gitlab_tools/models/celery.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from sqlalchemy.orm import relationship 3 | from sqlalchemy.sql import func 4 | from celery import schedules, states 5 | from gitlab_tools.extensions import db 6 | 7 | 8 | class ConstraintError(Exception): 9 | pass 10 | 11 | 12 | class BaseTable(db.Model): # type: ignore 13 | __abstract__ = True 14 | updated = db.Column(db.DateTime, default=func.now(), onupdate=func.current_timestamp()) 15 | created = db.Column(db.DateTime, default=func.now()) 16 | 17 | @classmethod 18 | def filter_by(cls, session, **kwargs): 19 | """ 20 | session.query(MyClass).filter_by(name = 'some name') 21 | :param kwargs: 22 | :param session: 23 | """ 24 | return session.query(cls).filter_by(**kwargs) 25 | 26 | @classmethod 27 | def get_or_create(cls, session_obj, defaults=None, **kwargs): 28 | obj = session_obj.query(cls).filter_by(**kwargs).first() 29 | if obj: 30 | return obj, False 31 | 32 | params = dict((k, v) for k, v in kwargs.items()) 33 | params.update(defaults or {}) 34 | obj = cls(**params) 35 | return obj, True 36 | 37 | @classmethod 38 | def update_or_create(cls, session_obj, defaults=None, **kwargs): 39 | obj = session_obj.query(cls).filter_by(**kwargs).first() 40 | if obj: 41 | for key, value in defaults.items(): 42 | setattr(obj, key, value) 43 | created = False 44 | else: 45 | params = dict((k, v) for k, v in kwargs.items()) 46 | params.update(defaults or {}) 47 | obj = cls(**params) 48 | created = True 49 | return obj, created 50 | 51 | 52 | class IntervalSchedule(BaseTable): 53 | __tablename__ = "interval_schedule" 54 | """ 55 | PERIOD_CHOICES = (('days', 'Days'), 56 | ('hours', 'Hours'), 57 | ('minutes', 'Minutes'), 58 | ('seconds', 'Seconds'), 59 | ('microseconds', 'Microseconds')) 60 | """ 61 | id = db.Column(db.Integer, primary_key=True) 62 | every = db.Column(db.Integer, nullable=False) 63 | period = db.Column(db.Unicode(255)) 64 | periodic_tasks = relationship('PeriodicTask') 65 | 66 | @property 67 | def schedule(self): 68 | return schedules.schedule(datetime.timedelta(**{self.period.code: self.every})) 69 | 70 | @classmethod 71 | def from_schedule(cls, session, schedule, period='seconds'): 72 | every = max(schedule.run_every.total_seconds(), 0) 73 | obj = cls.filter_by(session, every=every, period=period).first() 74 | if obj is None: 75 | return cls(every=every, period=period) 76 | 77 | return obj 78 | 79 | def __str__(self): 80 | if self.every == 1: 81 | return 'every {0.period_singular}'.format(self) 82 | return 'every {0.every} {0.period}'.format(self) 83 | 84 | @property 85 | def period_singular(self): 86 | return self.period[:-1] 87 | 88 | 89 | class CrontabSchedule(BaseTable): 90 | """ 91 | Task result/status. 92 | """ 93 | __tablename__ = "crontab_schedule" 94 | id = db.Column(db.Integer, primary_key=True) 95 | minute = db.Column(db.String(length=120), default="*") 96 | hour = db.Column(db.String(length=120), default="*") 97 | day_of_week = db.Column(db.String(length=120), default="*") 98 | day_of_month = db.Column(db.String(length=120), default="*") 99 | month_of_year = db.Column(db.String(length=120), default="*") 100 | periodic_tasks = relationship('PeriodicTask') 101 | 102 | def __str__(self): 103 | rfield = lambda f: f and str(f).replace(' ', '') or '*' 104 | return '{0} {1} {2} {3} {4} (m/h/d/dM/MY)'.format( 105 | rfield(self.minute), rfield(self.hour), rfield(self.day_of_week), 106 | rfield(self.day_of_month), rfield(self.month_of_year), 107 | ) 108 | 109 | @property 110 | def schedule(self): 111 | return schedules.crontab(minute=self.minute, 112 | hour=self.hour, 113 | day_of_week=self.day_of_week, 114 | day_of_month=self.day_of_month, 115 | month_of_year=self.month_of_year) 116 | 117 | @classmethod 118 | def from_schedule(cls, session, schedule): 119 | spec = {'minute': schedule._orig_minute, 120 | 'hour': schedule._orig_hour, 121 | 'day_of_week': schedule._orig_day_of_week, 122 | 'day_of_month': schedule._orig_day_of_month, 123 | 'month_of_year': schedule._orig_month_of_year} 124 | obj = cls.filter_by(session, **spec).first() 125 | if obj is None: 126 | return cls(**spec) 127 | return obj 128 | 129 | 130 | class PeriodicTasks(BaseTable): 131 | __tablename__ = "periodic_tasks" 132 | id = db.Column(db.Integer, primary_key=True) 133 | ident = db.Column(db.Integer, default=1, index=True) 134 | last_update = db.Column(db.DateTime, default=datetime.datetime.utcnow) 135 | 136 | @classmethod 137 | def changed(cls): 138 | found = PeriodicTasks.query.filter_by(ident=1).first() 139 | if not found: 140 | found = PeriodicTasks() 141 | found.last_update = datetime.datetime.now() 142 | db.session.add(found) 143 | 144 | @classmethod 145 | def last_change(cls, session): 146 | obj = cls.filter_by(session, ident=1).first() 147 | return obj.last_update if obj else None 148 | 149 | 150 | class PeriodicTask(BaseTable): 151 | __tablename__ = "periodic_task" 152 | id = db.Column(db.Integer, primary_key=True) 153 | name = db.Column(db.String(length=200), unique=True) 154 | task = db.Column(db.String(length=200)) 155 | crontab_id = db.Column(db.Integer, db.ForeignKey('crontab_schedule.id')) 156 | crontab = relationship("CrontabSchedule", back_populates="periodic_tasks") 157 | interval_id = db.Column(db.Integer, db.ForeignKey('interval_schedule.id')) 158 | interval = relationship("IntervalSchedule", back_populates="periodic_tasks") 159 | args = db.Column(db.Text, default='[]') 160 | kwargs = db.Column(db.Text, default='{}') 161 | queue = db.Column(db.String(length=200)) 162 | exchange = db.Column(db.String(length=200)) 163 | routing_key = db.Column(db.String(length=200)) 164 | headers = db.Column(db.Text, default='{}') 165 | priority = db.Column(db.Integer) 166 | expires = db.Column(db.DateTime) 167 | expire_seconds = db.Column(db.Integer) 168 | one_off = db.Column(db.Boolean, default=False) 169 | start_time = db.Column(db.DateTime) 170 | enabled = db.Column(db.Boolean, default=True) 171 | last_run_at = db.Column(db.DateTime) 172 | total_run_count = db.Column(db.Integer, default=0) 173 | 174 | pull_mirrors = relationship("PullMirror", order_by="PullMirror.id", backref="periodic_task", lazy='dynamic') 175 | no_changes = False 176 | 177 | def __str__(self): 178 | fmt = '{0.name}: {0.crontab}' 179 | return fmt.format(self) 180 | 181 | @property 182 | def schedule(self): 183 | if self.crontab: 184 | return self.crontab.schedule 185 | if self.interval: 186 | return self.interval.schedule 187 | 188 | return None 189 | 190 | 191 | class TaskMeta(db.Model): 192 | __tablename__ = 'celery_taskmeta' 193 | __table_args__ = {'sqlite_autoincrement': True} 194 | id = db.Column(db.Integer, db.Sequence('task_id_sequence'), 195 | primary_key=True, autoincrement=True) 196 | task_id = db.Column(db.String(155), unique=True) 197 | status = db.Column(db.String(50), default=states.PENDING) 198 | result = db.Column(db.PickleType, nullable=True) 199 | date_done = db.Column(db.DateTime, default=datetime.datetime.utcnow, 200 | onupdate=datetime.datetime.utcnow, nullable=True) 201 | traceback = db.Column(db.Text, nullable=True) 202 | task_result = relationship("TaskResult", backref="taskmeta", uselist=False) 203 | 204 | 205 | class TaskSet(db.Model): 206 | """TaskSet result.""" 207 | __tablename__ = 'celery_tasksetmeta' 208 | __table_args__ = {'sqlite_autoincrement': True} 209 | 210 | id = db.Column(db.Integer, db.Sequence('taskset_id_sequence'), 211 | autoincrement=True, primary_key=True) 212 | taskset_id = db.Column(db.String(155), unique=True) 213 | result = db.Column(db.PickleType, nullable=True) 214 | date_done = db.Column(db.DateTime, default=datetime.datetime.utcnow, 215 | nullable=True) 216 | --------------------------------------------------------------------------------