├── tests ├── __init__.py ├── automation │ ├── __init__.py │ ├── test_git_wrapper.py │ ├── conftest.py │ ├── test_gitlab.py │ └── test_git.py ├── strategies │ ├── __init__.py │ ├── test_overwrite.yml │ ├── test_no_extra_keys_schema.yml │ ├── test_if_missing.yml │ ├── test_detect_newline.py │ ├── test_sorted_unique_lines.yml │ ├── conftest.py │ ├── test_config_parser_merge_pylintrc.yml │ ├── test_template_hash.yml │ ├── test_config_parser_merge.yml │ ├── test_rendered_template_file_hash.yml │ └── test_setupcfg_merge.yml ├── test_metadata.py ├── test_cli.py ├── conftest.py ├── test_cookecutter.py ├── test_template.py ├── test_gitremotes.py ├── test_config.py └── test_rollup.py ├── src └── scaraplate │ ├── py.typed │ ├── automation │ ├── __init__.py │ ├── base.py │ ├── gitlab.py │ └── git.py │ ├── fields.py │ ├── __init__.py │ ├── compat.py │ ├── __main__.py │ ├── template.py │ ├── gitremotes.py │ ├── cookiecutter.py │ ├── config.py │ └── rollup.py ├── docs ├── _static │ ├── scarab.jpg │ └── license.svg ├── Makefile ├── gitremotes.rst ├── strategies.rst ├── conf.py ├── index.rst ├── automation.rst └── template.rst ├── MANIFEST.in ├── Makefile.inc ├── .editorconfig ├── tox.ini ├── .gitignore ├── LICENSE ├── RELEASE.md ├── README.md ├── .github └── workflows │ └── ci.yml ├── setup.py ├── CHANGES.rst ├── .pylintrc ├── setup.cfg └── Makefile /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/scaraplate/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/automation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/strategies/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_static/scarab.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rambler-digital-solutions/scaraplate/HEAD/docs/_static/scarab.jpg -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft tests 2 | global-exclude *.py[co] 3 | global-exclude __pycache__ 4 | include LICENSE 5 | include README.md 6 | include src/scaraplate/py.typed 7 | include VERSION 8 | -------------------------------------------------------------------------------- /tests/strategies/test_overwrite.yml: -------------------------------------------------------------------------------- 1 | 2 | strategy: Overwrite 3 | 4 | testcases: 5 | - name: binary_files_with_newlines_are_not_corrupted 6 | template: "a\ra" 7 | target: "b\nb" 8 | out: "a\ra" 9 | -------------------------------------------------------------------------------- /Makefile.inc: -------------------------------------------------------------------------------- 1 | 2 | .PHONY: check-docs 3 | check-docs: 4 | @# Doesn't generate any output but prints out errors and warnings. 5 | make -C docs dummy 6 | 7 | .PHONY: docs 8 | docs: 9 | make -C docs html 10 | -------------------------------------------------------------------------------- /tests/strategies/test_no_extra_keys_schema.yml: -------------------------------------------------------------------------------- 1 | strategy: Overwrite 2 | 3 | testcases: 4 | - name: extra_keys_are_not_allowed 5 | template: hi 6 | target: hello 7 | out: null 8 | config: 9 | my_unexpected_key: hey 10 | raises: marshmallow.exceptions.ValidationError 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # https://editorconfig.org/ 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | indent_size = 4 9 | indent_style = space 10 | insert_final_newline = true 11 | 12 | [*.py] 13 | max_line_length = 88 14 | trim_trailing_whitespace = true 15 | 16 | [Makefile] 17 | indent_style = tab 18 | -------------------------------------------------------------------------------- /src/scaraplate/automation/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import ProjectVCS, TemplateVCS, automatic_rollup 2 | from .git import GitCloneProjectVCS, GitCloneTemplateVCS 3 | 4 | __all__ = ( 5 | "GitCloneProjectVCS", 6 | "GitCloneTemplateVCS", 7 | "ProjectVCS", 8 | "TemplateVCS", 9 | "automatic_rollup", 10 | ) 11 | -------------------------------------------------------------------------------- /tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | from pkg_resources import get_distribution 2 | 3 | 4 | def test_metadata(): 5 | metadata_name = "scaraplate" 6 | pkg = get_distribution(metadata_name) 7 | assert pkg.version 8 | assert pkg.project_name == metadata_name 9 | 10 | 11 | def test_package_import(): 12 | import scaraplate 13 | 14 | assert scaraplate 15 | -------------------------------------------------------------------------------- /tests/strategies/test_if_missing.yml: -------------------------------------------------------------------------------- 1 | 2 | strategy: IfMissing 3 | 4 | testcases: 5 | - name: non_existing_target 6 | template: | 7 | from template! 8 | target: null 9 | out: | 10 | from template! 11 | 12 | - name: existing_target 13 | template: | 14 | from template! 15 | target: | 16 | from target! 17 | out: | 18 | from target! 19 | 20 | - name: binary_files_with_newlines_are_not_corrupted 21 | template: "a\ra" 22 | target: "b\nb" 23 | out: "b\nb" 24 | -------------------------------------------------------------------------------- /src/scaraplate/fields.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from marshmallow import ValidationError, fields 4 | 5 | 6 | class Pattern(fields.Field): 7 | def _serialize(self, value, attr, obj): 8 | self.fail("not implemented") 9 | 10 | def _deserialize(self, value, attr, data, **kwargs): 11 | if value is None: 12 | return None 13 | 14 | try: 15 | return re.compile(value) 16 | except re.error as exc: 17 | raise ValidationError(f"Unable to compile PCRE pattern: {exc}") 18 | -------------------------------------------------------------------------------- /src/scaraplate/__init__.py: -------------------------------------------------------------------------------- 1 | from .automation import ( 2 | GitCloneProjectVCS, 3 | GitCloneTemplateVCS, 4 | ProjectVCS, 5 | TemplateVCS, 6 | automatic_rollup, 7 | ) 8 | from .rollup import InvalidScaraplateTemplateError, rollup 9 | from .template import TemplateMeta 10 | 11 | __all__ = ( 12 | "GitCloneProjectVCS", 13 | "GitCloneTemplateVCS", 14 | "InvalidScaraplateTemplateError", 15 | "ProjectVCS", 16 | "TemplateMeta", 17 | "TemplateVCS", 18 | "automatic_rollup", 19 | "rollup", 20 | ) 21 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= -n 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | .DEFAULT: 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/gitremotes.rst: -------------------------------------------------------------------------------- 1 | Template Git Remotes 2 | ==================== 3 | 4 | 5 | .. automodule:: scaraplate.gitremotes 6 | :members: __doc__ 7 | 8 | 9 | .. autoclass:: scaraplate.template.TemplateMeta 10 | :show-inheritance: 11 | :members: 12 | :undoc-members: 13 | 14 | .. autoclass:: scaraplate.gitremotes.GitRemote 15 | :show-inheritance: 16 | :members: 17 | 18 | .. automethod:: __init__ 19 | 20 | 21 | Built-in Git Remotes 22 | -------------------- 23 | 24 | .. autoclass:: scaraplate.gitremotes.GitLab 25 | :show-inheritance: 26 | 27 | 28 | .. autoclass:: scaraplate.gitremotes.GitHub 29 | :show-inheritance: 30 | 31 | 32 | .. autoclass:: scaraplate.gitremotes.BitBucket 33 | :show-inheritance: 34 | -------------------------------------------------------------------------------- /src/scaraplate/compat.py: -------------------------------------------------------------------------------- 1 | try: 2 | from marshmallow import __version_info__ 3 | 4 | is_marshmallow_3 = __version_info__[0] >= 3 5 | except ImportError: # pragma: no cover 6 | is_marshmallow_3 = False 7 | 8 | 9 | def marshmallow_load_data(schema, data): 10 | if is_marshmallow_3: 11 | return schema().load(data) 12 | else: # pragma: no cover 13 | # 2.X line 14 | return schema(strict=True).load(data).data 15 | 16 | 17 | def marshmallow_pass_original_for_many(original_data, many): 18 | if is_marshmallow_3: 19 | return [original_data] 20 | else: # pragma: no cover 21 | if not many: 22 | # `many=True` field would contain a list here, otherwise 23 | # it would be a dict. 24 | original_data = [original_data] 25 | return original_data 26 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist= 3 | py{37,38,39,310,311,312}{-minimal,-full}, 4 | lint, 5 | check-docs, 6 | 7 | [gh-actions] 8 | python = 9 | 3.7: py37 10 | 3.8: py38 11 | 3.9: py39 12 | 3.10: py310 13 | 3.11: py311 14 | 3.12: py312 15 | 16 | [testenv] 17 | ; Skip coverage for -minimal, because it would fail due to low coverage 18 | ; since not everything is tested when some extras are not installed. 19 | commands = sh -c 'envname={envname}; if [ "$envname" != "$\{envname%-minimal\}" ]; then pytest; else make test; fi' 20 | extras = 21 | develop 22 | full: gitlab 23 | usedevelop = True 24 | allowlist_externals = 25 | make 26 | sh 27 | 28 | [testenv:lint] 29 | basepython = python3 30 | commands = make lint 31 | extras = 32 | develop 33 | gitlab 34 | 35 | [testenv:check-docs] 36 | basepython = python3 37 | commands = make check-docs 38 | extras = 39 | develop 40 | gitlab 41 | -------------------------------------------------------------------------------- /docs/_static/license.svg: -------------------------------------------------------------------------------- 1 | licenselicenseMITMIT -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *$py.class 2 | *.cover 3 | *.egg 4 | *.egg-info/ 5 | *.log 6 | *.manifest 7 | *.mo 8 | *.pot 9 | *.py[cod] 10 | *.sage.py 11 | *.so 12 | *.spec 13 | .cache 14 | .coverage 15 | .coverage.* 16 | .dmypy.json 17 | .eggs/ 18 | .env 19 | .hypothesis/ 20 | .idea 21 | .installed.cfg 22 | .ipynb_checkpoints 23 | .mypy_cache/ 24 | .nox/ 25 | .pyre/ 26 | .pytest_cache/ 27 | .Python 28 | .python-version 29 | .ropeproject 30 | .scrapy 31 | .spyderproject 32 | .spyproject 33 | .tox/ 34 | .venv 35 | .webassets-cache 36 | /site 37 | __pycache__/ 38 | build/ 39 | celerybeat-schedule 40 | coverage.xml 41 | db.sqlite3 42 | develop-eggs/ 43 | dist/ 44 | dmypy.json 45 | docs/_build/ 46 | downloads/ 47 | eggs/ 48 | env.bak/ 49 | env/ 50 | ENV/ 51 | htmlcov/ 52 | instance/ 53 | ipython_config.py 54 | junit.xml 55 | lib/ 56 | lib64/ 57 | local_settings.py 58 | MANIFEST 59 | nosetests.xml 60 | parts/ 61 | pip-delete-this-directory.txt 62 | pip-log.txt 63 | profile_default/ 64 | sdist/ 65 | share/python-wheels/ 66 | target/ 67 | var/ 68 | venv.bak/ 69 | venv/ 70 | VERSION 71 | wheels/ 72 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Rambler Digital Solutions 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release process 2 | 3 | ## Prepare 4 | 5 | 1. Update cookiecutter version in `docs/conf.py`, ensure `make docs` produces 6 | docs with browsable cookiecutter links. 7 | 1. Add missing `.. versionadded`/`.. versionchanged` directives 8 | where appropriate. 9 | 1. Add a changelog entry to the CHANGES.rst file. 10 | 1. Push the changes, ensure that the CI build is green and all tests pass. 11 | 12 | ## Release 13 | 14 | 1. Upload a new pypi release: 15 | ``` 16 | git tag -s 0.5 17 | make clean 18 | make dist 19 | git push origin 0.5 20 | twine upload -s dist/* 21 | ``` 22 | 1. Create a new release for the pushed tag at 23 | https://github.com/rambler-digital-solutions/scaraplate/releases 24 | 1. Upload a GPG signature of the tarball to the just created GitHub release, 25 | see https://wiki.debian.org/Creating%20signed%20GitHub%20releases 26 | 27 | ## Check 28 | 29 | 1. Ensure that the uploaded version works in a clean environment 30 | (e.g. `docker run -it --rm python:3.7 bash`) 31 | and execute the quickstart scenario in the docs. 32 | 1. Ensure that RTD builds have passed and the `stable` version has updated: 33 | https://readthedocs.org/projects/scaraplate/builds/ 34 | 1. Ensure that the CI build for the tag is green. 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # scaraplate 2 | 3 | [![GitHub Actions Build status][gha-badge]][gha-link] 4 | [![scaraplate on pypi][pypi-badge]][pypi-link] 5 | [![docs at readthedocs][docs-badge]][docs-link] 6 | [![Licensed under MIT][license-badge]][license-link] 7 | [![Code style: black][black-badge]][black-link] 8 | 9 | [gha-badge]: https://img.shields.io/github/actions/workflow/status/rambler-digital-solutions/scaraplate/ci.yml?branch=main 10 | [gha-link]: https://github.com/rambler-digital-solutions/scaraplate/actions 11 | [pypi-badge]: https://img.shields.io/pypi/v/scaraplate.svg 12 | [pypi-link]: https://pypi.org/project/scaraplate/ 13 | [docs-badge]: https://readthedocs.org/projects/scaraplate/badge/?version=latest 14 | [docs-link]: https://scaraplate.readthedocs.io/ 15 | [license-badge]: https://scaraplate.readthedocs.io/en/latest/_static/license.svg 16 | [license-link]: https://github.com/rambler-digital-solutions/scaraplate/blob/main/LICENSE 17 | [black-badge]: https://img.shields.io/badge/code%20style-black-000000.svg 18 | [black-link]: https://github.com/psf/black 19 | 20 | 21 | Scaraplate is a wrapper around [cookiecutter][cookiecutter] 22 | which allows to repeatedly rollup project templates onto concrete projects. 23 | 24 | [cookiecutter]: https://github.com/cookiecutter/cookiecutter 25 | 26 | Docs: https://scaraplate.readthedocs.io/ 27 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from click.testing import CliRunner 4 | 5 | import scaraplate.__main__ as main_module 6 | from scaraplate.__main__ import main 7 | 8 | 9 | def test_help(): 10 | runner = CliRunner() 11 | result = runner.invoke(main, ["--help"]) 12 | assert result.exit_code == 0, result.output 13 | 14 | 15 | def test_rollup_help(): 16 | runner = CliRunner() 17 | result = runner.invoke(main, ["rollup", "--help"]) 18 | assert result.exit_code == 0, result.output 19 | 20 | 21 | def test_extra_context(): 22 | with patch.object(main_module, "_rollup") as mock_rollup: 23 | runner = CliRunner() 24 | result = runner.invoke( 25 | main, ["rollup", ".", "mydest", "key1=value1", "key2=value2"] 26 | ) 27 | assert result.exit_code == 0, result.output 28 | 29 | assert mock_rollup.call_count == 1 30 | _, kwargs = mock_rollup.call_args 31 | assert kwargs["extra_context"] == {"key1": "value1", "key2": "value2"} 32 | 33 | 34 | def test_extra_context_incorrect(): 35 | with patch.object(main_module, "_rollup"): 36 | runner = CliRunner() 37 | result = runner.invoke(main, ["rollup", ".", "mydest", "key1value1"]) 38 | assert result.exit_code == 2, result.output 39 | assert ( 40 | "EXTRA_CONTEXT should contain items of the form key=value" in result.output 41 | ) 42 | -------------------------------------------------------------------------------- /docs/strategies.rst: -------------------------------------------------------------------------------- 1 | Strategies 2 | ========== 3 | 4 | .. automodule:: scaraplate.strategies 5 | :members: __doc__ 6 | 7 | 8 | .. autoclass:: scaraplate.strategies.Strategy() 9 | :show-inheritance: 10 | :members: __init__, apply 11 | 12 | .. autoclass:: scaraplate.strategies.Strategy.Schema() 13 | 14 | 15 | Built-in Strategies 16 | ------------------- 17 | 18 | .. autoclass:: scaraplate.strategies.Overwrite() 19 | :show-inheritance: 20 | 21 | .. autoclass:: scaraplate.strategies.Overwrite.Schema() 22 | 23 | 24 | .. autoclass:: scaraplate.strategies.IfMissing() 25 | :show-inheritance: 26 | 27 | .. autoclass:: scaraplate.strategies.IfMissing.Schema() 28 | 29 | 30 | .. autoclass:: scaraplate.strategies.SortedUniqueLines() 31 | :show-inheritance: 32 | 33 | .. autoclass:: scaraplate.strategies.SortedUniqueLines.Schema() 34 | 35 | 36 | .. autoclass:: scaraplate.strategies.TemplateHash() 37 | :show-inheritance: 38 | 39 | .. autoclass:: scaraplate.strategies.TemplateHash.Schema() 40 | 41 | 42 | .. autoclass:: scaraplate.strategies.RenderedTemplateFileHash() 43 | :show-inheritance: 44 | 45 | .. autoclass:: scaraplate.strategies.RenderedTemplateFileHash.Schema() 46 | 47 | 48 | .. autoclass:: scaraplate.strategies.ConfigParserMerge() 49 | :show-inheritance: 50 | 51 | .. autoclass:: scaraplate.strategies.ConfigParserMerge.Schema() 52 | 53 | 54 | .. autoclass:: scaraplate.strategies.SetupCfgMerge() 55 | :show-inheritance: 56 | 57 | .. autoclass:: scaraplate.strategies.SetupCfgMerge.Schema() 58 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: {} 5 | push: {} 6 | 7 | jobs: 8 | lint: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | with: 13 | fetch-depth: 0 14 | - name: Set up Python 3 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: '3.x' 18 | - name: Install dependencies 19 | run: | 20 | python3 -m pip install --upgrade pip 21 | python3 -m pip install tox tox-gh-actions 22 | - run: tox -e lint 23 | 24 | check-docs: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | with: 29 | fetch-depth: 0 30 | - name: Set up Python 3 31 | uses: actions/setup-python@v2 32 | with: 33 | python-version: '3.x' 34 | - name: Install dependencies 35 | run: | 36 | python3 -m pip install --upgrade pip 37 | python3 -m pip install tox tox-gh-actions 38 | - run: tox -e check-docs 39 | 40 | test: 41 | runs-on: ubuntu-latest 42 | continue-on-error: ${{ matrix.experimental }} 43 | strategy: 44 | fail-fast: false 45 | matrix: 46 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] 47 | experimental: [false] 48 | include: 49 | - python-version: "3.12-dev" 50 | experimental: true 51 | steps: 52 | - uses: actions/checkout@v3 53 | with: 54 | fetch-depth: 0 55 | - name: Set up Python ${{ matrix.python-version }} 56 | uses: actions/setup-python@v2 57 | with: 58 | python-version: ${{ matrix.python-version }} 59 | - name: Install dependencies 60 | run: | 61 | python3 -m pip install --upgrade pip setuptools 62 | python3 -m pip install tox tox-gh-actions 63 | - run: tox 64 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import tempfile 4 | from pathlib import Path 5 | from unittest.mock import patch 6 | 7 | import pytest 8 | 9 | 10 | @pytest.fixture 11 | def tempdir_path(): 12 | with tempfile.TemporaryDirectory() as tempdir_path: 13 | yield Path(tempdir_path).resolve() 14 | 15 | 16 | @pytest.fixture 17 | def init_git_and_commit(call_git): 18 | def _init_git_and_commit(path: Path, with_remote=True) -> None: 19 | call_git("git init", cwd=path) 20 | call_git("git add --all .", cwd=path) 21 | call_git('git commit -m "initial"', cwd=path) 22 | if with_remote: 23 | call_git( 24 | "git remote add origin https://gitlab.localhost/nonexisting/repo.git", 25 | cwd=path, 26 | ) 27 | 28 | return _init_git_and_commit 29 | 30 | 31 | @pytest.fixture(scope="session", autouse=True) 32 | def mock_git_env(): 33 | os.environ.pop("SSH_AUTH_SOCK", None) 34 | with patch.dict( 35 | os.environ, 36 | { 37 | "GIT_CONFIG_NOSYSTEM": "1", 38 | "GIT_CONFIG_GLOBAL": "", 39 | "GIT_CONFIG_SYSTEM": "", 40 | }, 41 | ): 42 | yield 43 | 44 | 45 | @pytest.fixture 46 | def call_git(): 47 | def _call_git(shell_cmd: str, cwd: Path) -> str: 48 | env = { 49 | "GIT_AUTHOR_EMAIL": "pytest@scaraplate", 50 | "GIT_AUTHOR_NAME": "tests_scaraplate", 51 | "GIT_COMMITTER_EMAIL": "pytest@scaraplate", 52 | "GIT_COMMITTER_NAME": "tests_scaraplate", 53 | "PATH": os.getenv("PATH", os.defpath), 54 | } 55 | out = subprocess.run( 56 | shell_cmd, 57 | shell=True, 58 | check=True, 59 | cwd=cwd, 60 | env=env, 61 | timeout=5, 62 | stdout=subprocess.PIPE, 63 | ) 64 | stdout = out.stdout.decode().strip() 65 | print(stdout) 66 | return stdout 67 | 68 | return _call_git 69 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | import os 5 | import subprocess 6 | 7 | from setuptools import setup 8 | from setuptools.command.sdist import sdist as sdist_orig 9 | 10 | ROOT = os.path.dirname(__file__) 11 | VERSION = os.path.join(ROOT, "VERSION") 12 | 13 | 14 | def main(): 15 | return setup( 16 | cmdclass={ 17 | "sdist": sdist, 18 | }, 19 | version=project_version(), 20 | ) 21 | 22 | 23 | def project_version(): 24 | version = None 25 | 26 | if not version: 27 | try: 28 | output = ( 29 | subprocess.check_output( 30 | ["git", "describe", "--tags", "--always"], 31 | stderr=open(os.devnull, "wb"), 32 | ) 33 | .strip() 34 | .decode() 35 | ) 36 | except (FileNotFoundError, subprocess.CalledProcessError): 37 | # Can't read the tag. That's probably project at initial stage, 38 | # or git is not available at all or something else. 39 | pass 40 | else: 41 | try: 42 | base, distance, commit_hash = output.split("-") 43 | except ValueError: 44 | # We're on release tag. 45 | version = output 46 | else: 47 | # Reformat git describe for PEP-440 48 | version = "{}.{}+{}".format(base, distance, commit_hash) 49 | 50 | if not version and os.path.exists(VERSION): 51 | with open(VERSION) as verfile: 52 | version = verfile.read().strip() 53 | 54 | if not version: 55 | raise RuntimeError("cannot detect project version") 56 | 57 | return version 58 | 59 | 60 | class sdist(sdist_orig): 61 | def run(self): 62 | # In case when user didn't eventually run `make version` ensure that 63 | # VERSION file will be included in source distribution. 64 | version = project_version() 65 | with open(VERSION, "w") as verfile: 66 | verfile.write(version) 67 | sdist_orig.run(self) 68 | 69 | 70 | if __name__ == "__main__": 71 | main() 72 | -------------------------------------------------------------------------------- /tests/strategies/test_detect_newline.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | from unittest.mock import sentinel 3 | 4 | import pytest 5 | 6 | from scaraplate.strategies import detect_newline 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "file_contents, expected_newline", 11 | [ 12 | # Typical newlines: 13 | (BytesIO(b"a\na\n"), b"\n"), 14 | (BytesIO(b"a\r\na\r\n"), b"\r\n"), 15 | (BytesIO(b"a\ra\r"), b"\r"), 16 | # Mixed newlines: 17 | (BytesIO(b"a\r\na\na"), b"\r\n"), 18 | # Multiple newlines in a row: 19 | (BytesIO(b"a\n\n\na"), b"\n"), 20 | (BytesIO(b"\n\n\n"), b"\n"), 21 | (BytesIO(b"\r\n\r\n"), b"\r\n"), 22 | (BytesIO(b"\r\n"), b"\r\n"), 23 | # No newlines: 24 | (BytesIO(b"a"), sentinel.default), 25 | (BytesIO(b""), sentinel.default), 26 | (None, sentinel.default), 27 | ], 28 | ) 29 | def test_detect_newline_single_file(file_contents, expected_newline): 30 | if file_contents is not None: 31 | assert file_contents.tell() == 0 32 | 33 | assert expected_newline == detect_newline(file_contents, default=sentinel.default) 34 | 35 | if file_contents is not None: 36 | assert file_contents.tell() == 0 37 | 38 | 39 | @pytest.mark.parametrize( 40 | "file_contents1, file_contents2, expected_newline", 41 | [ 42 | # One file is missing: 43 | (None, BytesIO(b"a\n"), b"\n"), 44 | (BytesIO(b"a\n"), None, b"\n"), 45 | # Prefer first file having a newline: 46 | (BytesIO(b"a\n"), BytesIO(b"a\r"), b"\n"), 47 | (BytesIO(b"a"), BytesIO(b"a\r"), b"\r"), 48 | # Fallback to default if no file contains a newline: 49 | (BytesIO(b"a"), BytesIO(b"a"), sentinel.default), 50 | ], 51 | ) 52 | def test_detect_newline_multiple_files( 53 | file_contents1, file_contents2, expected_newline 54 | ): 55 | if file_contents1 is not None: 56 | assert file_contents1.tell() == 0 57 | if file_contents2 is not None: 58 | assert file_contents2.tell() == 0 59 | 60 | assert expected_newline == detect_newline( 61 | file_contents1, file_contents2, default=sentinel.default 62 | ) 63 | 64 | if file_contents1 is not None: 65 | assert file_contents1.tell() == 0 66 | if file_contents2 is not None: 67 | assert file_contents2.tell() == 0 68 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | CHANGES 2 | ======= 3 | 4 | 0.5 5 | --- 6 | 2023-07-23 7 | 8 | * Fix windows path handling in rollup (#22). 9 | Contributed by john-zielke-snkeos. 10 | 11 | 12 | 0.4 13 | --- 14 | 2022-11-12 15 | 16 | Packaging changes: 17 | 18 | * Drop support for Python 3.6 19 | * Add support for Python 3.9, 3.10, 3.11 20 | * Add support for click>=8, jinja2 3, PyYAML>=6 21 | * Add support for cookiecutter>=2.0.1 (see notes below) 22 | 23 | Cookiecutter 2.0.1: 24 | 25 | In 2.0.1 the `cookiecutter` jinja2 variable has been extended with a new 26 | `_output_dir` key. In scaraplate this is some random dir in a temp space, 27 | so having it in the template context is unwanted, because it would cause 28 | the target project to be updated with the random tempdir on each rollup. 29 | 30 | So in order to support cookiecutter>=2.0.1 you need to make a change in 31 | your scaraplate template, where you write your cookiecutter context. 32 | Suppose you have the following in your `.scaraplate.conf`: 33 | 34 | [cookiecutter_context] 35 | {%- for key, value in cookiecutter.items()|sort %} 36 | {{ key }} = {{ value }} 37 | {%- endfor %} 38 | 39 | Then you need to add an exclusion for the `_output_dir` var, like this: 40 | 41 | [cookiecutter_context] 42 | {%- for key, value in cookiecutter.items()|sort %} 43 | {%- if key not in ('_output_dir',) %} 44 | {{ key }} = {{ value }} 45 | {%- endif %} 46 | {%- endfor %} 47 | 48 | 49 | 0.3 50 | --- 51 | 2020-05-11 52 | 53 | Packaging changes: 54 | 55 | * Add support for PyYAML 5 56 | * Add support for marshmallow 3 57 | 58 | 59 | 0.2 60 | --- 61 | 2019-10-22 62 | 63 | New features: 64 | 65 | * Add RenderedTemplateFileHash strategy (#7). 66 | Contributed by Jonathan Piron. 67 | * Raise an error when the cookiecutter context file is not generated (#9) 68 | * Add support for extra_context to cli (like in `cookiecutter` command) (#10) 69 | * Add jinja2 support to strategies mapping (#13) 70 | * Add automation via Python + built-in support for Git-based projects 71 | and GitLab (#11) 72 | 73 | Behaviour changes: 74 | 75 | * Strategies: detect newline type from the target file and preserve it (#12) 76 | 77 | Packaging changes: 78 | 79 | * Add ``jinja2`` requirement (#13) 80 | * Add ``setuptools`` requirement (for ``pkg_resources`` package) (#11) 81 | * Add ``[gitlab]`` extras for installing ``python-gitlab`` (#11) 82 | * Add support for Python 3.8 83 | 84 | 85 | 0.1 86 | --- 87 | 2019-07-27 88 | 89 | Initial public release. 90 | -------------------------------------------------------------------------------- /tests/automation/test_git_wrapper.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | from scaraplate.automation.git import Git 6 | 7 | 8 | @pytest.fixture 9 | def git_repo(init_git_and_commit, tempdir_path): 10 | repo_path = tempdir_path / "repo" 11 | repo_path.mkdir() 12 | (repo_path / "readme").write_text("hi") 13 | init_git_and_commit(repo_path, with_remote=False) 14 | 15 | return repo_path 16 | 17 | 18 | @pytest.mark.parametrize( 19 | "remote, ref, expected_ref", [("origin", "my-branch", "origin/my-branch")] 20 | ) 21 | def test_git_remote_ref(remote, ref, expected_ref): 22 | git = Git(cwd=Path("."), remote=remote) 23 | assert expected_ref == git.remote_ref(ref) 24 | 25 | 26 | def test_git_commit(git_repo, call_git): 27 | git = Git(git_repo) 28 | assert not git.is_dirty() 29 | 30 | (git_repo / "myfile").write_text("my precious") 31 | assert git.is_dirty() 32 | 33 | message = "my\nmultiline\nmessage" 34 | author = "John Doe " 35 | git.commit_all(message, author=author) 36 | assert not git.is_dirty() 37 | 38 | assert message == call_git("git log -1 --pretty=%B", cwd=git_repo) 39 | assert author == call_git("git log -1 --pretty='%an <%ae>'", cwd=git_repo) 40 | 41 | 42 | def test_git_is_existing_ref(git_repo): 43 | git = Git(git_repo) 44 | assert git.is_existing_ref("master") 45 | assert not git.is_existing_ref("origin/master") 46 | 47 | 48 | def test_git_are_one_commit_diffs_equal(git_repo, call_git): 49 | git = Git(git_repo) 50 | 51 | with pytest.raises(RuntimeError): 52 | # These refs don't exist yet 53 | git.are_one_commit_diffs_equal("t1", "t2") 54 | 55 | # Create branch t1 56 | git.checkout_branch("t1") 57 | (git_repo / "common").write_text("same") 58 | git.commit_all("m1") 59 | 60 | # Move master 1 commit ahead 61 | call_git("git checkout master", cwd=git_repo) 62 | (git_repo / "some_file").write_text("i'ma sta") 63 | call_git("git add .", cwd=git_repo) 64 | call_git("git commit -m next", cwd=git_repo) 65 | 66 | # Create branch t2 67 | git.checkout_branch("t2") 68 | (git_repo / "common").write_text("same") 69 | git.commit_all("m2") 70 | 71 | # /-() << t1 (/common == "same") 72 | # / 73 | # -> () -> () -> () << t2 (/common == "same") 74 | # \ 75 | # \ << master (/some_file == "i'ma sta") 76 | assert git.are_one_commit_diffs_equal("t1", "t2") 77 | assert git.are_one_commit_diffs_equal("master", "master") 78 | assert not git.are_one_commit_diffs_equal("master", "t2") 79 | -------------------------------------------------------------------------------- /src/scaraplate/__main__.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from typing import Mapping 3 | 4 | import click 5 | 6 | from scaraplate.rollup import rollup as _rollup 7 | 8 | 9 | def validate_extra_context(ctx, param, value): 10 | """Validate extra context.""" 11 | # vendored from https://github.com/cookiecutter/cookiecutter/blob/673f773bfaf591b056d977c4ab82b45d90dce11e/cookiecutter/cli.py#L35-L46 # noqa 12 | for s in value: 13 | if "=" not in s: 14 | raise click.BadParameter( 15 | "EXTRA_CONTEXT should contain items of the form key=value; " 16 | "'{}' doesn't match that form".format(s) 17 | ) 18 | 19 | # Convert tuple -- e.g.: (u'program_name=foobar', u'startsecs=66') 20 | # to dict -- e.g.: {'program_name': 'foobar', 'startsecs': '66'} 21 | return collections.OrderedDict(s.split("=", 1) for s in value) or None 22 | 23 | 24 | @click.group() 25 | @click.version_option() 26 | def main(): 27 | pass 28 | 29 | 30 | @main.command() 31 | @click.argument("template_dir", type=click.Path(exists=True, file_okay=False)) 32 | @click.argument("target_project_dir", type=click.Path(file_okay=False)) 33 | @click.argument("extra_context", nargs=-1, callback=validate_extra_context) 34 | @click.option( 35 | "--no-input", 36 | is_flag=True, 37 | help=( 38 | "Do not prompt for missing data. Cookiecutter will use " 39 | "the defaults provided by the `cookiecutter.json` " 40 | "in the TEMPLATE_DIR." 41 | ), 42 | ) 43 | def rollup( 44 | *, 45 | template_dir: str, 46 | target_project_dir: str, 47 | no_input: bool, 48 | extra_context: Mapping[str, str], 49 | ) -> None: 50 | """Rollup (apply) the cookiecutter template. 51 | 52 | The template from TEMPLATE_DIR is applied on top of TARGET_PROJECT_DIR. 53 | 54 | TEMPLATE_DIR must be a local path to the location of a git repo with 55 | the cookiecutter template to apply. It must contain `scaraplate.yaml` 56 | file in the root. The TEMPLATE_DIR must be in a git repo, because 57 | some strategies require the commit hash at HEAD and the git remote URL. 58 | 59 | TARGET_PROJECT_DIR should point to the directory where the template 60 | should be applied. Might not exist -- it will be created then. 61 | For monorepos this should point to the subproject inside the monorepo. 62 | 63 | EXTRA_CONTEXT is a list of `key=value` pairs, just like in 64 | `cookiecutter` command. 65 | """ 66 | _rollup( 67 | template_dir=template_dir, 68 | target_project_dir=target_project_dir, 69 | no_input=no_input, 70 | extra_context=extra_context, 71 | ) 72 | 73 | 74 | if __name__ == "__main__": 75 | main(prog_name="scaraplate") 76 | -------------------------------------------------------------------------------- /src/scaraplate/template.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | from pathlib import Path 4 | from typing import Dict, NamedTuple, Optional, Sequence, Type 5 | 6 | from scaraplate.gitremotes import GitRemote, make_git_remote 7 | 8 | 9 | class TemplateMeta(NamedTuple): 10 | """Metadata of the template's git repo status.""" 11 | 12 | git_project_url: str 13 | commit_hash: str 14 | commit_url: str 15 | is_git_dirty: bool 16 | head_ref: Optional[str] 17 | 18 | 19 | def get_template_meta_from_git( 20 | template_path: Path, *, git_remote_type: Optional[Type[GitRemote]] = None 21 | ) -> TemplateMeta: 22 | remote_url = _git_remote_origin(template_path) 23 | commit_hash = _git_head_commit_hash(template_path) 24 | 25 | git_remote = make_git_remote(remote_url, git_remote_type=git_remote_type) 26 | 27 | return TemplateMeta( 28 | git_project_url=git_remote.project_url(), 29 | commit_hash=commit_hash, 30 | commit_url=git_remote.commit_url(commit_hash), 31 | is_git_dirty=_is_git_dirty(template_path), 32 | head_ref=_git_resolve_head(template_path), 33 | ) 34 | 35 | 36 | def _git_head_commit_hash(cwd: Path) -> str: 37 | return _call_git(["rev-parse", "--verify", "HEAD"], cwd) 38 | 39 | 40 | def _is_git_dirty(cwd: Path) -> bool: 41 | return bool(_call_git(["status", "--porcelain"], cwd)) 42 | 43 | 44 | def _git_resolve_head(cwd: Path) -> Optional[str]: 45 | ref = _call_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd) 46 | if ref == "HEAD": 47 | # Detached HEAD (some specific commit has been checked out). 48 | return None 49 | return ref 50 | 51 | 52 | def _git_remote_origin(cwd: Path) -> str: 53 | # Would raise if there's no remote called `origin`. 54 | return _call_git(["config", "--get", "remote.origin.url"], cwd) 55 | 56 | 57 | def _call_git( 58 | command: Sequence[str], cwd: Path, *, env: Optional[Dict[str, str]] = None 59 | ) -> str: 60 | env_combined = dict(os.environ) 61 | if env: 62 | env_combined.update(env) 63 | 64 | try: 65 | out = subprocess.run( 66 | ["git", *command], 67 | cwd=cwd, 68 | stdout=subprocess.PIPE, 69 | stderr=subprocess.PIPE, 70 | check=True, 71 | env=env_combined, 72 | ) 73 | except subprocess.CalledProcessError as e: 74 | raise RuntimeError( 75 | f"{command} command failed in the directory " 76 | f"'{cwd}'. " 77 | f"Ensure that it is a valid git repo." 78 | f"\n" 79 | f"stdout:\n" 80 | f"{e.stdout.decode()}\n" 81 | f"stderr:\n" 82 | f"{e.stderr.decode()}\n" 83 | ) 84 | else: 85 | return out.stdout.decode().strip() 86 | -------------------------------------------------------------------------------- /tests/test_cookecutter.py: -------------------------------------------------------------------------------- 1 | from contextlib import ExitStack 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from scaraplate.cookiecutter import ScaraplateConf, SetupCfg 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "contents, expected_context", 11 | [ 12 | (None, None), 13 | ("", {}), 14 | ( 15 | """ 16 | [metadata] 17 | name = holywarrior 18 | """, 19 | {}, 20 | ), 21 | ( 22 | """ 23 | [tool:cookiecutter_context] 24 | """, 25 | {}, 26 | ), 27 | ( 28 | """ 29 | [metadata] 30 | name = holywarrior 31 | 32 | 33 | [tool:cookiecutter_context] 34 | metadata_author = Usermodel @ Rambler&Co 35 | coverage_fail_under = 90 36 | project_monorepo_name = 37 | """, 38 | { 39 | "metadata_author": "Usermodel @ Rambler&Co", 40 | "coverage_fail_under": "90", 41 | "project_monorepo_name": "", 42 | }, 43 | ), 44 | ], 45 | ) 46 | def test_setupcfg(tempdir_path: Path, contents, expected_context): 47 | if contents is not None: 48 | (tempdir_path / "setup.cfg").write_text(contents) 49 | 50 | with ExitStack() as stack: 51 | if expected_context is None: 52 | stack.enter_context(pytest.raises(FileNotFoundError)) 53 | 54 | cookiecutter_context = SetupCfg(tempdir_path) 55 | assert expected_context == cookiecutter_context.read() 56 | 57 | 58 | @pytest.mark.parametrize( 59 | "contents, expected_context", 60 | [ 61 | (None, None), 62 | ("", {}), 63 | ( 64 | """ 65 | [foreignsection] 66 | name = holywarrior 67 | """, 68 | {}, 69 | ), 70 | ( 71 | """ 72 | [cookiecutter_context] 73 | """, 74 | {}, 75 | ), 76 | ( 77 | """ 78 | [cookiecutter_context] 79 | metadata_author = Usermodel @ Rambler&Co 80 | coverage_fail_under = 90 81 | project_monorepo_name = 82 | """, 83 | { 84 | "metadata_author": "Usermodel @ Rambler&Co", 85 | "coverage_fail_under": "90", 86 | "project_monorepo_name": "", 87 | }, 88 | ), 89 | ], 90 | ) 91 | def test_scaraplate_conf(tempdir_path: Path, contents, expected_context): 92 | if contents is not None: 93 | (tempdir_path / ".scaraplate.conf").write_text(contents) 94 | 95 | with ExitStack() as stack: 96 | if expected_context is None: 97 | stack.enter_context(pytest.raises(FileNotFoundError)) 98 | 99 | cookiecutter_context = ScaraplateConf(tempdir_path) 100 | assert expected_context == cookiecutter_context.read() 101 | -------------------------------------------------------------------------------- /tests/test_template.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from scaraplate.template import ( 4 | _git_head_commit_hash, 5 | _git_remote_origin, 6 | _git_resolve_head, 7 | _is_git_dirty, 8 | ) 9 | 10 | 11 | def test_git_head_commit_hash_valid(init_git_and_commit, tempdir_path): 12 | repo_path = tempdir_path / "repo" 13 | repo_path.mkdir() 14 | (repo_path / "myfile").write_text("hi") 15 | init_git_and_commit(repo_path) 16 | 17 | assert 40 == len(_git_head_commit_hash(repo_path)) 18 | 19 | 20 | def test_git_head_commit_hash_invalid(tempdir_path): 21 | # tempdir_path is not under a git repo. 22 | with pytest.raises(RuntimeError): 23 | _git_head_commit_hash(tempdir_path) 24 | 25 | 26 | def test_git_head_ref_valid(init_git_and_commit, tempdir_path): 27 | repo_path = tempdir_path / "repo" 28 | repo_path.mkdir() 29 | (repo_path / "myfile").write_text("hi") 30 | init_git_and_commit(repo_path) 31 | 32 | assert "master" == _git_resolve_head(repo_path) 33 | 34 | 35 | def test_git_head_ref_valid_detached(init_git_and_commit, call_git, tempdir_path): 36 | repo_path = tempdir_path / "repo" 37 | repo_path.mkdir() 38 | (repo_path / "myfile").write_text("hi") 39 | init_git_and_commit(repo_path) 40 | 41 | # Detach the HEAD 42 | commit_hash = _git_head_commit_hash(repo_path) 43 | call_git(f"git checkout {commit_hash}", cwd=repo_path) 44 | 45 | assert _git_resolve_head(repo_path) is None 46 | 47 | 48 | def test_git_head_ref_invalid(tempdir_path): 49 | # tempdir_path is not under a git repo. 50 | with pytest.raises(RuntimeError): 51 | _git_resolve_head(tempdir_path) 52 | 53 | 54 | def test_git_is_dirty(init_git_and_commit, tempdir_path, call_git): 55 | repo_path = tempdir_path / "repo" 56 | repo_path.mkdir() 57 | (repo_path / "myfile").write_text("hi") 58 | init_git_and_commit(repo_path) 59 | 60 | assert not _is_git_dirty(repo_path) 61 | 62 | # Unstaged should be considered dirty 63 | (repo_path / "myfile2").write_text("hi") 64 | assert _is_git_dirty(repo_path) 65 | 66 | # Staged should be considered dirty as well 67 | call_git("git add myfile2", cwd=repo_path) 68 | assert _is_git_dirty(repo_path) 69 | 70 | 71 | def test_git_remote_origin_valid(init_git_and_commit, tempdir_path): 72 | repo_path = tempdir_path / "repo" 73 | repo_path.mkdir() 74 | (repo_path / "myfile").write_text("hi") 75 | init_git_and_commit(repo_path) 76 | 77 | assert _git_remote_origin(repo_path).startswith("https://") 78 | 79 | 80 | def test_git_remote_origin_invalid(init_git_and_commit, tempdir_path): 81 | repo_path = tempdir_path / "repo" 82 | repo_path.mkdir() 83 | (repo_path / "myfile").write_text("hi") 84 | init_git_and_commit(repo_path, with_remote=False) 85 | 86 | with pytest.raises(RuntimeError): 87 | _git_remote_origin(repo_path) 88 | -------------------------------------------------------------------------------- /tests/strategies/test_sorted_unique_lines.yml: -------------------------------------------------------------------------------- 1 | 2 | strategy: SortedUniqueLines 3 | 4 | testcases: 5 | - name: non_existing_target 6 | template: | 7 | zz с юникодом 8 | Py тон 9 | pysomething 10 | # comment 11 | graft 12 | target: null 13 | out: | 14 | # comment 15 | graft 16 | Py тон 17 | pysomething 18 | zz с юникодом 19 | 20 | - name: existing_target 21 | template: | 22 | include b 23 | graft a 24 | target: | 25 | include c 26 | graft a 27 | out: | 28 | graft a 29 | include b 30 | include c 31 | 32 | - name: same_case_a 33 | template: | 34 | env/ 35 | target: | 36 | ENV/ 37 | out: | 38 | ENV/ 39 | env/ 40 | 41 | - name: same_case_b 42 | template: | 43 | ENV/ 44 | target: | 45 | env/ 46 | out: | 47 | ENV/ 48 | env/ 49 | 50 | - name: empty 51 | template: '' 52 | target: null 53 | out: '' 54 | 55 | - name: with_license_header 56 | template: | 57 | # Licensed under blah blah 58 | # A Copyright blah blah 59 | 60 | include b 61 | graft a 62 | target: | 63 | # Licensed under blah blah 64 | # A Copyright 1970 blah blah 65 | 66 | include a 67 | out: | 68 | # Licensed under blah blah 69 | # A Copyright blah blah 70 | 71 | graft a 72 | include a 73 | include b 74 | 75 | - name: with_comments_in_between 76 | template: | 77 | # Licensed under blah blah 78 | # A Copyright blah blah 79 | 80 | include b 81 | # A T Copyright blah blah 82 | graft a 83 | target: | 84 | # Licensed under blah blah 85 | # A Copyright 1970 blah blah 86 | 87 | include a 88 | # A D Copyright blah blah 89 | out: | 90 | # Licensed under blah blah 91 | # A Copyright blah blah 92 | 93 | # A D Copyright blah blah 94 | # A T Copyright blah blah 95 | graft a 96 | include a 97 | include b 98 | 99 | - name: with_custom_comments_header 100 | template: | 101 | .. rst comment 102 | 103 | include b 104 | graft a 105 | target: | 106 | 107 | include a 108 | out: | 109 | .. rst comment 110 | 111 | graft a 112 | include a 113 | include b 114 | config: 115 | comment_pattern: "^ *[.][.] " 116 | 117 | - name: with_invalid_pattern 118 | template: hi 119 | target: hello 120 | out: null 121 | config: 122 | comment_pattern: '(' 123 | raises: marshmallow.exceptions.ValidationError 124 | 125 | - name: with_mixed_newlines 126 | template: "\ 127 | aaa\n\ 128 | ccc\n\ 129 | eee\n" 130 | target: "\ 131 | bbb\r\n\ 132 | ddd\r\n" 133 | out: "\ 134 | aaa\r\n\ 135 | bbb\r\n\ 136 | ccc\r\n\ 137 | ddd\r\n\ 138 | eee\r\n" 139 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [BASIC] 2 | argument-rgx = [a-z_][a-z0-9_]{2,30}$ 3 | attr-rgx = [a-z_][a-z0-9_]{2,30}$ 4 | bad-names = foo,bar,baz,toto,tutu,tata 5 | class-attribute-rgx = ([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 6 | class-rgx = [A-Z_][a-zA-Z0-9]+$ 7 | const-rgx = (([A-Z_][A-Z0-9_]*)|(__.*__))$ 8 | docstring-min-length = -1 9 | function-rgx = [a-z_][a-z0-9_]{2,30}$ 10 | good-names = i,j,k,_,it 11 | include-naming-hint = no 12 | inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$ 13 | method-rgx = [a-z_][a-z0-9_]{2,30}$ 14 | module-rgx = (([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 15 | name-group = 16 | no-docstring-rgx = ^_ 17 | property-classes = abc.abstractproperty 18 | variable-rgx = [a-z_][a-z0-9_]{2,30}$ 19 | 20 | [CLASSES] 21 | defining-attr-methods = __init__,__new__,setUp 22 | exclude-protected = _asdict,_fields,_replace,_source,_make 23 | valid-classmethod-first-arg = cls 24 | valid-metaclass-classmethod-first-arg = mcs 25 | 26 | [DESIGN] 27 | ignored-argument-names = _.* 28 | max-args = 5 29 | max-attributes = 7 30 | max-bool-expr = 5 31 | max-branches = 12 32 | max-locals = 15 33 | max-parents = 7 34 | max-public-methods = 20 35 | max-returns = 6 36 | max-statements = 50 37 | min-public-methods = 2 38 | 39 | [ELIF] 40 | max-nested-blocks = 5 41 | 42 | [FORMAT] 43 | expected-line-ending-format = 44 | ignore-long-lines = ^\s*(# )??$ 45 | indent-after-paren = 4 46 | indent-string = ' ' 47 | max-line-length = 88 48 | max-module-lines = 1000 49 | single-line-if-stmt = no 50 | 51 | [IMPORTS] 52 | analyse-fallback-blocks = no 53 | deprecated-modules = regsub,TERMIOS,Bastion,rexec 54 | ext-import-graph = 55 | import-graph = 56 | int-import-graph = 57 | known-standard-library = 58 | known-third-party = enchant 59 | 60 | [LOGGING] 61 | logging-modules = logging 62 | 63 | [MASTER] 64 | extension-pkg-whitelist = numpy,ujson 65 | ignore = CVS 66 | ignore-patterns = 67 | jobs = 1 68 | load-plugins = 69 | persistent = yes 70 | unsafe-load-any-extension = no 71 | 72 | [MESSAGES CONTROL] 73 | confidence = 74 | disable = I, 75 | 76 | [MISCELLANEOUS] 77 | notes = FIXME,XXX,TODO 78 | 79 | [REPORTS] 80 | evaluation = 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 81 | output-format = text 82 | reports = yes 83 | 84 | [SIMILARITIES] 85 | ignore-comments = yes 86 | ignore-docstrings = yes 87 | ignore-imports = no 88 | min-similarity-lines = 6 89 | 90 | [SPELLING] 91 | spelling-dict = 92 | spelling-ignore-words = 93 | spelling-private-dict-file = 94 | spelling-store-unknown-words = no 95 | 96 | [TYPECHECK] 97 | contextmanager-decorators = contextlib.contextmanager 98 | generated-members = 99 | ignore-mixin-members = yes 100 | ignored-classes = optparse.Values,thread._local,_thread._local 101 | ignored-modules = 102 | 103 | [VARIABLES] 104 | additional-builtins = 105 | callbacks = cb_,_cb 106 | dummy-variables-rgx = (_+[a-zA-Z0-9]*?$)|dummy 107 | init-import = no 108 | redefining-builtins-modules = six.moves,future.builtins 109 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # http://www.sphinx-doc.org/en/master/config 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import subprocess 15 | 16 | 17 | # import sys 18 | # sys.path.insert(0, os.path.abspath('.')) 19 | 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | 24 | def get_metadata_value(property_name): 25 | # Requires python >=3.5 26 | 27 | setup_py_dir = os.path.join(os.path.dirname(__file__), "..") 28 | setup_py_file = os.path.join(setup_py_dir, "setup.py") 29 | 30 | out = subprocess.run( 31 | ["python", setup_py_file, "-q", "--%s" % property_name], 32 | stdout=subprocess.PIPE, 33 | cwd=setup_py_dir, 34 | check=True, 35 | ) 36 | property_value = out.stdout.decode().strip() 37 | return property_value 38 | 39 | 40 | project = get_metadata_value("name") 41 | author = get_metadata_value("author") 42 | 43 | _copyright_year = 2019 44 | copyright = "%s, %s" % (_copyright_year, author) 45 | 46 | # The full version, including alpha/beta/rc tags 47 | release = get_metadata_value("version") 48 | # The short X.Y version 49 | version = release.rsplit(".", 1)[0] # `1.0.16+g40b2401` -> `1.0` 50 | 51 | # -- General configuration --------------------------------------------------- 52 | 53 | # Add any Sphinx extension module names here, as strings. They can be 54 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 55 | # ones. 56 | extensions = [ 57 | "sphinx.ext.autodoc", 58 | "sphinx.ext.intersphinx", 59 | ] 60 | 61 | # The master toctree document. 62 | master_doc = 'index' 63 | 64 | # Add any paths that contain templates here, relative to this directory. 65 | templates_path = ['_templates'] 66 | 67 | # List of patterns, relative to source directory, that match files and 68 | # directories to ignore when looking for source files. 69 | # This pattern also affects html_static_path and html_extra_path. 70 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 71 | 72 | 73 | # -- Options for HTML output ------------------------------------------------- 74 | 75 | # The theme to use for HTML and HTML Help pages. See the documentation for 76 | # a list of builtin themes. 77 | # 78 | html_theme = 'sphinx_rtd_theme' 79 | 80 | # Add any paths that contain custom static files (such as style sheets) here, 81 | # relative to this directory. They are copied after the builtin static files, 82 | # so a file named "default.css" will overwrite the builtin "default.css". 83 | html_static_path = ['_static'] 84 | 85 | 86 | # -- Extension configuration ------------------------------------------------- 87 | 88 | intersphinx_mapping = { 89 | "cookiecutter_rtd": ("https://cookiecutter.readthedocs.io/en/2.2.3/", None), 90 | "python": ("https://docs.python.org/", None), 91 | } 92 | 93 | # https://github.com/sphinx-doc/sphinx/pull/6397 94 | autodoc_typehints = "none" 95 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test = pytest 3 | 4 | [coverage:report] 5 | exclude_lines = 6 | @abc.abstractmethod 7 | @abc.abstractproperty 8 | CancelledError 9 | NotImplementedError 10 | pragma: no cover 11 | __repr__ 12 | __str__ 13 | fail_under = 98 14 | precision = 2 15 | show_missing = True 16 | 17 | [coverage:run] 18 | branch = True 19 | source = 20 | src 21 | tests 22 | 23 | [flake8] 24 | ignore = E203,W503 25 | max-line-length = 88 26 | 27 | [isort] 28 | multi_line_output = 3 29 | profile = black 30 | 31 | [metadata] 32 | author = Usermodel @ Rambler&Co 33 | author_email = um@rambler-co.ru 34 | classifiers = 35 | Development Status :: 4 - Beta 36 | Environment :: Console 37 | Intended Audience :: Developers 38 | License :: OSI Approved :: MIT License 39 | Natural Language :: English 40 | Operating System :: OS Independent 41 | Programming Language :: Python :: 3 :: Only 42 | Topic :: Software Development 43 | description = Scaraplate is a wrapper around cookiecutter which allows to repeatedly rollup project templates onto concrete projects. 44 | license = MIT 45 | license_files = LICENSE 46 | long_description = file: README.md 47 | long_description_content_type = text/markdown 48 | maintainer = Kostya Esmukov 49 | maintainer_email = kostya@esmukov.net 50 | name = scaraplate 51 | project_urls = 52 | Docs = https://scaraplate.readthedocs.io/ 53 | Issue Tracker = https://github.com/rambler-digital-solutions/scaraplate/issues 54 | Source Code = https://github.com/rambler-digital-solutions/scaraplate 55 | url = https://github.com/rambler-digital-solutions/scaraplate 56 | version = 57 | 58 | [mypy] 59 | check_untyped_defs = True 60 | 61 | [mypy-cookiecutter.*] 62 | ignore_missing_imports = True 63 | 64 | [options] 65 | include_package_data = True 66 | install_requires = 67 | click>=6 68 | cookiecutter>=1.6,<3 69 | jinja2>=2.7,<4 70 | marshmallow>=2.15,<4 71 | packaging 72 | pyyaml>=3 73 | setuptools 74 | package_dir = 75 | = src 76 | packages = find: 77 | python_requires = >=3.7 78 | 79 | [options.entry_points] 80 | console_scripts = 81 | scaraplate = scaraplate.__main__:main 82 | 83 | [options.extras_require] 84 | develop = 85 | black==23.3.0 86 | coverage==7.2.7 87 | flake8==5.0.4 88 | isort==5.11.5 89 | mypy==1.4.1 90 | pylint==2.17.4 91 | pytest==7.4.0 92 | sphinx-rtd-theme==1.2.2 93 | sphinx==4.3.2 94 | types-PyYAML 95 | types-setuptools 96 | gitlab = 97 | python-gitlab>=1.6.0,<4 98 | 99 | [options.packages.find] 100 | where = src 101 | 102 | [tool:cookiecutter_context] 103 | _template = new-python-project 104 | coverage_fail_under = 98 105 | metadata_author = Usermodel @ Rambler&Co 106 | metadata_author_email = um@rambler-co.ru 107 | metadata_description = Scaraplate is a wrapper around cookiecutter which allows to repeatedly rollup project templates onto concrete projects. 108 | metadata_long_description = file: README.md 109 | metadata_name = scaraplate 110 | metadata_url = https://github.com/rambler-digital-solutions/scaraplate 111 | mypy_enabled = 1 112 | project_dest = scaraplate 113 | project_monorepo_name = 114 | python_package = scaraplate 115 | 116 | [tool:pytest] 117 | addopts = 118 | -ra 119 | --junitxml=junit.xml 120 | --showlocals 121 | --verbose 122 | --verbose 123 | -------------------------------------------------------------------------------- /tests/strategies/conftest.py: -------------------------------------------------------------------------------- 1 | import io 2 | from contextlib import ExitStack 3 | from typing import Any, BinaryIO, Dict, Optional, Type 4 | 5 | import pytest 6 | import yaml 7 | 8 | from scaraplate import strategies 9 | from scaraplate.config import class_from_str 10 | from scaraplate.template import TemplateMeta 11 | 12 | 13 | def pytest_collect_file(parent, file_path): 14 | # https://docs.pytest.org/en/latest/example/nonpython.html 15 | if file_path.suffix == ".yml" and file_path.name.startswith("test"): 16 | return YamlFile.from_parent(parent, path=file_path) 17 | 18 | 19 | class YamlFile(pytest.File): 20 | def collect(self): 21 | spec = yaml.safe_load(self.path.open()) 22 | strategy = spec["strategy"] 23 | testcases = spec["testcases"] 24 | 25 | for testcase in testcases: 26 | yield YamlItem.from_parent( 27 | self, name=testcase["name"], strategy=strategy, testcase=testcase 28 | ) 29 | 30 | 31 | class YamlItem(pytest.Item): 32 | def __init__(self, *, strategy, testcase, **kwargs): 33 | super().__init__(**kwargs) 34 | self.strategy = strategy 35 | self.testcase = testcase 36 | 37 | def runtest(self): 38 | strategy_type = getattr(strategies, self.strategy) 39 | run_strategy_test( 40 | strategy_type=strategy_type, 41 | template=self.testcase["template"], 42 | target=self.testcase["target"], 43 | is_git_dirty=self.testcase.get("is_git_dirty", False), 44 | out=self.testcase["out"], 45 | config=self.testcase.get("config", {}), 46 | raises=self.testcase.get("raises"), 47 | ) 48 | 49 | def reportinfo(self): 50 | return self.fspath, 0, f"name: {self.name}" 51 | 52 | 53 | def run_strategy_test( 54 | strategy_type: Type[strategies.Strategy], 55 | template: str, 56 | target: Optional[str], 57 | is_git_dirty: bool, 58 | out: Optional[str], 59 | config: Dict[str, Any], 60 | raises: Optional[str], 61 | ): 62 | if target is not None: 63 | target_contents: Optional[BinaryIO] = io.BytesIO(target.encode()) 64 | else: 65 | target_contents = None 66 | 67 | template_contents = io.BytesIO(template.encode()) 68 | 69 | raises_cls: Optional[Type] = class_from_str(raises) if raises is not None else None 70 | 71 | with ExitStack() as stack: 72 | if raises_cls is not None: 73 | assert out is None 74 | stack.enter_context(pytest.raises(raises_cls)) 75 | 76 | strategy = strategy_type( 77 | target_contents=target_contents, 78 | template_contents=template_contents, 79 | template_meta=TemplateMeta( 80 | git_project_url=( 81 | "https://github.com/rambler-digital-solutions/" 82 | "scaraplate-example-template" 83 | ), 84 | commit_hash="1111111111111111111111111111111111111111", 85 | commit_url=( 86 | "https://github.com/rambler-digital-solutions/" 87 | "scaraplate-example-template" 88 | "/commit/1111111111111111111111111111111111111111" 89 | ), 90 | is_git_dirty=is_git_dirty, 91 | head_ref="master", 92 | ), 93 | config=config, 94 | ) 95 | 96 | strategy_out = strategy.apply().read().decode() 97 | assert out == strategy_out 98 | -------------------------------------------------------------------------------- /tests/strategies/test_config_parser_merge_pylintrc.yml: -------------------------------------------------------------------------------- 1 | strategy: ConfigParserMerge 2 | 3 | testcases: 4 | - name: non_existing_target 5 | template: | 6 | [MASTER] 7 | # Specify a configuration file. 8 | #rcfile= 9 | 10 | persistent=yes 11 | optimize-ast=no 12 | target: null 13 | # Comments are removed, keys are sorted 14 | out: | 15 | [MASTER] 16 | optimize-ast = no 17 | persistent = yes 18 | config: &default_config 19 | preserve_sections: [] 20 | preserve_keys: 21 | - sections: ^MASTER$ 22 | keys: ^extension-pkg-whitelist$ 23 | - sections: ^TYPECHECK$ 24 | keys: ^ignored- 25 | 26 | - name: target_with_extraneous_sections 27 | template: | 28 | [MASTER] 29 | # Specify a configuration file. 30 | #rcfile= 31 | persistent=yes 32 | optimize-ast=no 33 | target: | 34 | [MASTER] 35 | #rcfile= 36 | optimize-ast=yes 37 | 38 | [MESSAGES CONTROL] 39 | confidence= 40 | # Extraneous sections from target are removed 41 | out: | 42 | [MASTER] 43 | optimize-ast = no 44 | persistent = yes 45 | config: 46 | <<: *default_config 47 | 48 | - name: target_with_ignored_keys 49 | template: | 50 | [MASTER] 51 | # Specify a configuration file. 52 | #rcfile= 53 | persistent=yes 54 | optimize-ast=no 55 | 56 | [TYPECHECK] 57 | ignored-modules=aaa,bbb 58 | ignored-classes=aaa,bbb 59 | target: | 60 | [MASTER] 61 | #rcfile= 62 | optimize-ast=yes 63 | [TYPECHECK] 64 | ignored-modules=ccc,bbb 65 | ignored-classes=ccc,bbb 66 | # `ignored-*` keys of the TYPECHECK section are preserved 67 | out: | 68 | [MASTER] 69 | optimize-ast = no 70 | persistent = yes 71 | 72 | [TYPECHECK] 73 | ignored-classes = ccc,bbb 74 | ignored-modules = ccc,bbb 75 | config: 76 | <<: *default_config 77 | 78 | - name: target_containing_extra_ext_whitelist 79 | template: | 80 | [MASTER] 81 | extension-pkg-whitelist=numpy,ujson 82 | target: | 83 | [MASTER] 84 | extension-pkg-whitelist=ujson,lxml 85 | out: | 86 | [MASTER] 87 | extension-pkg-whitelist = ujson,lxml 88 | config: 89 | <<: *default_config 90 | 91 | - name: empty_config_raises 92 | template: | 93 | [MASTER] 94 | extension-pkg-whitelist=numpy,ujson 95 | target: | 96 | [MASTER] 97 | extension-pkg-whitelist=ujson,lxml 98 | out: null 99 | raises: marshmallow.exceptions.ValidationError 100 | 101 | - name: empty_preserve_keys_passes 102 | template: | 103 | [MASTER] 104 | extension-pkg-whitelist=numpy,ujson 105 | target: | 106 | [MASTER] 107 | extension-pkg-whitelist=ujson,lxml 108 | out: | 109 | [MASTER] 110 | extension-pkg-whitelist = numpy,ujson 111 | config: 112 | preserve_sections: [] 113 | preserve_keys: [] 114 | 115 | - name: bad_preserve_keys_item_raises 116 | template: | 117 | [MASTER] 118 | extension-pkg-whitelist=numpy,ujson 119 | target: | 120 | [MASTER] 121 | extension-pkg-whitelist=ujson,lxml 122 | out: null 123 | config: 124 | preserve_sections: [] 125 | preserve_keys: 126 | - keys: ^extension-pkg$ 127 | raises: marshmallow.exceptions.ValidationError 128 | -------------------------------------------------------------------------------- /src/scaraplate/gitremotes.py: -------------------------------------------------------------------------------- 1 | """Scaraplate assumes that the template dir is a git repo. 2 | 3 | Strategies receive a :class:`scaraplate.template.TemplateMeta` instance 4 | which contains URLs to the template's project and the HEAD git commit 5 | on a git remote's web interface (such as GitHub). These URLs might be 6 | rendered in the target files by the strategies. 7 | 8 | Scaraplate has built-in support for some popular git remotes. The remote 9 | is attempted to be detected automatically, but if that fails, it should 10 | be specified manually. 11 | 12 | 13 | Sample ``scaraplate.yaml`` excerpt: 14 | 15 | :: 16 | 17 | git_remote_type: scaraplate.gitremotes.GitHub 18 | 19 | """ 20 | import abc 21 | import re 22 | from typing import Optional, Type 23 | 24 | 25 | def _dot_git_remote_to_https(remote_url: str) -> str: 26 | url = remote_url 27 | # `git@gitlab.com:` -> `https://gitlab.com/` 28 | url = re.sub(r"^[^@]*@([^:]+):", r"https://\1/", url) 29 | url = re.sub(r".git$", "", url) 30 | return url 31 | 32 | 33 | def make_git_remote( 34 | remote: str, *, git_remote_type: Optional[Type["GitRemote"]] = None 35 | ) -> "GitRemote": 36 | if git_remote_type is not None: 37 | return git_remote_type(remote) 38 | 39 | if "gitlab" in remote.lower(): 40 | return GitLab(remote) 41 | elif "github" in remote.lower(): 42 | return GitHub(remote) 43 | elif "bitbucket" in remote.lower(): 44 | return BitBucket(remote) 45 | else: 46 | raise ValueError( 47 | "Unable to automatically determine the GitRemote type. " 48 | "Please set a specific one in the `scaraplate.yaml` config " 49 | "using the `git_remote_type` option." 50 | ) 51 | 52 | 53 | class GitRemote(abc.ABC): 54 | """Base class for a git remote implementation, which generates http 55 | URLs from a git remote (either ssh of http) and a commit hash. 56 | """ 57 | 58 | def __init__(self, remote: str) -> None: 59 | """Init the git remote. 60 | 61 | :param remote: A git remote, either ssh or http(s). 62 | """ 63 | self.remote = remote 64 | 65 | @abc.abstractmethod 66 | def project_url(self) -> str: 67 | """Return a project URL at the given git remote.""" 68 | pass 69 | 70 | @abc.abstractmethod 71 | def commit_url(self, commit_hash: str) -> str: 72 | """Return a commit URL at the given git remote. 73 | 74 | :param commit_hash: Git commit hash. 75 | """ 76 | pass 77 | 78 | 79 | class GitLab(GitRemote): 80 | """GitLab git remote implementation.""" 81 | 82 | def project_url(self) -> str: 83 | return _dot_git_remote_to_https(self.remote) 84 | 85 | def commit_url(self, commit_hash: str) -> str: 86 | project_url = self.project_url() 87 | return f"{project_url.rstrip('/')}/commit/{commit_hash}" 88 | 89 | 90 | class GitHub(GitRemote): 91 | """GitHub git remote implementation.""" 92 | 93 | def project_url(self) -> str: 94 | return _dot_git_remote_to_https(self.remote) 95 | 96 | def commit_url(self, commit_hash: str) -> str: 97 | project_url = self.project_url() 98 | return f"{project_url.rstrip('/')}/commit/{commit_hash}" 99 | 100 | 101 | class BitBucket(GitRemote): 102 | """BitBucket git remote implementation.""" 103 | 104 | def project_url(self) -> str: 105 | return _dot_git_remote_to_https(self.remote) 106 | 107 | def commit_url(self, commit_hash: str) -> str: 108 | project_url = self.project_url() 109 | return f"{project_url.rstrip('/')}/commits/{commit_hash}" 110 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to scaraplate's documentation! 2 | ====================================== 3 | 4 | 5 | :Documentation: https://scaraplate.readthedocs.io/ 6 | :Source Code: https://github.com/rambler-digital-solutions/scaraplate 7 | :Issue Tracker: https://github.com/rambler-digital-solutions/scaraplate/issues 8 | :PyPI: https://pypi.org/project/scaraplate/ 9 | 10 | 11 | .. TODO add logo 12 | 13 | 14 | Introduction 15 | ------------ 16 | 17 | Scaraplate is a wrapper around :doc:`cookiecutter ` 18 | which allows to repeatedly rollup project templates onto concrete projects. 19 | 20 | Cookiecutter is a great tool which allows to create projects from templates. 21 | However, it lacks ability to update the already created projects from updated 22 | templates. Scaraplate provides a solution to this problem. 23 | 24 | To use scaraplate, you would have to add a ``scaraplate.yaml`` file near 25 | the ``cookiecutter.json`` of your :doc:`cookiecutter template `. 26 | Then, to rollup the changes from the updated template, you will need 27 | to simply call this command (which can even be automated!): 28 | 29 | :: 30 | 31 | scaraplate rollup ./path/to/template ./path/to/project --no-input 32 | 33 | This allows to easily (and automatically) sync the projects from the template, 34 | greatly simplifying the unification of the projects' structure. 35 | 36 | CI pipelines, code linting settings, test runners, directory structures, 37 | artifacts building tend to greatly vary between the projects. 38 | Once described in the template which is easy to rollup onto the specific 39 | projects, projects unification becomes a trivial task. Everything can be 40 | defined in the template just once and then regularly be synced onto your 41 | projects, see :doc:`automation`. 42 | 43 | 44 | .. _how_it_works: 45 | 46 | How it works 47 | ------------ 48 | 49 | The ``scaraplate rollup`` command does the following: 50 | 51 | 1. Retrieve cookiecutter template variables from the previous rollup 52 | (see :doc:`template`). 53 | 2. Create a temporary dir, apply `cookiecutter` command with the retrieved 54 | variables to create a new temporary project. 55 | 3. For each file in the temporary project, apply a `strategy` 56 | (see :doc:`strategies`) which merges the file from the temporary project 57 | with the corresponding file in the target project. 58 | 59 | Only the files which exist in the temporary project are touched by 60 | scaraplate in the target project. 61 | 62 | The key component of scaraplate are the `strategies`. 63 | 64 | Note that none of the strategies use git history or any git-like 65 | merging. In fact, scaraplate strategies make no assumptions about 66 | the code versioning system used by the target project. 67 | Instead, the merging between the files is defined solely by `strategies` 68 | which generate the output based on the two files and the settings in 69 | the ``scaraplate.yaml``. 70 | 71 | Scaraplate is quite extensible. Many parts are replaceable with custom 72 | implementations in Python. 73 | 74 | Quickstart 75 | ---------- 76 | 77 | `scaraplate` requires Python 3.7 or newer. 78 | 79 | Installation: 80 | 81 | :: 82 | 83 | pip install scaraplate 84 | 85 | 86 | Scaraplate also requires ``git`` to be installed in the system 87 | (see :doc:`template`). 88 | 89 | To get started with scaraplate, you need to: 90 | 91 | 1. Prepare a template (see :doc:`template` and specifically 92 | :ref:`scaraplate_example_template`). 93 | 2. Roll it up on your projects. 94 | 95 | 96 | Project Name 97 | ------------ 98 | 99 | .. image:: _static/scarab.jpg 100 | :width: 80% 101 | :align: center 102 | :alt: Scarab bug 103 | 104 | The project name is inspired by a bug which rolls up the brown balls 105 | of ... well, stuff, (the template) everywhere (the projects). 106 | 107 | **scara**\ b + tem\ **plate** = **scaraplate** 108 | 109 | 110 | .. toctree:: 111 | :maxdepth: 2 112 | :caption: Contents: 113 | 114 | template 115 | strategies 116 | gitremotes 117 | automation 118 | 119 | 120 | Indices and tables 121 | ================== 122 | 123 | * :ref:`genindex` 124 | * :ref:`modindex` 125 | * :ref:`search` 126 | -------------------------------------------------------------------------------- /src/scaraplate/cookiecutter.py: -------------------------------------------------------------------------------- 1 | """cookiecutter context are the variables specified in ``cookiecutter.json``, 2 | which should be provided to cookiecutter to cut a project from the template. 3 | 4 | The context should be generated by one of the files in the template, 5 | so scaraplate could read these variables and rollup the template automatically 6 | (i.e. without asking for these variables). 7 | 8 | The default context reader is :class:`.ScaraplateConf`, but a custom one 9 | might be specified in ``scaraplate.yaml`` like this: 10 | 11 | :: 12 | 13 | cookiecutter_context_type: scaraplate.cookiecutter.SetupCfg 14 | """ 15 | import abc 16 | from configparser import ConfigParser 17 | from pathlib import Path 18 | from typing import Dict, NewType 19 | 20 | CookieCutterContextDict = NewType("CookieCutterContextDict", Dict[str, str]) 21 | 22 | 23 | def _configparser_from_path(cfg_path: Path) -> ConfigParser: 24 | parser = ConfigParser() 25 | text = cfg_path.read_text() 26 | parser.read_string(text, source=str(cfg_path)) 27 | return parser 28 | 29 | 30 | class CookieCutterContext(abc.ABC): 31 | """The abstract base class for retrieving cookiecutter context from 32 | the target project. 33 | 34 | This class can be extended to provide a custom implementation of 35 | the context reader. 36 | """ 37 | 38 | def __init__(self, target_path: Path) -> None: 39 | """Init the context reader.""" 40 | self.target_path = target_path 41 | 42 | @abc.abstractmethod 43 | def read(self) -> CookieCutterContextDict: 44 | """Retrieve the context. 45 | 46 | If the target file doesn't exist, :class:`FileNotFoundError` 47 | must be raised. 48 | 49 | If the file doesn't contain the context, an empty dict 50 | should be returned. 51 | """ 52 | pass 53 | 54 | 55 | class ScaraplateConf(CookieCutterContext): 56 | """A default context reader which assumes that the cookiecutter 57 | template contains the following file named ``.scaraplate.conf`` 58 | in the root of the project dir: 59 | 60 | :: 61 | 62 | [cookiecutter_context] 63 | {%- for key, value in cookiecutter.items()|sort %} 64 | {%- if key not in ('_output_dir',) %} 65 | {{ key }} = {{ value }} 66 | {%- endif %} 67 | {%- endfor %} 68 | 69 | Cookiecutter context would be rendered in the target project by this 70 | file, and this class is able to retrieve that context from it. 71 | """ 72 | 73 | section_name = "cookiecutter_context" 74 | 75 | def __init__(self, target_path: Path) -> None: 76 | super().__init__(target_path) 77 | self.scaraplate_conf = target_path / ".scaraplate.conf" 78 | 79 | def read(self) -> CookieCutterContextDict: 80 | configparser = _configparser_from_path(self.scaraplate_conf) 81 | context_configparser = dict(configparser).get(self.section_name) 82 | context: Dict[str, str] = dict(context_configparser or {}) 83 | return CookieCutterContextDict(context) 84 | 85 | def __str__(self): 86 | return f"{self.scaraplate_conf}" 87 | 88 | 89 | class SetupCfg(CookieCutterContext): 90 | """A context reader which retrieves the cookiecutter context from 91 | a section in ``setup.cfg`` file. 92 | 93 | The ``setup.cfg`` file must be in the cookiecutter template and must 94 | contain the following section: 95 | 96 | :: 97 | 98 | [tool:cookiecutter_context] 99 | {%- for key, value in cookiecutter.items()|sort %} 100 | {%- if key not in ('_output_dir',) %} 101 | {{ key }} = {{ value }} 102 | {%- endif %} 103 | {%- endfor %} 104 | """ 105 | 106 | section_name = "tool:cookiecutter_context" 107 | 108 | def __init__(self, target_path: Path) -> None: 109 | super().__init__(target_path) 110 | self.setup_cfg = target_path / "setup.cfg" 111 | 112 | def read(self) -> CookieCutterContextDict: 113 | configparser = _configparser_from_path(self.setup_cfg) 114 | context_configparser = dict(configparser).get(self.section_name) 115 | context: Dict[str, str] = dict(context_configparser or {}) 116 | return CookieCutterContextDict(context) 117 | 118 | def __str__(self): 119 | return f"{self.setup_cfg}" 120 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # PN - Project Name 2 | PN := scaraplate 3 | # PN - Project Version 4 | PV := `python setup.py -q --version` 5 | 6 | PYTHON := python3 7 | SHELL := /bin/sh 8 | 9 | LINT_TARGET := setup.py src/ tests/ 10 | MYPY_TARGET := src/${PN} tests/ 11 | 12 | # Create the file below if you need to override the variables above 13 | # or add additional make targets. 14 | -include Makefile.inc 15 | 16 | 17 | .PHONY: all 18 | all: help 19 | 20 | 21 | .PHONY: check 22 | # target: check - Run all checks: linters and tests (with coverage) 23 | check: lint test 24 | @${PYTHON} setup.py check 25 | 26 | 27 | .PHONY: clean 28 | # target: clean - Remove intermediate and generated files 29 | clean: 30 | @${PYTHON} setup.py clean 31 | @find . -type f -name '*.py[co]' -delete 32 | @find . -type d -name '__pycache__' -delete 33 | @rm -rf {build,htmlcov,cover,coverage,dist,.coverage,.hypothesis} 34 | @rm -rf src/*.egg-info 35 | @rm -f VERSION 36 | 37 | 38 | .PHONY: develop 39 | # target: develop - Install package in editable mode with `develop` extras 40 | develop: 41 | @${PYTHON} -m pip install --upgrade pip setuptools wheel 42 | @${PYTHON} -m pip install -e .[develop,gitlab] 43 | 44 | 45 | .PHONY: dist 46 | # target: dist - Build all artifacts 47 | dist: dist-sdist dist-wheel 48 | 49 | 50 | .PHONY: dist-sdist 51 | # target: dist-sdist - Build sdist artifact 52 | dist-sdist: 53 | @${PYTHON} setup.py sdist 54 | 55 | 56 | .PHONY: dist-wheel 57 | # target: dist-wheel - Build wheel artifact 58 | dist-wheel: 59 | @${PYTHON} setup.py bdist_wheel 60 | 61 | 62 | .PHONY: distcheck 63 | # target: distcheck - Verify distributed artifacts 64 | distcheck: distcheck-clean sdist 65 | @mkdir -p dist/$(PN) 66 | @tar -xf dist/$(PN)-$(PV).tar.gz -C dist/$(PN) --strip-components=1 67 | @$(MAKE) -C dist/$(PN) venv 68 | . dist/$(PN)/venv/bin/activate && $(MAKE) -C dist/$(PN) develop 69 | . dist/$(PN)/venv/bin/activate && $(MAKE) -C dist/$(PN) check 70 | @rm -rf dist/$(PN) 71 | 72 | 73 | .PHONY: distcheck-clean 74 | distcheck-clean: 75 | @rm -rf dist/$(PN) 76 | 77 | 78 | .PHONY: format 79 | # target: format - Format the code according to the coding styles 80 | format: format-black format-isort 81 | 82 | 83 | .PHONY: format-black 84 | format-black: 85 | @black ${LINT_TARGET} 86 | 87 | 88 | .PHONY: format-isort 89 | format-isort: 90 | @isort ${LINT_TARGET} 91 | 92 | 93 | .PHONY: help 94 | # target: help - Print this help 95 | help: 96 | @egrep "^# target: " Makefile \ 97 | | sed -e 's/^# target: //g' \ 98 | | sort -h \ 99 | | awk '{printf(" %-16s", $$1); $$1=$$2=""; print "-" $$0}' 100 | 101 | 102 | .PHONY: install 103 | # target: install - Install the project 104 | install: 105 | @pip install . 106 | 107 | 108 | .PHONY: lint 109 | # target: lint - Check source code with linters 110 | lint: lint-isort lint-black lint-flake8 lint-mypy lint-pylint 111 | 112 | 113 | .PHONY: lint-black 114 | lint-black: 115 | @${PYTHON} -m black --check --diff ${LINT_TARGET} 116 | 117 | 118 | .PHONY: lint-flake8 119 | lint-flake8: 120 | @${PYTHON} -m flake8 --statistics ${LINT_TARGET} 121 | 122 | 123 | .PHONY: lint-isort 124 | lint-isort: 125 | @${PYTHON} -m isort --check-only ${LINT_TARGET} 126 | 127 | 128 | .PHONY: lint-mypy 129 | lint-mypy: 130 | @${PYTHON} -m mypy ${MYPY_TARGET} 131 | 132 | 133 | .PHONY: lint-pylint 134 | lint-pylint: 135 | @${PYTHON} -m pylint --rcfile=.pylintrc --errors-only ${LINT_TARGET} 136 | 137 | 138 | .PHONY: purge 139 | # target: purge - Remove all unversioned files and reset working copy 140 | purge: 141 | @git reset --hard HEAD 142 | @git clean -xdff 143 | 144 | 145 | .PHONY: report-coverage 146 | # target: report-coverage - Print coverage report 147 | report-coverage: 148 | @${PYTHON} -m coverage report 149 | 150 | 151 | .PHONY: report-pylint 152 | # target: report-pylint - Generate pylint report 153 | report-pylint: 154 | @${PYTHON} -m pylint ${LINT_TARGET} 155 | 156 | 157 | .PHONY: test 158 | # target: test - Run tests with coverage 159 | test: 160 | @${PYTHON} -m coverage run -m pytest 161 | @${PYTHON} -m coverage report 162 | 163 | 164 | .PHONY: uninstall 165 | # target: uninstall - Uninstall the project 166 | uninstall: 167 | @pip uninstall $(PN) 168 | 169 | 170 | # `venv` target is intentionally not PHONY. 171 | # target: venv - Creates virtual environment 172 | venv: 173 | @${PYTHON} -m venv venv 174 | 175 | 176 | .PHONY: version 177 | # target: version - Generate and print project version in PEP-440 format 178 | version: VERSION 179 | @cat VERSION 180 | VERSION: 181 | @echo ${PV} > $@ 182 | -------------------------------------------------------------------------------- /tests/strategies/test_template_hash.yml: -------------------------------------------------------------------------------- 1 | 2 | strategy: TemplateHash 3 | 4 | testcases: 5 | - name: non_existing_target 6 | template: | 7 | from template! 8 | target: null 9 | out: | 10 | from template! 11 | 12 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 13 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 14 | 15 | - name: existing_target_without_hash 16 | template: | 17 | from template! 18 | target: | 19 | from target! 20 | out: | 21 | from template! 22 | 23 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 24 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 25 | 26 | - name: existing_target_with_current_hash 27 | template: | 28 | from template! 29 | target: | 30 | from target! 31 | 32 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 33 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 34 | out: | 35 | from target! 36 | 37 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 38 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 39 | 40 | - name: existing_target_with_different_hash 41 | template: | 42 | from template! 43 | target: | 44 | from target! 45 | 46 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 47 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/0000000000000000000000000000000000000000 48 | out: | 49 | from template! 50 | 51 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 52 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 53 | 54 | - name: existing_target_with_current_hash_dirty 55 | template: | 56 | from template! 57 | target: | 58 | from target! 59 | 60 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 61 | # From (dirty) https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 62 | is_git_dirty: True 63 | out: | 64 | from template! 65 | 66 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 67 | # From (dirty) https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 68 | 69 | - name: non_existing_target_python 70 | template: | 71 | from template! 72 | target: null 73 | out: | 74 | from template! 75 | 76 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 77 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 # noqa 78 | config: 79 | line_comment_start: '#' 80 | max_line_length: 87 81 | max_line_linter_ignore_mark: ' # noqa' 82 | 83 | - name: non_existing_target_groovy 84 | template: | 85 | from template! 86 | target: null 87 | out: | 88 | from template! 89 | 90 | // Generated by https://github.com/rambler-digital-solutions/scaraplate 91 | // From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 92 | config: 93 | line_comment_start: '//' 94 | 95 | - name: existing_target_without_hash_with_mixed_newlines 96 | template: "\ 97 | from template!\r" 98 | target: " 99 | from target!\r\n" 100 | out: "\ 101 | from template!\r\n\ 102 | \r\n\ 103 | # Generated by https://github.com/rambler-digital-solutions/scaraplate\r\n\ 104 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111\r\n" 105 | 106 | - name: existing_target_without_hash_and_template_without_newlines 107 | template: "\ 108 | from template!" 109 | target: " 110 | from target!\r\n" 111 | out: "\ 112 | from template!\r\n\ 113 | \r\n\ 114 | # Generated by https://github.com/rambler-digital-solutions/scaraplate\r\n\ 115 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111\r\n" 116 | -------------------------------------------------------------------------------- /src/scaraplate/automation/base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | from contextlib import ExitStack 4 | from pathlib import Path 5 | from typing import ContextManager, Mapping, Optional 6 | 7 | from scaraplate.rollup import rollup 8 | from scaraplate.template import TemplateMeta 9 | 10 | logger = logging.getLogger("scaraplate") 11 | 12 | __all__ = ("TemplateVCS", "ProjectVCS", "automatic_rollup") 13 | 14 | 15 | def automatic_rollup( 16 | *, 17 | template_vcs_ctx: ContextManager["TemplateVCS"], 18 | project_vcs_ctx: ContextManager["ProjectVCS"], 19 | extra_context: Optional[Mapping[str, str]] = None 20 | ) -> None: 21 | """The main function of the automated rollup implementation. 22 | 23 | This function accepts two context managers, which should return 24 | two classes: :class:`.TemplateVCS` and :class:`.ProjectVCS`, 25 | which represent the cloned template and target project 26 | correspondingly. 27 | 28 | The context managers should prepare the repos, e.g. they should 29 | create a temporary directory, clone a repo there, and produce 30 | a :class:`.TemplateVCS` or :class:`.ProjectVCS` class instance. 31 | 32 | This function then applies scaraplate rollup of the template 33 | to the target project in :ref:`no-input mode `. 34 | If the target project contains any changes (as reported by 35 | :meth:`.ProjectVCS.is_dirty`), they will be committed by calling 36 | :meth:`.ProjectVCS.commit_changes`. 37 | 38 | .. versionadded:: 0.2 39 | """ 40 | 41 | with ExitStack() as stack: 42 | # Clone target project and template 43 | project_vcs = stack.enter_context(project_vcs_ctx) 44 | template_vcs = stack.enter_context(template_vcs_ctx) 45 | 46 | rollup( 47 | template_dir=template_vcs.dest_path, 48 | target_project_dir=project_vcs.dest_path, 49 | no_input=True, 50 | extra_context=extra_context, 51 | ) 52 | 53 | if not project_vcs.is_dirty(): 54 | logger.info( 55 | "scaraplate rollup didn't change anything -- the project " 56 | "is in sync with template" 57 | ) 58 | return 59 | 60 | logger.info("scaraplate rollup produced some changes -- committing them...") 61 | project_vcs.commit_changes(template_vcs.template_meta) 62 | logger.info("scaraplate changes have been committed successfully") 63 | 64 | 65 | class TemplateVCS(abc.ABC): 66 | """A base class representing a template retrieved from a VCS 67 | (probably residing in a temporary directory). 68 | 69 | The resulting directory with template must be within a git repository, 70 | see :doc:`template` for details. But it doesn't mean that it must 71 | be retrieved from git. Template might be retrieved from anywhere, 72 | it just has to be in git at the end. That git repo will be used 73 | to fill the :class:`.TemplateMeta` structure. 74 | """ 75 | 76 | @property 77 | @abc.abstractmethod 78 | def dest_path(self) -> Path: 79 | """Path to the root directory of the template.""" 80 | pass 81 | 82 | @property 83 | @abc.abstractmethod 84 | def template_meta(self) -> TemplateMeta: 85 | """:class:`.TemplateMeta` filled using the template's git repo.""" 86 | pass 87 | 88 | 89 | class ProjectVCS(abc.ABC): 90 | """A base class representing a project retrieved from a VCS 91 | (probably residing in a temporary directory). 92 | 93 | The project might use any VCS, at this point there're no assumptions 94 | made by scaraplate about the VCS. 95 | """ 96 | 97 | @property 98 | @abc.abstractmethod 99 | def dest_path(self) -> Path: 100 | """Path to the root directory of the project.""" 101 | pass 102 | 103 | @abc.abstractmethod 104 | def is_dirty(self) -> bool: 105 | """Tell whether the project has any changes not committed 106 | to the VCS.""" 107 | pass 108 | 109 | @abc.abstractmethod 110 | def commit_changes(self, template_meta: TemplateMeta) -> None: 111 | """Commit the changes made to the project. This method is 112 | responsible for delivering the changes back to the place 113 | the project was retrieved from. For example, if the project 114 | is using ``git`` and it was cloned to a temporary directory, 115 | then this method should commit the changes and push them back 116 | to git remote. 117 | 118 | This method will be called only if :meth:`.ProjectVCS.is_dirty` 119 | has returned True. 120 | """ 121 | pass 122 | -------------------------------------------------------------------------------- /src/scaraplate/config.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | import importlib 3 | from pathlib import Path 4 | from typing import Any, Dict, Mapping, NamedTuple, Optional, Type, Union, cast 5 | 6 | import jinja2 7 | import yaml 8 | 9 | from .cookiecutter import CookieCutterContext, CookieCutterContextDict, ScaraplateConf 10 | from .gitremotes import GitRemote 11 | from .strategies import Strategy 12 | 13 | env = jinja2.Environment(undefined=jinja2.StrictUndefined) 14 | 15 | 16 | class StrategyNode(NamedTuple): 17 | strategy: Type[Strategy] 18 | config: Dict[str, Any] 19 | 20 | 21 | class ScaraplateYamlOptions(NamedTuple): 22 | git_remote_type: Optional[Type[GitRemote]] 23 | cookiecutter_context_type: Type[CookieCutterContext] 24 | 25 | 26 | class ScaraplateYamlStrategies(NamedTuple): 27 | default_strategy: StrategyNode 28 | strategies_mapping: Mapping[str, StrategyNode] 29 | 30 | 31 | def get_scaraplate_yaml_options(template_path: Path) -> ScaraplateYamlOptions: 32 | config = yaml.safe_load((template_path / "scaraplate.yaml").read_text()) 33 | 34 | git_remote_type_name = config.get("git_remote_type") 35 | git_remote_type = ( 36 | class_from_str(git_remote_type_name, ensure_subclass=GitRemote) 37 | if git_remote_type_name is not None 38 | else None 39 | ) 40 | assert git_remote_type is None or issubclass(git_remote_type, GitRemote) # mypy 41 | 42 | cookiecutter_context_type_name = config.get("cookiecutter_context_type") 43 | cookiecutter_context_type = ( 44 | class_from_str( 45 | cookiecutter_context_type_name, ensure_subclass=CookieCutterContext 46 | ) 47 | if cookiecutter_context_type_name is not None 48 | else cast(Type[CookieCutterContext], ScaraplateConf) # mypy 49 | ) 50 | assert cookiecutter_context_type is None or issubclass( 51 | cookiecutter_context_type, CookieCutterContext 52 | ) # mypy 53 | 54 | return ScaraplateYamlOptions( 55 | git_remote_type=git_remote_type, 56 | cookiecutter_context_type=cookiecutter_context_type, 57 | ) 58 | 59 | 60 | def get_scaraplate_yaml_strategies( 61 | template_path: Path, cookiecutter_context_dict: CookieCutterContextDict 62 | ) -> ScaraplateYamlStrategies: 63 | config = yaml.safe_load((template_path / "scaraplate.yaml").read_text()) 64 | default_strategy = _parse_strategy_node( 65 | "default_strategy", config["default_strategy"] 66 | ) 67 | 68 | strategies_mapping: Dict[str, StrategyNode] = { 69 | env.from_string(path).render( 70 | cookiecutter=cookiecutter_context_dict 71 | ): _parse_strategy_node(str(path), strategy_node) 72 | for path, strategy_node in config["strategies_mapping"].items() 73 | } 74 | 75 | return ScaraplateYamlStrategies( 76 | default_strategy=default_strategy, strategies_mapping=strategies_mapping 77 | ) 78 | 79 | 80 | def _parse_strategy_node(path: str, raw: Union[str, Dict[str, Any]]) -> StrategyNode: 81 | if isinstance(raw, str): 82 | strategy = raw 83 | config: Dict[str, Any] = {} 84 | elif isinstance(raw, collections.abc.Mapping): 85 | strategy_ = raw.get("strategy") 86 | if not isinstance(strategy_, str): 87 | raise ValueError( 88 | f"Unexpected `strategy` value for {path}: " 89 | f"a string is expected, got {strategy_!r}" 90 | ) 91 | strategy = strategy_ # mypy 92 | 93 | config = raw.get("config", {}) 94 | if not isinstance(config, collections.abc.Mapping): 95 | raise ValueError( 96 | f"Unexpected strategy `config` value for {path}: " 97 | f"a mapping is expected, got {config!r}" 98 | ) 99 | else: 100 | raise ValueError(f"Unexpected strategy value type for {path}: got {raw!r}") 101 | 102 | cls = class_from_str(strategy, ensure_subclass=Strategy) 103 | assert issubclass(cls, Strategy) # mypy 104 | return StrategyNode(strategy=cls, config=config) 105 | 106 | 107 | def class_from_str(ref: str, *, ensure_subclass: Optional[Type] = None) -> Type[object]: 108 | if not isinstance(ref, str) or "." not in ref: 109 | raise ValueError( 110 | f"A Python class reference must look like " 111 | f"`mypackage.mymodule.MyClass`, got {ref!r}" 112 | ) 113 | module_s, cls_s = ref.rsplit(".", 1) 114 | module = importlib.import_module(module_s) 115 | cls = getattr(module, cls_s) 116 | if ensure_subclass is not None and ( 117 | not issubclass(cls, ensure_subclass) or cls is ensure_subclass 118 | ): 119 | raise ValueError(f"`{cls}` is not a subclass of {ensure_subclass}") 120 | return cls 121 | -------------------------------------------------------------------------------- /tests/test_gitremotes.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from scaraplate.gitremotes import BitBucket, GitHub, GitLab, make_git_remote 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "remote, expected_remote", 8 | [ 9 | ("git@gitlab.com:pycqa/flake8.git", GitLab), 10 | ("https://gitlab.com/pycqa/flake8.git", GitLab), 11 | ("https://gitlab.example.org/pycqa/flake8.git", GitLab), 12 | ("git@github.com:geopy/geopy.git", GitHub), 13 | ("https://github.com/geopy/geopy.git", GitHub), 14 | ("https://github.example.org/geopy/geopy.git", GitHub), 15 | ("https://bitbucket.org/someuser/someproject.git", BitBucket), 16 | ("https://git.example.org/pycqa/flake8.git", None), 17 | ], 18 | ) 19 | def test_make_git_remote_detection(remote, expected_remote): 20 | if expected_remote is None: 21 | with pytest.raises(ValueError): 22 | make_git_remote(remote) 23 | else: 24 | assert expected_remote is type(make_git_remote(remote)) # noqa 25 | 26 | 27 | @pytest.mark.parametrize( 28 | "remote, git_remote_type, expected_remote", 29 | [("https://git.example.org/pycqa/flake8.git", GitLab, GitLab)], 30 | ) 31 | def test_make_git_remote_custom(remote, git_remote_type, expected_remote): 32 | assert expected_remote is type( # noqa 33 | make_git_remote(remote, git_remote_type=git_remote_type) 34 | ) 35 | 36 | 37 | @pytest.mark.parametrize( 38 | "remote, expected", 39 | [ 40 | ("git@gitlab.com:pycqa/flake8.git", "https://gitlab.com/pycqa/flake8"), 41 | ("https://gitlab.com/pycqa/flake8.git", "https://gitlab.com/pycqa/flake8"), 42 | ( 43 | "git@gitlab.example.org:pycqa/flake8.git", 44 | "https://gitlab.example.org/pycqa/flake8", 45 | ), 46 | ( 47 | "https://gitlab.example.org/pycqa/flake8.git", 48 | "https://gitlab.example.org/pycqa/flake8", 49 | ), 50 | ], 51 | ) 52 | def test_gitlab_url_from_remote(remote, expected): 53 | assert expected == GitLab(remote).project_url() 54 | 55 | 56 | @pytest.mark.parametrize( 57 | "remote, commit_hash, expected_url", 58 | [ 59 | ( 60 | "https://gitlab.com/pycqa/flake8.git", 61 | "1111111111111111111111111111111111111111", 62 | "https://gitlab.com/pycqa/flake8/commit/" 63 | "1111111111111111111111111111111111111111", 64 | ) 65 | ], 66 | ) 67 | def test_gitlab_commit_url(remote, commit_hash, expected_url): 68 | assert expected_url == GitLab(remote).commit_url(commit_hash) 69 | 70 | 71 | @pytest.mark.parametrize( 72 | "remote, expected", 73 | [ 74 | ("git@github.com:geopy/geopy.git", "https://github.com/geopy/geopy"), 75 | ("https://github.com/geopy/geopy.git", "https://github.com/geopy/geopy"), 76 | ( 77 | "git@github.example.org:geopy/geopy.git", 78 | "https://github.example.org/geopy/geopy", 79 | ), 80 | ( 81 | "https://github.example.org/geopy/geopy.git", 82 | "https://github.example.org/geopy/geopy", 83 | ), 84 | ], 85 | ) 86 | def test_github_url_from_remote(remote, expected): 87 | assert expected == GitHub(remote).project_url() 88 | 89 | 90 | @pytest.mark.parametrize( 91 | "remote, commit_hash, expected_url", 92 | [ 93 | ( 94 | "https://github.com/geopy/geopy.git", 95 | "1111111111111111111111111111111111111111", 96 | "https://github.com/geopy/geopy/commit/" 97 | "1111111111111111111111111111111111111111", 98 | ) 99 | ], 100 | ) 101 | def test_github_commit_url(remote, commit_hash, expected_url): 102 | assert expected_url == GitHub(remote).commit_url(commit_hash) 103 | 104 | 105 | @pytest.mark.parametrize( 106 | "remote, expected", 107 | [ 108 | ( 109 | "git@bitbucket.org:someuser/someproject.git", 110 | "https://bitbucket.org/someuser/someproject", 111 | ), 112 | ( 113 | "https://bitbucket.org/someuser/someproject.git", 114 | "https://bitbucket.org/someuser/someproject", 115 | ), 116 | ], 117 | ) 118 | def test_bitbucket_url_from_remote(remote, expected): 119 | assert expected == BitBucket(remote).project_url() 120 | 121 | 122 | @pytest.mark.parametrize( 123 | "remote, commit_hash, expected_url", 124 | [ 125 | ( 126 | "https://bitbucket.org/someuser/someproject.git", 127 | "1111111111111111111111111111111111111111", 128 | "https://bitbucket.org/someuser/someproject/commits/" 129 | "1111111111111111111111111111111111111111", 130 | ) 131 | ], 132 | ) 133 | def test_bitbucket_commit_url(remote, commit_hash, expected_url): 134 | assert expected_url == BitBucket(remote).commit_url(commit_hash) 135 | -------------------------------------------------------------------------------- /tests/strategies/test_config_parser_merge.yml: -------------------------------------------------------------------------------- 1 | strategy: ConfigParserMerge 2 | 3 | testcases: 4 | 5 | - name: non_existing_target 6 | template: | 7 | [DEFAULT] 8 | key_in_default_section = hello 9 | 10 | [mysection] 11 | # a comment which will be stripped 12 | mykey=hi 13 | target: null 14 | out: | 15 | [DEFAULT] 16 | key_in_default_section = hello 17 | 18 | [mysection] 19 | mykey = hi 20 | config: 21 | preserve_sections: [] 22 | preserve_keys: [] 23 | 24 | - name: existing_target 25 | template: | 26 | [DEFAULT] 27 | key_in_default_section = hello 28 | template_key = 1 29 | 30 | [mysectiona] 31 | mykey=hi 32 | target: | 33 | [DEFAULT] 34 | key_in_default_section = hi 35 | target_key = 1 36 | 37 | [mysectionb] 38 | mykey=hi 39 | out: | 40 | [DEFAULT] 41 | key_in_default_section = hello 42 | template_key = 1 43 | 44 | [mysectiona] 45 | mykey = hi 46 | config: 47 | preserve_sections: [] 48 | preserve_keys: [] 49 | 50 | - name: preserve_sections 51 | template: | 52 | [DEFAULT] 53 | key_in_default_section = hello 54 | template_key = 1 55 | 56 | [mysectiona] 57 | mykey=hi 58 | 59 | [mysectionc] 60 | mykey=hi 61 | mykey2 = hi 62 | 63 | [notmy] 64 | aa = 1 65 | mykey = hi 66 | target: | 67 | [DEFAULT] 68 | key_in_default_section = hi 69 | target_key = 1 70 | 71 | [mysectionb] 72 | mykey=hi 73 | 74 | [mysectionc] 75 | mykey=hello 76 | mykey3 = hi 77 | 78 | [notmy] 79 | bb = 1 80 | mykey = hello 81 | out: | 82 | [DEFAULT] 83 | key_in_default_section = hi 84 | target_key = 1 85 | 86 | [mysectiona] 87 | mykey = hi 88 | 89 | [mysectionb] 90 | mykey = hi 91 | 92 | [mysectionc] 93 | mykey = hello 94 | mykey3 = hi 95 | 96 | [notmy] 97 | bb = 1 98 | mykey = hi 99 | config: 100 | preserve_sections: 101 | - sections: ^DEFAULT$ 102 | - sections: ^mysection 103 | - sections: ^nonexisting$ 104 | - sections: ^notmy$ 105 | excluded_keys: ^mykey 106 | preserve_keys: [] 107 | 108 | - name: preserve_keys 109 | template: | 110 | [DEFAULT] 111 | key_in_default_section = hello 112 | template_key = 1 113 | 114 | [mysectiona] 115 | mykey=hi 116 | 117 | [mysectionc] 118 | mykey=hi 119 | mykey2 = hi 120 | 121 | [notmy] 122 | aa = 1 123 | mykey = hi 124 | target: | 125 | [DEFAULT] 126 | key_in_default_section = hi 127 | target_key = 1 128 | 129 | [mysectionb] 130 | mykey=hi 131 | 132 | [mysectionc] 133 | mykey=hello 134 | mykey3 = hi 135 | 136 | [notmy] 137 | bb = 1 138 | mykey = hello 139 | out: | 140 | [DEFAULT] 141 | key_in_default_section = hi 142 | template_key = 1 143 | 144 | [mysectiona] 145 | mykey = hi 146 | 147 | [mysectionb] 148 | mykey = hi 149 | 150 | [mysectionc] 151 | mykey = hello 152 | mykey2 = hi 153 | mykey3 = hi 154 | 155 | [notmy] 156 | aa = 1 157 | bb = 1 158 | mykey = hi 159 | config: 160 | preserve_sections: [] 161 | preserve_keys: 162 | - sections: ^DEFAULT$ 163 | keys: ^key_in_default_section$ 164 | - sections: ^mysection 165 | keys: ^mykey 166 | - sections: ^notmy$ 167 | keys: ^bb$ 168 | 169 | - name: empty_config_raises 170 | template: | 171 | [mysection] 172 | mykey = hi 173 | target: | 174 | [mysection] 175 | mykey = hi 176 | out: null 177 | raises: marshmallow.exceptions.ValidationError 178 | 179 | - name: minimal_passing_config 180 | template: | 181 | [mysection] 182 | mykey = hi 183 | target: | 184 | [mysection] 185 | mykey = hi 186 | out: | 187 | [mysection] 188 | mykey = hi 189 | config: 190 | preserve_sections: [] 191 | preserve_keys: [] 192 | 193 | - name: bad_preserve_keys_item_raises 194 | template: | 195 | [MASTER] 196 | extension-pkg-whitelist=numpy,ujson 197 | target: | 198 | [MASTER] 199 | extension-pkg-whitelist=ujson,lxml 200 | out: null 201 | config: 202 | preserve_sections: [] 203 | preserve_keys: 204 | - keys: ^extension-pkg$ 205 | raises: marshmallow.exceptions.ValidationError 206 | 207 | - name: with_mixed_newlines 208 | template: "\ 209 | [mysection]\r\ 210 | mykey = template\r" 211 | target: "\ 212 | [mysection]\r\n\ 213 | mykey = target\r\n" 214 | out: "\ 215 | [mysection]\r\n\ 216 | mykey = template\r\n" 217 | config: 218 | preserve_sections: [] 219 | preserve_keys: [] 220 | -------------------------------------------------------------------------------- /tests/strategies/test_rendered_template_file_hash.yml: -------------------------------------------------------------------------------- 1 | 2 | strategy: RenderedTemplateFileHash 3 | 4 | testcases: 5 | - name: non_existing_target 6 | template: | 7 | from template! 8 | target: null 9 | out: | 10 | from template! 11 | 12 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 13 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 14 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 15 | 16 | - name: existing_target_without_hash 17 | template: | 18 | from template! 19 | target: | 20 | from target! 21 | out: | 22 | from template! 23 | 24 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 25 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 26 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 27 | 28 | - name: existing_target_with_current_hash 29 | template: | 30 | from template! 31 | target: | 32 | from target! 33 | 34 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 35 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 36 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 37 | out: | 38 | from target! 39 | 40 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 41 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 42 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 43 | 44 | - name: existing_target_with_current_hash_but_different_commit 45 | template: | 46 | from template! 47 | target: | 48 | from target! 49 | 50 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 51 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 52 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/2222222222222222222222222222222222222222 53 | out: | 54 | from target! 55 | 56 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 57 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 58 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/2222222222222222222222222222222222222222 59 | 60 | - name: existing_target_with_different_hash 61 | template: | 62 | from template! 63 | target: | 64 | from target! 65 | 66 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 67 | # RenderedTemplateFileHash 00000000 68 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 69 | out: | 70 | from template! 71 | 72 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 73 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 74 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 75 | 76 | - name: existing_target_with_different_hash_dirty 77 | template: | 78 | from template! 79 | target: | 80 | from target! 81 | 82 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 83 | # RenderedTemplateFileHash 00000000 84 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 85 | is_git_dirty: True 86 | out: | 87 | from template! 88 | 89 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 90 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 91 | # From (dirty) https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 92 | 93 | - name: non_existing_target_python 94 | template: | 95 | from template! 96 | target: null 97 | out: | 98 | from template! 99 | 100 | # Generated by https://github.com/rambler-digital-solutions/scaraplate 101 | # RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 102 | # From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 # noqa 103 | config: 104 | line_comment_start: '#' 105 | max_line_length: 80 106 | max_line_linter_ignore_mark: ' # noqa' 107 | 108 | - name: non_existing_target_groovy 109 | template: | 110 | from template! 111 | target: null 112 | out: | 113 | from template! 114 | 115 | // Generated by https://github.com/rambler-digital-solutions/scaraplate 116 | // RenderedTemplateFileHash a211c05944823a78a4d63111a1ddfdec 117 | // From https://github.com/rambler-digital-solutions/scaraplate-example-template/commit/1111111111111111111111111111111111111111 118 | config: 119 | line_comment_start: '//' 120 | -------------------------------------------------------------------------------- /docs/automation.rst: -------------------------------------------------------------------------------- 1 | Rollup Automation 2 | ================= 3 | 4 | Once you get comfortable with manual rollups, you might want to set up 5 | regularly executed automated rollups. 6 | 7 | At this moment scaraplate doesn't provide a CLI for that, but there's 8 | a quite extensible Python code which simplifies implementation of custom 9 | scenarios. 10 | 11 | Supported automation scenarios 12 | ------------------------------ 13 | 14 | GitLab Merge Request 15 | ++++++++++++++++++++ 16 | 17 | GitLab integration requires `python-gitlab `_ 18 | package, which can be installed with: 19 | 20 | :: 21 | 22 | pip install 'scaraplate[gitlab]' 23 | 24 | 25 | Sample ``rollup.py`` script: 26 | 27 | :: 28 | 29 | from scaraplate import automatic_rollup 30 | from scaraplate.automation.gitlab import ( 31 | GitLabCloneTemplateVCS, 32 | GitLabMRProjectVCS, 33 | ) 34 | 35 | 36 | automatic_rollup( 37 | template_vcs_ctx=GitLabCloneTemplateVCS.clone( 38 | project_url="https://mygitlab.example.org/myorg/mytemplate", 39 | private_token="your_access_token", 40 | clone_ref="master", 41 | ), 42 | project_vcs_ctx=GitLabMRProjectVCS.clone( 43 | gitlab_url="https://mygitlab.example.org", 44 | full_project_name="myorg/mytargetproject", 45 | private_token="your_access_token", 46 | changes_branch="scheduled-template-update", 47 | clone_ref="master", 48 | ), 49 | ) 50 | 51 | 52 | This script would do the following: 53 | 54 | 1. ``git clone`` the template repo to a tempdir; 55 | 2. ``git clone`` the project repo to a tempdir; 56 | 3. Run ``scaraplate rollup ... --no-input``; 57 | 4. Would do nothing if rollup didn't change anything; otherwise it would 58 | create a commit with the changes, push it to the ``scheduled-template-update`` 59 | branch and open a GitLab Merge Request from this branch. 60 | 61 | If a MR already exists, :class:`~scaraplate.automation.gitlab.GitLabMRProjectVCS` 62 | does the following: 63 | 64 | 1. A one-commit git diff is compared between the already existing MR's branch 65 | and the locally committed branch (in a tempdir). If diffs are equal, 66 | nothing is done. 67 | 2. If diffs are different, the existing MR's branch is removed from the remote, 68 | effectively closing the old MR, and a new branch is pushed, which 69 | is followed by creation of a new MR. 70 | 71 | To have this script run daily, crontab can be used. Assuming that the script 72 | is located at ``/opt/rollup.py`` and the desired time for execution is 9:00, 73 | it might look like this: 74 | 75 | :: 76 | 77 | $ crontab -e 78 | # Add the following line: 79 | 00 9 * * * python3 /opt/rollup.py 80 | 81 | 82 | Git push 83 | ++++++++ 84 | 85 | :class:`~scaraplate.automation.gitlab.GitLabCloneTemplateVCS` and 86 | :class:`~scaraplate.automation.gitlab.GitLabMRProjectVCS` are based off 87 | :class:`~scaraplate.automation.git.GitCloneTemplateVCS` and 88 | :class:`~scaraplate.automation.git.GitCloneProjectVCS` 89 | correspondingly. GitLab classes add GitLab-specific git-clone URL 90 | generation and Merge Request creation. The rest (git clone, commit, push) 91 | is done in the :class:`~scaraplate.automation.git.GitCloneTemplateVCS` 92 | and :class:`~scaraplate.automation.git.GitCloneProjectVCS` classes. 93 | 94 | :class:`~scaraplate.automation.git.GitCloneTemplateVCS` and 95 | :class:`~scaraplate.automation.git.GitCloneProjectVCS` classes work 96 | with any git remote. If you're okay with just pushing a branch with 97 | updates (without opening a Merge Request/Pull Request), then you can 98 | use the following: 99 | 100 | Sample ``rollup.py`` script: 101 | 102 | :: 103 | 104 | from scaraplate import automatic_rollup, GitCloneProjectVCS, GitCloneTemplateVCS 105 | 106 | 107 | automatic_rollup( 108 | template_vcs_ctx=GitCloneTemplateVCS.clone( 109 | clone_url="https://github.com/rambler-digital-solutions/scaraplate-example-template.git", 110 | clone_ref="master", 111 | ), 112 | project_vcs_ctx=GitCloneProjectVCS.clone( 113 | clone_url="https://mygit.example.org/myrepo.git", 114 | clone_ref="master", 115 | changes_branch="scheduled-template-update", 116 | commit_author="scaraplate ", 117 | ), 118 | ) 119 | 120 | 121 | Python API 122 | ---------- 123 | 124 | .. autofunction:: scaraplate.automation.base.automatic_rollup 125 | 126 | .. autoclass:: scaraplate.automation.base.TemplateVCS 127 | :show-inheritance: 128 | :members: 129 | 130 | .. autoclass:: scaraplate.automation.base.ProjectVCS 131 | :show-inheritance: 132 | :members: 133 | 134 | .. autoclass:: scaraplate.automation.git.GitCloneTemplateVCS 135 | :show-inheritance: 136 | :members: clone 137 | 138 | .. autoclass:: scaraplate.automation.git.GitCloneProjectVCS 139 | :show-inheritance: 140 | :members: clone 141 | 142 | .. autoclass:: scaraplate.automation.gitlab.GitLabCloneTemplateVCS 143 | :show-inheritance: 144 | :members: clone 145 | 146 | .. autoclass:: scaraplate.automation.gitlab.GitLabMRProjectVCS 147 | :show-inheritance: 148 | :members: clone 149 | -------------------------------------------------------------------------------- /tests/automation/conftest.py: -------------------------------------------------------------------------------- 1 | import http.server 2 | import socketserver 3 | import threading 4 | from concurrent.futures import Future 5 | from pathlib import Path 6 | from typing import Any, Callable, List, Optional 7 | 8 | import pytest 9 | 10 | 11 | def pytest_configure(config): 12 | config.addinivalue_line( 13 | "markers", "template_with_sense_vars: write cookiecutter context to a file" 14 | ) 15 | 16 | 17 | def convert_git_repo_to_bare(call_git, *, cwd: Path) -> None: 18 | """Git bare repo is a git repo without a working copy. `git clone` 19 | can clone these repos my simply pointing at their location in the local 20 | filesystem. 21 | """ 22 | # https://stackoverflow.com/a/3251126 23 | call_git("git config --bool core.bare true", cwd=cwd) 24 | 25 | 26 | @pytest.fixture 27 | def template_bare_git_repo(request, tempdir_path, init_git_and_commit, call_git): 28 | template_path = tempdir_path / "remote_template" 29 | 30 | cookiecutter_path = template_path / "{{cookiecutter.project_dest}}" 31 | cookiecutter_path.mkdir(parents=True) 32 | if request.node.get_closest_marker("template_with_sense_vars") is not None: 33 | (cookiecutter_path / "sense_vars").write_text("{{ cookiecutter|jsonify }}\n") 34 | (cookiecutter_path / ".scaraplate.conf").write_text( 35 | """[cookiecutter_context] 36 | {%- for key, value in cookiecutter.items()|sort %} 37 | {%- if key not in ('_output_dir',) %} 38 | {{ key }} = {{ value }} 39 | {%- endif %} 40 | {%- endfor %} 41 | """ 42 | ) 43 | (template_path / "cookiecutter.json").write_text( 44 | '{"project_dest": "test", "key1": null, "key2": null}' 45 | ) 46 | (template_path / "scaraplate.yaml").write_text( 47 | """ 48 | git_remote_type: scaraplate.gitremotes.GitLab 49 | default_strategy: scaraplate.strategies.Overwrite 50 | strategies_mapping: {} 51 | """ 52 | ) 53 | init_git_and_commit(template_path, with_remote=False) 54 | call_git("git branch master2", cwd=template_path) 55 | 56 | convert_git_repo_to_bare(call_git, cwd=template_path) 57 | 58 | return template_path 59 | 60 | 61 | @pytest.fixture 62 | def project_bare_git_repo(tempdir_path, init_git_and_commit, call_git): 63 | target_project_path = tempdir_path / "remote_project" 64 | 65 | target_project_path.mkdir(parents=True) 66 | (target_project_path / "readme").write_text("hi") 67 | init_git_and_commit(target_project_path, with_remote=False) 68 | call_git("git branch master2", cwd=target_project_path) 69 | 70 | convert_git_repo_to_bare(call_git, cwd=target_project_path) 71 | 72 | return target_project_path 73 | 74 | 75 | @pytest.fixture 76 | def http_server(): 77 | server_thread = HttpServerThread() 78 | server_thread.start() 79 | yield server_thread 80 | server_thread.stop() 81 | server_thread.join() 82 | 83 | 84 | class HttpServerThread(threading.Thread): 85 | spinup_timeout = 10 86 | 87 | def __init__(self): 88 | self.server_host = "127.0.0.1" 89 | self.server_port = None # randomly selected by OS 90 | 91 | self.http_server = None 92 | self._errors: List[Exception] = [] 93 | self.request_handler: Optional[Callable[..., Any]] = None 94 | 95 | # Future is generic. Pylint is wrong. 96 | # https://github.com/python/typeshed/blob/6e4708ebf3a482aa8d524196712c37c9fd645953/stdlib/3/concurrent/futures/_base.pyi#L26 # noqa 97 | self.socket_created_future: Future[ # pylint: disable=unsubscriptable-object 98 | bool 99 | ] = Future() 100 | 101 | super().__init__() 102 | self.daemon = True 103 | 104 | def set_request_handler( 105 | self, request_handler: Callable[..., Any] 106 | ) -> Callable[..., Any]: 107 | assert self.request_handler is None 108 | self.request_handler = request_handler 109 | return request_handler 110 | 111 | def get_url(self): 112 | assert self.socket_created_future.result(self.spinup_timeout) 113 | return f"http://{self.server_host}:{self.server_port}" 114 | 115 | def run(self): 116 | assert ( 117 | self.http_server is None 118 | ), "This class is not reentrable. Please create a new instance." 119 | 120 | thread = self 121 | 122 | class Server(http.server.BaseHTTPRequestHandler): 123 | def __request_handler(self, method): 124 | if thread.request_handler is None: # pragma: no cover 125 | self.send_error(500, "request_handler is None") 126 | else: 127 | try: 128 | # https://stackoverflow.com/a/20879937 129 | body = self.rfile.read( 130 | int(self.headers.get("content-length", 0)) 131 | ) 132 | ( 133 | code, 134 | headers, 135 | body, 136 | ) = thread.request_handler( 137 | method, self.path, headers=self.headers, body=body 138 | ) 139 | self.send_response(code) 140 | for name, value in headers.items(): 141 | self.send_header(name, value) 142 | self.end_headers() 143 | self.wfile.write(body) 144 | except Exception as e: # pragma: no cover 145 | thread._errors.append(e) 146 | 147 | def do_HEAD(self): # pragma: no cover 148 | self.__request_handler("HEAD") 149 | 150 | def do_GET(self): 151 | self.__request_handler("GET") 152 | 153 | def do_POST(self): # pragma: no cover 154 | self.__request_handler("POST") 155 | 156 | def do_PATCH(self): # pragma: no cover 157 | self.__request_handler("PATCH") 158 | 159 | # ThreadingTCPServer offloads connections to separate threads, so 160 | # the serve_forever loop doesn't block until connection is closed 161 | # (unlike TCPServer). This allows to shutdown the serve_forever loop 162 | # even if there's an open connection. 163 | try: 164 | # TODO: switch to http.server.ThreadingHTTPServer 165 | # when the py36 support would be dropped 166 | self.http_server = socketserver.ThreadingTCPServer( 167 | (self.server_host, 0), Server 168 | ) 169 | 170 | # don't hang if there're some open connections 171 | self.http_server.daemon_threads = True 172 | 173 | self.server_port = self.http_server.server_address[1] 174 | except Exception as e: # pragma: no cover 175 | self.socket_created_future.set_exception(e) 176 | raise 177 | else: 178 | self.socket_created_future.set_result(True) 179 | 180 | self.http_server.serve_forever() 181 | 182 | def stop(self): 183 | assert self.http_server is not None 184 | self.http_server.shutdown() # stop serve_forever() 185 | self.http_server.server_close() 186 | 187 | # assert not self._errors 188 | # ^^^ gives unreadable assertions in pytest 189 | # This is better: 190 | if self._errors: 191 | raise self._errors[0] 192 | -------------------------------------------------------------------------------- /tests/strategies/test_setupcfg_merge.yml: -------------------------------------------------------------------------------- 1 | 2 | strategy: SetupCfgMerge 3 | 4 | testcases: 5 | - name: non_existing_target 6 | template: | 7 | [metadata] 8 | # Project name 9 | version = 10 | name=foo 11 | target: null 12 | # Comments are removed, keys are sorted 13 | out: | 14 | [metadata] 15 | name = foo 16 | version = 17 | config: &default_config 18 | merge_requirements: 19 | - sections: ^options$ 20 | keys: ^install_requires$ 21 | - sections: ^options\.extras_require$ 22 | keys: ^develop$ 23 | preserve_keys: 24 | - sections: ^tool:pytest$ 25 | keys: ^testpaths$ 26 | - sections: ^build$ 27 | keys: ^executable$ 28 | preserve_sections: 29 | - sections: ^freebsd$ 30 | - sections: ^infra\.dependencies_updater$ 31 | - sections: ^mypy- 32 | - sections: ^options\.data_files$ 33 | - sections: ^options\.entry_points$ 34 | - sections: ^options\.extras_require$ 35 | 36 | - name: extraneous_sections 37 | template: | 38 | [metadata] 39 | name=foo 40 | version = 1 41 | [isort] 42 | atomic = true 43 | target: | 44 | [aliases] 45 | test = pytest 46 | [metadata] 47 | name=foo 48 | url = http:// 49 | out: | 50 | [isort] 51 | atomic = true 52 | 53 | [metadata] 54 | name = foo 55 | version = 1 56 | config: 57 | <<: *default_config 58 | 59 | - name: mypy_sections 60 | template: | 61 | [mypy] 62 | a = 1 63 | [mypy-aiohttp.*] 64 | ignore_missing_imports = True 65 | [mypy-pytest.*] 66 | ignore_missing_imports = True 67 | target: | 68 | [mypy] 69 | a = 2 70 | b = 1 71 | [mypy-packaging.*] 72 | ignore_missing_imports = True 73 | [mypy-pytest.*] 74 | ignore_missing_imports = True 75 | [aliases] 76 | test = pytest 77 | out: | 78 | [mypy] 79 | a = 1 80 | 81 | [mypy-aiohttp.*] 82 | ignore_missing_imports = True 83 | 84 | [mypy-packaging.*] 85 | ignore_missing_imports = True 86 | 87 | [mypy-pytest.*] 88 | ignore_missing_imports = True 89 | config: 90 | <<: *default_config 91 | 92 | - name: requirements_nonexisting_target 93 | template: | 94 | [options] 95 | install_requires = 96 | Black 97 | isort==4.3 98 | 99 | [options.extras_require] 100 | develop = 101 | pytest 102 | target: null 103 | out: | 104 | [options] 105 | install_requires = 106 | Black 107 | isort==4.3 108 | 109 | [options.extras_require] 110 | develop = 111 | pytest 112 | config: 113 | <<: *default_config 114 | 115 | - name: requirements_update 116 | template: | 117 | [options] 118 | install_requires = 119 | Black 120 | isort==4.3 121 | 122 | [options.extras_require] 123 | develop = 124 | pytest 125 | target: | 126 | [options] 127 | install_requires = 128 | aiohttp==4.3 129 | isorT==1.3 130 | 131 | [options.extras_require] 132 | develop = 133 | flask 134 | fast = 135 | librabbitmq 136 | out: | 137 | [options] 138 | install_requires = 139 | aiohttp==4.3 140 | Black 141 | isorT==1.3 142 | 143 | [options.extras_require] 144 | develop = 145 | flask 146 | pytest 147 | fast = 148 | librabbitmq 149 | config: 150 | <<: *default_config 151 | 152 | - name: pytest_testpaths 153 | template: | 154 | [tool:pytest] 155 | addopts = 156 | --verbose 157 | --showlocals 158 | target: | 159 | [tool:pytest] 160 | addopts = 161 | --verbose 162 | --showlocals 163 | --tb=short 164 | testpaths = 165 | tests/unit/ 166 | tests/integration/ 167 | out: | 168 | [tool:pytest] 169 | addopts = 170 | --verbose 171 | --showlocals 172 | testpaths = 173 | tests/unit/ 174 | tests/integration/ 175 | config: 176 | <<: *default_config 177 | 178 | - name: build_executable 179 | template: | 180 | [metadata] 181 | name=foo 182 | target: | 183 | [build] 184 | executable = /usr/bin/env python3.6 185 | out: | 186 | [build] 187 | executable = /usr/bin/env python3.6 188 | 189 | [metadata] 190 | name = foo 191 | config: 192 | <<: *default_config 193 | 194 | - name: freebsd 195 | template: | 196 | [metadata] 197 | name=foo 198 | target: | 199 | [build] 200 | executable = /usr/bin/env python3.6 201 | 202 | [freebsd] 203 | user = nobody 204 | group = nobody 205 | out: | 206 | [build] 207 | executable = /usr/bin/env python3.6 208 | 209 | [freebsd] 210 | group = nobody 211 | user = nobody 212 | 213 | [metadata] 214 | name = foo 215 | config: 216 | <<: *default_config 217 | 218 | - name: infra.dependencies_updater 219 | template: | 220 | [metadata] 221 | name=foo 222 | target: | 223 | [build] 224 | executable = /usr/bin/env python3.6 225 | 226 | [infra.dependencies_updater] 227 | pinned = 228 | xgboost==0.6 229 | out: | 230 | [build] 231 | executable = /usr/bin/env python3.6 232 | 233 | [infra.dependencies_updater] 234 | pinned = 235 | xgboost==0.6 236 | 237 | [metadata] 238 | name = foo 239 | config: 240 | <<: *default_config 241 | 242 | - name: requirements_update_multiple_extras 243 | template: | 244 | [options] 245 | install_requires = 246 | isort==4.3 247 | 248 | [options.extras_require] 249 | nondev = 250 | django 251 | dev-a = 252 | pytest 253 | dev-b = 254 | flake8 255 | target: | 256 | [options] 257 | install_requires = 258 | aiohttp 259 | 260 | [options.extras_require] 261 | nondevtarget = 262 | django 263 | dev-c = 264 | flask 265 | dev-b = 266 | isort 267 | out: | 268 | [options] 269 | install_requires = 270 | aiohttp 271 | isort==4.3 272 | 273 | [options.extras_require] 274 | dev-a = 275 | pytest 276 | dev-b = 277 | flake8 278 | isort 279 | dev-c = 280 | flask 281 | nondev = 282 | django 283 | config: 284 | merge_requirements: 285 | - sections: ^options$ 286 | keys: ^install_requires$ 287 | - sections: ^options\.extras_require$ 288 | keys: ^dev- 289 | preserve_keys: [] 290 | preserve_sections: [] 291 | 292 | - name: requirements_update_with_mixed_newlines 293 | template: "\ 294 | [options]\n\ 295 | install_requires =\n\ 296 | \ Black\n\ 297 | \ isort==4.3\n" 298 | target: "\ 299 | [options]\r\n\ 300 | install_requires =\r\n\ 301 | \ aiohttp==4.3\r\n\ 302 | \ isorT==1.3\r\n" 303 | out: "\ 304 | [options]\r\n\ 305 | install_requires =\r\n\ 306 | \ aiohttp==4.3\r\n\ 307 | \ Black\r\n\ 308 | \ isorT==1.3\r\n" 309 | config: 310 | <<: *default_config 311 | -------------------------------------------------------------------------------- /tests/automation/test_gitlab.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest.mock import call, patch, sentinel 3 | 4 | import pytest 5 | 6 | import scaraplate.automation.gitlab 7 | from scaraplate.automation.gitlab import ( 8 | GitLabCloneTemplateVCS, 9 | GitLabMRProjectVCS, 10 | gitlab_available, 11 | gitlab_clone_url, 12 | gitlab_project_url, 13 | ) 14 | from scaraplate.template import TemplateMeta 15 | 16 | 17 | @pytest.mark.parametrize( 18 | "project_url, private_token, expected_url", 19 | [ 20 | ( 21 | "https://mygitlab.com/myorg/myproj.git", 22 | "xxxtokenxxx", 23 | "https://oauth2:xxxtokenxxx@mygitlab.com/myorg/myproj.git", 24 | ), 25 | ( 26 | "https://mygitlab.com/myorg/myproj", 27 | "xxxtokenxxx", 28 | "https://oauth2:xxxtokenxxx@mygitlab.com/myorg/myproj.git", 29 | ), 30 | ( 31 | "https://mygitlab.com/myorg/myproj", 32 | None, 33 | "https://mygitlab.com/myorg/myproj.git", 34 | ), 35 | ], 36 | ) 37 | def test_gitlab_clone_url(project_url, private_token, expected_url): 38 | assert expected_url == gitlab_clone_url(project_url, private_token) 39 | 40 | 41 | @pytest.mark.parametrize( 42 | "gitlab_url, full_project_name, expected_url", 43 | [ 44 | ("https://mygitlab.com", "myorg/myproj", "https://mygitlab.com/myorg/myproj"), 45 | ( 46 | "https://mygitlab.com/", 47 | "myorg/myproj.git", 48 | "https://mygitlab.com/myorg/myproj.git", 49 | ), 50 | ], 51 | ) 52 | def test_gitlab_project_url(gitlab_url, full_project_name, expected_url): 53 | assert expected_url == gitlab_project_url(gitlab_url, full_project_name) 54 | 55 | 56 | def test_gitlab_clone_template(template_bare_git_repo, call_git): 57 | with patch.object( 58 | scaraplate.automation.gitlab, 59 | "gitlab_clone_url", 60 | return_value=str(template_bare_git_repo), 61 | ) as mock_gitlab_clone_url: 62 | with GitLabCloneTemplateVCS.clone( 63 | project_url=sentinel.project_url, private_token=sentinel.private_token 64 | ) as template_vcs: 65 | # Ensure this is a valid git repo: 66 | call_git("git status", cwd=template_vcs.dest_path) 67 | 68 | assert template_vcs.template_meta.head_ref == "master" 69 | assert mock_gitlab_clone_url.call_count == 1 70 | assert mock_gitlab_clone_url.call_args == call( 71 | sentinel.project_url, sentinel.private_token 72 | ) 73 | 74 | 75 | @pytest.mark.skipif( 76 | gitlab_available, reason="gitlab should not be installed for this test" 77 | ) 78 | def test_gitlab_mr_project_raises_without_gitlab(): 79 | with pytest.raises(ImportError): 80 | with GitLabMRProjectVCS.clone( 81 | gitlab_url="http://a", 82 | full_project_name="a/aa", 83 | private_token="zzz", 84 | changes_branch="update", 85 | ): 86 | raise AssertionError("should not be executed") 87 | 88 | 89 | @pytest.mark.skipif( 90 | not gitlab_available, reason="gitlab should be installed for this test" 91 | ) 92 | @pytest.mark.parametrize( 93 | "mr_exists, changes_branch", 94 | [ 95 | (False, "update"), 96 | (True, "update"), 97 | (None, "master"), # MRs cannot be created from default branch 98 | ], 99 | ) 100 | def test_gitlab_mr_project( 101 | project_bare_git_repo, call_git, http_server, mr_exists, changes_branch 102 | ): 103 | requests = iter( 104 | [ 105 | ( 106 | "GET", 107 | "/api/v4/user", 108 | { 109 | # https://docs.gitlab.com/ee/api/users.html#list-current-user-for-normal-users 110 | "name": "test user", 111 | "email": "test@notemail", 112 | }, 113 | None, 114 | ), 115 | ( 116 | "GET", 117 | "/api/v4/projects/a%2Faa", 118 | { 119 | # https://docs.gitlab.com/ee/api/projects.html#get-single-project 120 | "id": 42, 121 | "name": "test project", 122 | "default_branch": "master", 123 | }, 124 | None, 125 | ), 126 | ] 127 | + ( 128 | [] 129 | if mr_exists is None 130 | else [ 131 | ( 132 | "GET", 133 | "/api/v4/projects/42/merge_requests" 134 | "?state=opened&source_branch=update&target_branch=master", 135 | [] if not mr_exists else [{"id": 98, "iid": 98}], 136 | None, 137 | ) 138 | ] 139 | ) 140 | + ( 141 | [] 142 | if mr_exists or mr_exists is None 143 | else [ 144 | ( 145 | "POST", 146 | "/api/v4/projects/42/merge_requests", 147 | {"id": 99, "iid": 99}, 148 | lambda body: mr_body_checker(body), 149 | ) 150 | ] 151 | ) 152 | ) 153 | 154 | def mr_body_checker(body): 155 | data = json.loads(body.decode()) 156 | assert "1111111111111111111111111111111111111111" in data["description"] 157 | 158 | @http_server.set_request_handler 159 | def request_handler(method, path, headers, body): 160 | assert headers["PRIVATE-TOKEN"] == "zzz" 161 | try: 162 | exp_method, exp_path, response, body_checker = next(requests) 163 | except StopIteration: # pragma: no cover 164 | raise AssertionError(f"Received unexpected request {method} {path}") 165 | assert method == exp_method 166 | assert path == exp_path 167 | if body_checker is not None: 168 | body_checker(body) 169 | return 200, {"Content-type": "application/json"}, json.dumps(response).encode() 170 | 171 | with patch.object( 172 | scaraplate.automation.gitlab, 173 | "gitlab_clone_url", 174 | return_value=str(project_bare_git_repo), 175 | ) as mock_gitlab_clone_url: 176 | with GitLabMRProjectVCS.clone( 177 | gitlab_url=http_server.get_url(), 178 | full_project_name="a/aa", 179 | private_token="zzz", 180 | changes_branch=changes_branch, 181 | ) as project_vcs: 182 | # Ensure this is a valid git repo: 183 | call_git("git status", cwd=project_vcs.dest_path) 184 | 185 | assert not project_vcs.is_dirty() 186 | 187 | (project_vcs.dest_path / "hi").write_text("change!") 188 | project_vcs.commit_changes( 189 | TemplateMeta( 190 | git_project_url="https://testdomain/t/tt", 191 | commit_hash="1111111111111111111111111111111111111111", 192 | commit_url=( 193 | "https://testdomain/t/tt" 194 | "/commit/1111111111111111111111111111111111111111" 195 | ), 196 | is_git_dirty=False, 197 | head_ref="master", 198 | ) 199 | ) 200 | 201 | commit_message = call_git( 202 | f'git log --pretty=format:"%B" -n 1 {changes_branch}', 203 | cwd=project_bare_git_repo, 204 | ) 205 | assert "1111111111111111111111111111111111111111" in commit_message 206 | assert "Scheduled" in commit_message 207 | 208 | commit_author = call_git( 209 | f'git log --pretty=format:"%aN <%aE>" -n 1 {changes_branch}', 210 | cwd=project_bare_git_repo, 211 | ) 212 | assert commit_author == "test user " 213 | 214 | assert "change!" == call_git( 215 | f"git show {changes_branch}:hi", cwd=project_bare_git_repo 216 | ) 217 | 218 | assert mock_gitlab_clone_url.call_count == 1 219 | assert mock_gitlab_clone_url.call_args == call( 220 | f"{http_server.get_url()}/a/aa", "zzz" 221 | ) 222 | 223 | # ensure that all expected queries have been executed: 224 | assert next(requests, ...) is ... 225 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import jinja2.exceptions 4 | import pytest 5 | 6 | import scaraplate.strategies 7 | from scaraplate.config import ( 8 | ScaraplateYamlOptions, 9 | ScaraplateYamlStrategies, 10 | StrategyNode, 11 | get_scaraplate_yaml_options, 12 | get_scaraplate_yaml_strategies, 13 | ) 14 | from scaraplate.cookiecutter import CookieCutterContextDict, ScaraplateConf, SetupCfg 15 | from scaraplate.gitremotes import GitHub 16 | 17 | 18 | @pytest.mark.parametrize( 19 | "yaml_text, expected", 20 | [ 21 | ( 22 | """ 23 | git_remote_type: scaraplate.gitremotes.GitHub 24 | """, 25 | ScaraplateYamlOptions( 26 | git_remote_type=GitHub, cookiecutter_context_type=ScaraplateConf 27 | ), 28 | ), 29 | ( 30 | """ 31 | cookiecutter_context_type: scaraplate.cookiecutter.SetupCfg 32 | """, 33 | ScaraplateYamlOptions( 34 | git_remote_type=None, cookiecutter_context_type=SetupCfg 35 | ), 36 | ), 37 | ], 38 | ) 39 | def test_get_scaraplate_yaml_options_valid( 40 | tempdir_path: Path, yaml_text, expected 41 | ) -> None: 42 | (tempdir_path / "scaraplate.yaml").write_text(yaml_text) 43 | scaraplate_yaml_options = get_scaraplate_yaml_options(tempdir_path) 44 | assert scaraplate_yaml_options == expected 45 | 46 | 47 | @pytest.mark.parametrize( 48 | "yaml_text, cookiecutter_context_dict, expected", 49 | [ 50 | # Basic syntax: 51 | ( 52 | """ 53 | default_strategy: scaraplate.strategies.Overwrite 54 | strategies_mapping: 55 | Jenkinsfile: scaraplate.strategies.TemplateHash 56 | 'some/nested/setup.py': scaraplate.strategies.TemplateHash 57 | """, 58 | {}, 59 | ScaraplateYamlStrategies( 60 | default_strategy=StrategyNode( 61 | strategy=scaraplate.strategies.Overwrite, config={} 62 | ), 63 | strategies_mapping={ 64 | "Jenkinsfile": StrategyNode( 65 | strategy=scaraplate.strategies.TemplateHash, config={} 66 | ), 67 | "some/nested/setup.py": StrategyNode( 68 | strategy=scaraplate.strategies.TemplateHash, config={} 69 | ), 70 | }, 71 | ), 72 | ), 73 | # Extended syntax (with config): 74 | ( 75 | """ 76 | default_strategy: 77 | strategy: scaraplate.strategies.Overwrite 78 | config: 79 | some_key: True 80 | strategies_mapping: 81 | other_file.txt: 82 | strategy: scaraplate.strategies.IfMissing 83 | unrelated_key_but_okay: 1 84 | another_file.txt: 85 | strategy: scaraplate.strategies.SortedUniqueLines 86 | config: 87 | some_key: True 88 | """, 89 | {}, 90 | ScaraplateYamlStrategies( 91 | default_strategy=StrategyNode( 92 | strategy=scaraplate.strategies.Overwrite, config={"some_key": True} 93 | ), 94 | strategies_mapping={ 95 | "other_file.txt": StrategyNode( 96 | strategy=scaraplate.strategies.IfMissing, config={} 97 | ), 98 | "another_file.txt": StrategyNode( 99 | strategy=scaraplate.strategies.SortedUniqueLines, 100 | config={"some_key": True}, 101 | ), 102 | }, 103 | ), 104 | ), 105 | # Jinja2 filenames and globs: 106 | ( 107 | """ 108 | default_strategy: scaraplate.strategies.Overwrite 109 | strategies_mapping: 110 | '*.txt': scaraplate.strategies.IfMissing 111 | 'src/{{ cookiecutter.file1 }}': scaraplate.strategies.TemplateHash 112 | 'src/{{ cookiecutter.file2 }}': scaraplate.strategies.IfMissing 113 | """, 114 | {"file1": "zzz.py", "file2": "aaa.py"}, 115 | ScaraplateYamlStrategies( 116 | default_strategy=StrategyNode( 117 | strategy=scaraplate.strategies.Overwrite, config={} 118 | ), 119 | strategies_mapping={ 120 | "*.txt": StrategyNode( 121 | strategy=scaraplate.strategies.IfMissing, config={} 122 | ), 123 | "src/aaa.py": StrategyNode( 124 | strategy=scaraplate.strategies.IfMissing, config={} 125 | ), 126 | "src/zzz.py": StrategyNode( 127 | strategy=scaraplate.strategies.TemplateHash, config={} 128 | ), 129 | }, 130 | ), 131 | ), 132 | ], 133 | ) 134 | def test_get_scaraplate_yaml_strategies_valid( 135 | tempdir_path: Path, yaml_text, cookiecutter_context_dict, expected 136 | ) -> None: 137 | (tempdir_path / "scaraplate.yaml").write_text(yaml_text) 138 | scaraplate_yaml_strategies = get_scaraplate_yaml_strategies( 139 | tempdir_path, cookiecutter_context_dict 140 | ) 141 | assert scaraplate_yaml_strategies == expected 142 | 143 | 144 | @pytest.mark.parametrize( 145 | "cls", 146 | [ 147 | "tempfile.TemporaryDirectory", 148 | "tempfile", 149 | "scaraplate.strategies.Strategy", 150 | '{"strategy": "tempfile.TemporaryDirectory"}', 151 | '{"strategy": 42}', 152 | '{"config": {}}', # strategy is missing 153 | '{"strategy": "scaraplate.strategies.Overwrite", "config": 42}', 154 | "42", 155 | ], 156 | ) 157 | @pytest.mark.parametrize("mutation_target", ["default_strategy", "strategies_mapping"]) 158 | def test_get_scaraplate_yaml_strategies_invalid( 159 | tempdir_path: Path, cls: str, mutation_target: str 160 | ) -> None: 161 | classes = dict( 162 | default_strategy="scaraplate.strategies.Overwrite", 163 | strategies_mapping="scaraplate.strategies.Overwrite", 164 | ) 165 | classes[mutation_target] = cls 166 | 167 | yaml_text = f""" 168 | default_strategy: {classes['default_strategy']} 169 | strategies_mapping: 170 | Jenkinsfile: {classes['strategies_mapping']} 171 | """ 172 | (tempdir_path / "scaraplate.yaml").write_text(yaml_text) 173 | with pytest.raises(ValueError): 174 | get_scaraplate_yaml_strategies(tempdir_path, CookieCutterContextDict({})) 175 | 176 | 177 | @pytest.mark.parametrize( 178 | "key, cookiecutter_context_dict", 179 | [ 180 | ("'{{ cookiecutter.missingkey }}'", {"anotherkey": "42"}), 181 | ("'{{ somevar }}'", {"somavar": "42"}), # doesn't start with `cookiecutter.` 182 | ], 183 | ) 184 | def test_get_scaraplate_yaml_strategies_invalid_keys( 185 | tempdir_path: Path, key, cookiecutter_context_dict 186 | ) -> None: 187 | yaml_text = f""" 188 | default_strategy: scaraplate.strategies.Overwrite 189 | strategies_mapping: 190 | {key}: scaraplate.strategies.Overwrite 191 | """ 192 | (tempdir_path / "scaraplate.yaml").write_text(yaml_text) 193 | with pytest.raises(jinja2.exceptions.UndefinedError): 194 | get_scaraplate_yaml_strategies(tempdir_path, cookiecutter_context_dict) 195 | 196 | 197 | @pytest.mark.parametrize( 198 | "cls", 199 | [ 200 | "tempfile.TemporaryDirectory", 201 | "tempfile", 202 | "scaraplate.gitremotes.GitRemote", 203 | "scaraplate.cookiecutter.ScaraplateConf", 204 | '{"strategy": "scaraplate.gitremotes.GitLab"}', 205 | "42", 206 | ], 207 | ) 208 | def test_get_scaraplate_yaml_options_invalid_git_remotes( 209 | tempdir_path: Path, cls: str 210 | ) -> None: 211 | yaml_text = f""" 212 | git_remote_type: {cls} 213 | """ 214 | (tempdir_path / "scaraplate.yaml").write_text(yaml_text) 215 | with pytest.raises(ValueError): 216 | get_scaraplate_yaml_options(tempdir_path) 217 | 218 | 219 | @pytest.mark.parametrize( 220 | "cls", 221 | [ 222 | "tempfile.TemporaryDirectory", 223 | "tempfile", 224 | "scaraplate.gitremotes.GitHub", 225 | "scaraplate.cookiecutter.CookieCutterContext", 226 | '{"strategy": "scaraplate.cookiecutter.ScaraplateConf"}', 227 | "42", 228 | ], 229 | ) 230 | def test_get_scaraplate_yaml_options_invalid_cookiecutter_context( 231 | tempdir_path: Path, cls: str 232 | ) -> None: 233 | yaml_text = f""" 234 | cookiecutter_context_type: {cls} 235 | """ 236 | (tempdir_path / "scaraplate.yaml").write_text(yaml_text) 237 | with pytest.raises(ValueError): 238 | get_scaraplate_yaml_options(tempdir_path) 239 | -------------------------------------------------------------------------------- /tests/automation/test_git.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import json 3 | from pathlib import Path 4 | from unittest.mock import ANY 5 | 6 | import pytest 7 | 8 | from scaraplate.automation.base import automatic_rollup 9 | from scaraplate.automation.git import ( 10 | GitCloneProjectVCS, 11 | GitCloneTemplateVCS, 12 | strip_credentials_from_git_remote, 13 | ) 14 | 15 | 16 | @contextlib.contextmanager 17 | def git_unbare_repo(git_path: Path, call_git): 18 | call_git("git config --bool core.bare false", cwd=git_path) 19 | try: 20 | yield 21 | finally: 22 | call_git("git config --bool core.bare true", cwd=git_path) 23 | 24 | 25 | def convert_bare_to_monorepo(dest: str, git_path: Path, call_git): 26 | with git_unbare_repo(git_path, call_git): 27 | dest_path = git_path / dest 28 | dest_path.mkdir() 29 | 30 | for child in git_path.iterdir(): 31 | if child.name in (".git", dest): 32 | continue 33 | child.rename(dest_path / child.name) 34 | 35 | call_git("git add --all .", cwd=git_path) 36 | call_git('git commit -m "convert to monorepo"', cwd=git_path) 37 | call_git("git branch -f master2 master", cwd=git_path) 38 | 39 | 40 | @pytest.mark.parametrize( 41 | "target_branch, clone_ref", 42 | [ 43 | ("updates", None), 44 | ("updates", "master2"), 45 | ("updates", "master"), 46 | ("master", None), 47 | # master-master2 is excluded, because cloning a non-default branch and 48 | # committing the changes back to default is ridiculous 49 | ("master", "master"), 50 | ], 51 | ) 52 | @pytest.mark.parametrize("monorepo_inner_path", ["inner", None]) 53 | def test_automatic_rollup( 54 | template_bare_git_repo: Path, 55 | project_bare_git_repo: Path, 56 | target_branch, 57 | clone_ref, 58 | monorepo_inner_path, 59 | call_git, 60 | ): 61 | if monorepo_inner_path is not None: 62 | convert_bare_to_monorepo(monorepo_inner_path, template_bare_git_repo, call_git) 63 | convert_bare_to_monorepo(monorepo_inner_path, project_bare_git_repo, call_git) 64 | 65 | assert "master" == call_git( 66 | "git rev-parse --abbrev-ref HEAD", cwd=project_bare_git_repo 67 | ) 68 | master_commit_hash = call_git( 69 | "git rev-parse --verify master", cwd=project_bare_git_repo 70 | ) 71 | template_commit_hash = call_git( 72 | f"git rev-parse --verify {clone_ref or 'master'}", cwd=template_bare_git_repo 73 | ) 74 | 75 | automatic_rollup( 76 | template_vcs_ctx=GitCloneTemplateVCS.clone( 77 | clone_url=str(template_bare_git_repo), # a clonable remote URL 78 | clone_ref=clone_ref, 79 | monorepo_inner_path=monorepo_inner_path, 80 | ), 81 | project_vcs_ctx=GitCloneProjectVCS.clone( 82 | clone_url=str(project_bare_git_repo), # a clonable remote URL 83 | clone_ref=clone_ref, 84 | monorepo_inner_path=monorepo_inner_path, 85 | changes_branch=target_branch, 86 | commit_author="pytest ", 87 | ), 88 | extra_context={"key1": "value1", "key2": "value2"}, 89 | ) 90 | 91 | target_branch_commit_hash = call_git( 92 | f"git rev-parse --verify {target_branch}", cwd=project_bare_git_repo 93 | ) 94 | if target_branch == "master": 95 | # Ensure that master branch has advanced 96 | assert master_commit_hash != target_branch_commit_hash 97 | 98 | commit_message = call_git( 99 | f'git log --pretty=format:"%B" -n 1 {target_branch}', cwd=project_bare_git_repo 100 | ) 101 | assert (clone_ref or "master") in commit_message # template clone ref 102 | assert template_commit_hash in commit_message 103 | 104 | # Apply again (this time we expect no changes) 105 | automatic_rollup( 106 | template_vcs_ctx=GitCloneTemplateVCS.clone( 107 | clone_url=str(template_bare_git_repo), # a clonable remote URL 108 | clone_ref=clone_ref, 109 | monorepo_inner_path=monorepo_inner_path, 110 | ), 111 | project_vcs_ctx=GitCloneProjectVCS.clone( 112 | clone_url=str(project_bare_git_repo), # a clonable remote URL 113 | clone_ref=clone_ref, 114 | monorepo_inner_path=monorepo_inner_path, 115 | changes_branch=target_branch, 116 | commit_author="pytest ", 117 | ), 118 | extra_context={"key1": "value1", "key2": "value2"}, 119 | ) 120 | 121 | target_branch_commit_hash_2 = call_git( 122 | f"git rev-parse --verify {target_branch}", cwd=project_bare_git_repo 123 | ) 124 | assert target_branch_commit_hash == target_branch_commit_hash_2, call_git( 125 | f"git show -p {target_branch}", cwd=project_bare_git_repo 126 | ) 127 | 128 | 129 | def test_automatic_rollup_with_existing_target_branch( 130 | template_bare_git_repo: Path, project_bare_git_repo: Path, call_git 131 | ): 132 | target_branch = "update" 133 | 134 | assert "master" == call_git( 135 | "git rev-parse --abbrev-ref HEAD", cwd=project_bare_git_repo 136 | ) 137 | master_commit_hash = call_git( 138 | "git rev-parse --verify master", cwd=project_bare_git_repo 139 | ) 140 | 141 | # Create `update` branch with one version 142 | automatic_rollup( 143 | template_vcs_ctx=GitCloneTemplateVCS.clone( 144 | clone_url=str(template_bare_git_repo) # a clonable remote URL 145 | ), 146 | project_vcs_ctx=GitCloneProjectVCS.clone( 147 | clone_url=str(project_bare_git_repo), # a clonable remote URL 148 | changes_branch=target_branch, 149 | commit_author="pytest ", 150 | ), 151 | extra_context={"key1": "first1", "key2": "first2"}, 152 | ) 153 | 154 | target_branch_commit_hash = call_git( 155 | f"git rev-parse --verify {target_branch}", cwd=project_bare_git_repo 156 | ) 157 | 158 | # Apply again (but another version of template -- note the changed 159 | # extra_context 160 | automatic_rollup( 161 | template_vcs_ctx=GitCloneTemplateVCS.clone( 162 | clone_url=str(template_bare_git_repo) # a clonable remote URL 163 | ), 164 | project_vcs_ctx=GitCloneProjectVCS.clone( 165 | clone_url=str(project_bare_git_repo), # a clonable remote URL 166 | changes_branch=target_branch, 167 | commit_author="pytest ", 168 | ), 169 | extra_context={"key1": "second1", "key2": "second2"}, 170 | ) 171 | 172 | master_commit_hash_2 = call_git( 173 | "git rev-parse --verify master", cwd=project_bare_git_repo 174 | ) 175 | target_branch_commit_hash_2 = call_git( 176 | f"git rev-parse --verify {target_branch}", cwd=project_bare_git_repo 177 | ) 178 | # The first push should be replaced with the second one: 179 | assert target_branch_commit_hash != target_branch_commit_hash_2 180 | # master should not be changed: 181 | assert master_commit_hash == master_commit_hash_2 182 | 183 | 184 | @pytest.mark.parametrize("monorepo_inner_path", ["inner", None]) 185 | @pytest.mark.template_with_sense_vars 186 | def test_automatic_rollup_preserves_template_dirname( 187 | template_bare_git_repo: Path, 188 | project_bare_git_repo: Path, 189 | monorepo_inner_path, 190 | call_git, 191 | ): 192 | target_branch = "update" 193 | 194 | if monorepo_inner_path is not None: 195 | convert_bare_to_monorepo(monorepo_inner_path, template_bare_git_repo, call_git) 196 | 197 | automatic_rollup( 198 | template_vcs_ctx=GitCloneTemplateVCS.clone( 199 | clone_url=str(template_bare_git_repo), # a clonable remote URL 200 | monorepo_inner_path=monorepo_inner_path, 201 | ), 202 | project_vcs_ctx=GitCloneProjectVCS.clone( 203 | clone_url=str(project_bare_git_repo), # a clonable remote URL 204 | changes_branch=target_branch, 205 | commit_author="pytest ", 206 | ), 207 | extra_context={"key1": "value1", "key2": "value2"}, 208 | ) 209 | 210 | cookiecutter_context_text = call_git( 211 | f"git show {target_branch}:sense_vars", cwd=project_bare_git_repo 212 | ) 213 | 214 | assert json.loads(cookiecutter_context_text) == { 215 | "_output_dir": ANY, 216 | "_repo_dir": ANY, 217 | "_template": monorepo_inner_path if monorepo_inner_path else "remote_template", 218 | "project_dest": "remote_project", 219 | "key1": "value1", 220 | "key2": "value2", 221 | } 222 | 223 | 224 | @pytest.mark.parametrize( 225 | "remote_url, expected_url", 226 | [ 227 | ( 228 | "https://oauth2:xxxtokenxxx@mygitlab.com/myorg/myproj.git", 229 | "https://mygitlab.com/myorg/myproj.git", 230 | ), 231 | ( 232 | "https://mygitlab.com/myorg/myproj.git", 233 | "https://mygitlab.com/myorg/myproj.git", 234 | ), 235 | # I suppose ssh urls cannot contain a password? 236 | ("git@mygitlab.com:myorg/myproj.git", "git@mygitlab.com:myorg/myproj.git"), 237 | ("/var/myrepos/mybare", "/var/myrepos/mybare"), 238 | ], 239 | ) 240 | def test_strip_credentials_from_git_remote(remote_url, expected_url): 241 | assert expected_url == strip_credentials_from_git_remote(remote_url) 242 | -------------------------------------------------------------------------------- /src/scaraplate/automation/gitlab.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import logging 3 | from pathlib import Path 4 | from typing import Iterator, Optional 5 | from urllib.parse import urljoin, urlparse, urlunparse 6 | 7 | from scaraplate.automation.base import ProjectVCS, TemplateVCS 8 | from scaraplate.automation.git import ( 9 | GitCloneProjectVCS, 10 | GitCloneTemplateVCS, 11 | scaraplate_version, 12 | ) 13 | from scaraplate.template import TemplateMeta 14 | 15 | try: 16 | import gitlab 17 | except ImportError: 18 | gitlab_available = False 19 | else: 20 | gitlab_available = True 21 | 22 | 23 | logger = logging.getLogger("scaraplate") 24 | 25 | 26 | def ensure_gitlab_is_installed(): 27 | if not gitlab_available: 28 | raise ImportError( 29 | "python-gitlab must be installed in order to use GitLab integration. " 30 | 'Install with `pip install "scaraplate[gitlab]"`.' 31 | ) 32 | 33 | 34 | def gitlab_clone_url(project_url: str, private_token: Optional[str]) -> str: 35 | parsed = urlparse(project_url) 36 | assert parsed.scheme 37 | 38 | if private_token: 39 | clean = parsed._replace(netloc=f"oauth2:{private_token}@{parsed.hostname}") 40 | else: 41 | clean = parsed 42 | if not clean.path.endswith(".git"): 43 | clean = clean._replace(path=f"{clean.path}.git") 44 | return urlunparse(clean) 45 | 46 | 47 | def gitlab_project_url(gitlab_url: str, full_project_name: str) -> str: 48 | return urljoin(gitlab_url, full_project_name) 49 | 50 | 51 | class GitLabCloneTemplateVCS(TemplateVCS): 52 | """A class which extends :class:`.GitCloneTemplateVCS` with 53 | GitLab-specific ``clone_url`` generation. 54 | """ 55 | 56 | def __init__(self, git_clone: GitCloneTemplateVCS) -> None: 57 | self._git_clone = git_clone 58 | 59 | @property 60 | def dest_path(self) -> Path: 61 | return self._git_clone.dest_path 62 | 63 | @property 64 | def template_meta(self) -> TemplateMeta: 65 | return self._git_clone.template_meta 66 | 67 | @classmethod 68 | @contextlib.contextmanager 69 | def clone( 70 | cls, 71 | project_url: str, 72 | private_token: Optional[str] = None, 73 | *, 74 | clone_ref: Optional[str] = None, 75 | monorepo_inner_path: Optional[Path] = None, 76 | ) -> Iterator["GitLabCloneTemplateVCS"]: 77 | """Same as :meth:`.GitCloneTemplateVCS.clone` except that 78 | ``clone_url`` is replaced with ``project_url`` and ``private_token``. 79 | 80 | The ``private_token`` allows to clone private repos, which are 81 | visible only for an authenticated user. 82 | 83 | :param project_url: A URL to a GitLab project, e.g. 84 | ``https://gitlab.example.org/myorganization/myproject``. 85 | :param private_token: GitLab access token, 86 | see ``_. 87 | """ 88 | 89 | with GitCloneTemplateVCS.clone( 90 | clone_url=gitlab_clone_url(project_url, private_token), 91 | clone_ref=clone_ref, 92 | monorepo_inner_path=monorepo_inner_path, 93 | ) as git_clone: 94 | yield cls(git_clone) 95 | 96 | 97 | class GitLabMRProjectVCS(ProjectVCS): 98 | """A class which extends :class:`.GitCloneProjectVCS` with 99 | GitLab-specific ``clone_url`` generation and opens a GitLab Merge Request 100 | after ``git push``. 101 | """ 102 | 103 | def __init__( 104 | self, 105 | git_clone: GitCloneProjectVCS, 106 | *, 107 | gitlab_project, 108 | mr_title_template: str, 109 | mr_description_markdown_template: str, 110 | ) -> None: 111 | self._git_clone = git_clone 112 | self._gitlab_project = gitlab_project 113 | self.mr_title_template = mr_title_template 114 | self.mr_description_markdown_template = mr_description_markdown_template 115 | 116 | @property 117 | def dest_path(self) -> Path: 118 | return self._git_clone.dest_path 119 | 120 | def is_dirty(self) -> bool: 121 | return self._git_clone.is_dirty() 122 | 123 | def commit_changes(self, template_meta: TemplateMeta) -> None: 124 | self._git_clone.commit_changes(template_meta) 125 | 126 | if self._git_clone.changes_branch == self._gitlab_project.default_branch: 127 | logger.info( 128 | "Skipping MR creation step as `changes_branch` and `default_branch` " 129 | "are the same, i.e. the changes are already in the target branch." 130 | ) 131 | else: 132 | self.create_merge_request( 133 | title=self.format_merge_request_title(template_meta=template_meta), 134 | description=self.format_merge_request_description( 135 | template_meta=template_meta 136 | ), 137 | ) 138 | 139 | def format_merge_request_title(self, *, template_meta: TemplateMeta) -> str: 140 | return self.mr_title_template.format( 141 | update_time=self._git_clone.update_time, template_meta=template_meta 142 | ) 143 | 144 | def format_merge_request_description(self, *, template_meta: TemplateMeta) -> str: 145 | # The returned string would be treated as markdown markup. 146 | return self.mr_description_markdown_template.format( 147 | update_time=self._git_clone.update_time, 148 | scaraplate_version=scaraplate_version(), 149 | template_meta=template_meta, 150 | ) 151 | 152 | def create_merge_request(self, *, title: str, description: str) -> None: 153 | existing_mr = self.get_merge_request() 154 | if existing_mr is not None: 155 | logger.info(f"Skipping MR creation as it already exists: {existing_mr!r}") 156 | return 157 | 158 | self._gitlab_project.mergerequests.create( 159 | { 160 | "description": description, 161 | "should_remove_source_branch": True, 162 | "source_branch": self._git_clone.changes_branch, 163 | "target_branch": self._gitlab_project.default_branch, 164 | "title": title, 165 | } 166 | ) 167 | 168 | def get_merge_request(self): 169 | merge_requests = self._gitlab_project.mergerequests.list( 170 | state="opened", 171 | source_branch=self._git_clone.changes_branch, 172 | target_branch=self._gitlab_project.default_branch, 173 | ) 174 | 175 | if not merge_requests: 176 | return None 177 | 178 | assert len(merge_requests) == 1, merge_requests 179 | merge_request = merge_requests[0] 180 | return merge_request 181 | 182 | @classmethod 183 | @contextlib.contextmanager 184 | def clone( 185 | cls, 186 | gitlab_url: str, 187 | full_project_name: str, 188 | private_token: str, 189 | *, 190 | mr_title_template: str = "Scheduled template update ({update_time:%Y-%m-%d})", 191 | mr_description_markdown_template: str = ( 192 | "* scaraplate version: `{scaraplate_version}`\n" 193 | "* template commit: {template_meta.commit_url}\n" 194 | "* template ref: {template_meta.head_ref}\n" 195 | ), 196 | commit_author: Optional[str] = None, 197 | **kwargs, 198 | ) -> Iterator["GitLabMRProjectVCS"]: 199 | """Same as :meth:`.GitCloneProjectVCS.clone` with the following 200 | exceptions: 201 | 202 | - ``clone_url`` is replaced with ``gitlab_url``, ``full_project_name`` 203 | and ``private_token``. 204 | - A GitLab Merge Request (MR) is opened after a successful 205 | ``git push``. 206 | 207 | The ``private_token`` allows to clone private repos, which are 208 | visible only for an authenticated user. 209 | 210 | As in :meth:`.GitCloneProjectVCS.clone`, the ``changes_branch`` 211 | might be the same as ``clone_ref``. In this case no MR will be 212 | opened. 213 | 214 | A MR will be created only if there're any changes produced 215 | by scaraplate rollup. If a ``changes_branch`` is already present 216 | in remote (i.e. there is a previous automatic rollup which wasn't 217 | merged yet), there're two possibilities: 218 | 219 | - If one-commit diffs between the remote's ``changes_branch`` 220 | and the local ``changes_branch`` are the same, nothing 221 | is done. It means that a MR already exists and it has the same 222 | patch as the one which was just produced locally. 223 | - If the diffs are different, the remote branch will be deleted, 224 | effectively closing the old MR, and a new one will be pushed 225 | instead, and a new MR will be opened. 226 | 227 | The opened MRs are expected to be merged manually. 228 | 229 | :param gitlab_url: A URL to the GitLab instance, e.g. 230 | ``https://gitlab.example.org``. 231 | :param full_project_name: Project name within gitlab, e.g. 232 | ``myorganization/myproject``. 233 | :param private_token: GitLab access token, 234 | see ``_. 235 | :param mr_title_template: :meth:`str.format` template 236 | which is used to produce a MR title. 237 | Available format variables are: 238 | 239 | - ``update_time`` [:class:`datetime.datetime`] -- the time 240 | of update 241 | - ``template_meta`` [:class:`.TemplateMeta`] -- template meta 242 | returned by :meth:`.TemplateVCS.template_meta` 243 | :param mr_description_markdown_template: :meth:`str.format` template 244 | which is used to produce a MR description (which will be rendered 245 | as markdown). Available format variables are: 246 | 247 | - ``update_time`` [:class:`datetime.datetime`] -- the time 248 | of update 249 | - ``scaraplate_version`` [:class:`str`] -- scaraplate package 250 | version 251 | - ``template_meta`` [:class:`.TemplateMeta`] -- template meta 252 | returned by :meth:`.TemplateVCS.template_meta` 253 | :param commit_author: Author name to use for ``git commit``, e.g. 254 | ``John Doe ``. If ``None``, will be retrieved 255 | from GitLab as the name of the currently authenticated user 256 | (using ``private_token``). 257 | """ 258 | 259 | ensure_gitlab_is_installed() 260 | client = gitlab.Gitlab(url=gitlab_url, private_token=private_token, timeout=30) 261 | client.auth() 262 | 263 | gitlab_project = client.projects.get(full_project_name) 264 | project_url = gitlab_project_url(gitlab_url, full_project_name) 265 | user = client.user 266 | assert user is not None 267 | commit_author = commit_author or f"{user.name} <{user.email}>" 268 | 269 | # pylint wants mandatory arguments to be passed explicitly: 270 | changes_branch = kwargs.pop("changes_branch") 271 | 272 | with GitCloneProjectVCS.clone( 273 | clone_url=gitlab_clone_url(project_url, private_token), 274 | changes_branch=changes_branch, 275 | commit_author=commit_author, 276 | **kwargs, 277 | ) as git_clone: 278 | yield cls( 279 | git_clone, 280 | gitlab_project=gitlab_project, 281 | mr_title_template=mr_title_template, 282 | mr_description_markdown_template=mr_description_markdown_template, 283 | ) 284 | -------------------------------------------------------------------------------- /src/scaraplate/rollup.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import fnmatch 3 | import io 4 | import os 5 | import pprint 6 | import tempfile 7 | from pathlib import Path 8 | from typing import BinaryIO, Mapping, Optional, Tuple, Union 9 | 10 | import click 11 | from cookiecutter.main import cookiecutter 12 | 13 | from .config import ( 14 | ScaraplateYamlOptions, 15 | ScaraplateYamlStrategies, 16 | StrategyNode, 17 | get_scaraplate_yaml_options, 18 | get_scaraplate_yaml_strategies, 19 | ) 20 | from .cookiecutter import CookieCutterContextDict 21 | from .template import TemplateMeta, get_template_meta_from_git 22 | 23 | __all__ = ("rollup", "InvalidScaraplateTemplateError") 24 | 25 | 26 | class InvalidScaraplateTemplateError(Exception): 27 | """Something is wrong with the template. 28 | 29 | .. versionadded:: 0.2 30 | """ 31 | 32 | 33 | def rollup( 34 | template_dir: Union[Path, str], 35 | target_project_dir: Union[Path, str], 36 | *, 37 | no_input: bool, 38 | extra_context: Optional[Mapping[str, str]] = None, 39 | ) -> None: 40 | """The essence of scaraplate: the rollup function, which can be called 41 | from Python. 42 | 43 | Raises :class:`.InvalidScaraplateTemplateError`. 44 | 45 | .. versionchanged:: 0.2 46 | Added `extra_context` argument which supersedes 47 | :class:`scaraplate.cookiecutter.CookieCutterContext` context. 48 | Useful to provide context values non-interactively. 49 | 50 | ``scaraplate.yaml`` now supports cookiecutter variables 51 | in the `strategies_mapping`` keys. 52 | """ 53 | template_path = Path(template_dir) 54 | target_path = Path(target_project_dir) 55 | 56 | scaraplate_yaml_options = get_scaraplate_yaml_options(template_path) 57 | template_meta = get_template_meta_from_git( 58 | template_path, git_remote_type=scaraplate_yaml_options.git_remote_type 59 | ) 60 | 61 | target_path.mkdir(parents=True, exist_ok=True, mode=0o755) 62 | project_dest = get_project_dest(target_path) 63 | 64 | cookiecutter_context = get_target_project_cookiecutter_context( 65 | target_path, scaraplate_yaml_options 66 | ) 67 | cookiecutter_context = CookieCutterContextDict( 68 | {**cookiecutter_context, **(extra_context or {})} 69 | ) 70 | 71 | with tempfile.TemporaryDirectory() as tempdir_path: 72 | output_dir = Path(tempdir_path) / "out" 73 | output_dir.mkdir(mode=0o700) 74 | if not no_input: 75 | click.echo(f'`project_dest` must equal to "{project_dest}"') 76 | 77 | # By default cookiecutter preserves all entered variables in 78 | # the user's home and tries to reuse it on subsequent cookiecutter 79 | # executions. 80 | # 81 | # Give cookiecutter a fake config so it would write its stuff 82 | # to the tempdir, which would then be removed. 83 | cookiecutter_config_path = Path(tempdir_path) / "cookiecutter_home" 84 | cookiecutter_config_path.mkdir(parents=True) 85 | cookiecutter_config = cookiecutter_config_path / "cookiecutterrc.yaml" 86 | cookiecutter_config.write_text( 87 | f""" 88 | cookiecutters_dir: "{(cookiecutter_config_path.resolve() / 'cookiecutters').as_posix()}" 89 | replay_dir: "{(cookiecutter_config_path.resolve() / 'replay').as_posix()}" 90 | """ 91 | ) 92 | 93 | cookiecutter_context.setdefault( # pylint: disable=no-member 94 | "project_dest", project_dest 95 | ) 96 | 97 | template_root_path, template_dir_name = get_template_root_and_dir(template_path) 98 | 99 | with with_cwd(template_root_path): 100 | # Cookiecutter preserves its template values to 101 | # .scaraplate.conf/setup.cfg (this is specified in the template). 102 | # 103 | # These values contain a `_template` key, which points to 104 | # the template just like it was passed to cookiecutter 105 | # (that is the first positional arg in the command below). 106 | # 107 | # In order to specify only the directory name of the template, 108 | # we change cwd (current working directory) and pass 109 | # the path to template as just a directory name, effectively 110 | # stripping off the path to the template in the local 111 | # filesystem. 112 | cookiecutter( 113 | template_dir_name, 114 | no_input=no_input, 115 | extra_context=cookiecutter_context, 116 | output_dir=str(output_dir.resolve()), 117 | config_file=str(cookiecutter_config), 118 | ) 119 | 120 | # Say the `target_path` looks like `/some/path/to/myproject`. 121 | # 122 | # Cookiecutter puts the generated project files to 123 | # `output_path / "{{ cookiecutter.project_dest }}"`. 124 | # 125 | # We assume that `project_dest` equals the dirname of 126 | # the `target_path` (i.e. `myproject`). 127 | # 128 | # Later we need to move the generated files from 129 | # `output_path / "{{ cookiecutter.project_dest }}" / ...` 130 | # to `/some/path/to/myproject/...`. 131 | # 132 | # For that we need to ensure that cookiecutter did indeed 133 | # put the generated files to `output_path / 'myproject'` 134 | # and no extraneous files or dirs were generated. 135 | actual_items_in_tempdir = os.listdir(output_dir) 136 | expected_items_in_tempdir = [project_dest] 137 | if actual_items_in_tempdir != expected_items_in_tempdir: 138 | raise RuntimeError( 139 | f"A project generated by cookiecutter has an unexpected " 140 | f"file structure.\n" 141 | f"Expected directory listing: {expected_items_in_tempdir}\n" 142 | f"Actual: {actual_items_in_tempdir}\n" 143 | f"\n" 144 | f"Does the TARGET_PROJECT_DIR name match " 145 | f"the cookiecutter's `project_dest` value?" 146 | ) 147 | 148 | with with_cwd(output_dir / project_dest): 149 | # Pass a relative path to CookieCutterContext so the __str__ 150 | # wouldn't include a full absolute path to the temp dir. 151 | cookiecutter_context_dict = get_cookiecutter_context_from_temp_project( 152 | Path("."), scaraplate_yaml_options 153 | ) 154 | 155 | scaraplate_yaml_strategies = get_scaraplate_yaml_strategies( 156 | template_path, cookiecutter_context_dict 157 | ) 158 | apply_generated_project( 159 | output_dir / project_dest, 160 | target_path, 161 | template_meta=template_meta, 162 | scaraplate_yaml_strategies=scaraplate_yaml_strategies, 163 | ) 164 | 165 | click.echo("Done!") 166 | 167 | 168 | def get_project_dest(target_dir_path: Path) -> str: 169 | return target_dir_path.resolve().name 170 | 171 | 172 | def get_template_root_and_dir(template_path: Path) -> Tuple[Path, str]: 173 | template_resolved_path = template_path.resolve() 174 | template_root_path = template_resolved_path.parents[0] 175 | template_dir_name = template_resolved_path.name 176 | return template_root_path, template_dir_name 177 | 178 | 179 | def get_target_project_cookiecutter_context( 180 | target_path: Path, scaraplate_yaml_options: ScaraplateYamlOptions 181 | ) -> CookieCutterContextDict: 182 | cookiecutter_context = scaraplate_yaml_options.cookiecutter_context_type( 183 | target_path 184 | ) 185 | 186 | try: 187 | context: CookieCutterContextDict = cookiecutter_context.read() 188 | except FileNotFoundError: 189 | click.echo( 190 | f"`{cookiecutter_context}` file doesn't exist, " 191 | f"continuing with an empty context..." 192 | ) 193 | return CookieCutterContextDict({}) 194 | else: 195 | if context: 196 | click.echo( 197 | f"Continuing with the following context from " 198 | f"the `{cookiecutter_context}` file:\n{pprint.pformat(context)}" 199 | ) 200 | else: 201 | click.echo( 202 | f"No context found in the `{cookiecutter_context}` file, " 203 | f"continuing with an empty one..." 204 | ) 205 | return CookieCutterContextDict(dict(context)) 206 | 207 | 208 | def get_cookiecutter_context_from_temp_project( 209 | target_path: Path, scaraplate_yaml_options: ScaraplateYamlOptions 210 | ) -> CookieCutterContextDict: 211 | cookiecutter_context = scaraplate_yaml_options.cookiecutter_context_type( 212 | target_path 213 | ) 214 | 215 | try: 216 | context: CookieCutterContextDict = cookiecutter_context.read() 217 | except FileNotFoundError: 218 | raise InvalidScaraplateTemplateError( 219 | f"cookiecutter context file `{cookiecutter_context}` doesn't exist " 220 | f"in the rendered template. Ensure you have added its generation " 221 | f"to the template. See docs for {type(cookiecutter_context)}" 222 | ) 223 | else: 224 | if not context: 225 | raise InvalidScaraplateTemplateError( 226 | f"cookiecutter context file `{cookiecutter_context}` is empty " 227 | f"in the rendered template. Ensure you have correctly added its " 228 | f"generation to the template. See docs for {type(cookiecutter_context)}" 229 | ) 230 | return CookieCutterContextDict(dict(context)) 231 | 232 | 233 | def apply_generated_project( 234 | generated_path: Path, 235 | target_path: Path, 236 | *, 237 | template_meta: TemplateMeta, 238 | scaraplate_yaml_strategies: ScaraplateYamlStrategies, 239 | ) -> None: 240 | generated_path = generated_path.resolve() 241 | 242 | for root, dirs, files in os.walk(generated_path): 243 | current_root_path = Path(root) 244 | path_from_template_root = current_root_path.relative_to(generated_path) 245 | target_root_path = target_path / path_from_template_root 246 | target_root_path.mkdir(parents=True, exist_ok=True, mode=0o755) 247 | 248 | for d in dirs: 249 | (target_root_path / d).mkdir(parents=True, exist_ok=True, mode=0o755) 250 | 251 | for f in files: 252 | file_path = current_root_path / f 253 | target_file_path = target_root_path / f 254 | 255 | strategy_node = get_strategy( 256 | scaraplate_yaml_strategies, path_from_template_root / f 257 | ) 258 | 259 | template_contents = io.BytesIO(file_path.read_bytes()) 260 | if target_file_path.exists(): 261 | target_contents: Optional[BinaryIO] = io.BytesIO( 262 | target_file_path.read_bytes() 263 | ) 264 | else: 265 | target_contents = None 266 | 267 | strategy = strategy_node.strategy( 268 | target_contents=target_contents, 269 | template_contents=template_contents, 270 | template_meta=template_meta, 271 | config=strategy_node.config, 272 | ) 273 | 274 | target_contents = strategy.apply() 275 | target_file_path.write_bytes(target_contents.read()) 276 | 277 | # https://stackoverflow.com/a/5337329 278 | chmod = file_path.stat().st_mode & 0o777 279 | target_file_path.chmod(chmod) 280 | 281 | 282 | def get_strategy( 283 | scaraplate_yaml_strategies: ScaraplateYamlStrategies, path: Path 284 | ) -> StrategyNode: 285 | for glob_pattern, strategy_node in sorted( 286 | scaraplate_yaml_strategies.strategies_mapping.items() 287 | ): 288 | if fnmatch.fnmatch(str(path), glob_pattern): 289 | return strategy_node 290 | return scaraplate_yaml_strategies.default_strategy 291 | 292 | 293 | @contextlib.contextmanager 294 | def with_cwd(cwd: Path): 295 | initial_cwd = os.getcwd() 296 | try: 297 | os.chdir(cwd) 298 | yield 299 | finally: 300 | os.chdir(initial_cwd) 301 | -------------------------------------------------------------------------------- /docs/template.rst: -------------------------------------------------------------------------------- 1 | Scaraplate Template 2 | =================== 3 | 4 | Scaraplate uses cookiecutter under the hood, so the scaraplate template 5 | is a :doc:`cookiecutter template ` with 6 | the following properties: 7 | 8 | - There must be a ``scaraplate.yaml`` config in the root of 9 | the template dir (near the ``cookiecutter.json``). 10 | - The template dir must be a git repo (because some strategies might render 11 | URLs to the template project and HEAD commit, making it easy to find out 12 | what template was used to rollup, see :doc:`gitremotes`). 13 | - The cookiecutter's project dir must be called ``project_dest``, 14 | i.e. the template must reside in the ``{{cookiecutter.project_dest}}`` 15 | directory. 16 | - The template must contain a file which renders the current 17 | cookiecutter context. Scaraplate then reads that context to re-apply 18 | cookiecutter template on subsequent rollups 19 | (see :ref:`cookiecutter_context_types`). 20 | 21 | ``scaraplate.yaml`` contains: 22 | 23 | - strategies (see :doc:`strategies`), 24 | - cookiecutter context type (see :ref:`cookiecutter_context_types`), 25 | - template git remote (see :doc:`gitremotes`). 26 | 27 | .. note:: 28 | Neither ``scaraplate.yaml`` nor ``cookiecutter.json`` would get 29 | to the target project. These two files exist only in the template 30 | repo. The files that would get to the target project are located 31 | in the inner ``{{cookiecutter.project_dest}}`` directory of the template repo. 32 | 33 | 34 | .. _no_input_mode: 35 | 36 | ``scaraplate rollup`` has a ``--no-input`` switch which doesn't ask for 37 | cookiecutter context values. This can be used to automate rollups 38 | when the cookiecutter context is already present in the target project 39 | (i.e. ``scaraplate rollup`` has been applied before). But the first rollup 40 | should be done without the ``--no-input`` option, so the cookiecutter 41 | context values could be filled by hand interactively. 42 | 43 | The arguments to the ``scaraplate rollup`` command must be local 44 | directories (i.e. the template git repo must be cloned manually, 45 | scaraplate doesn't support retrieving templates from git remote directly). 46 | 47 | 48 | .. _scaraplate_example_template: 49 | 50 | Scaraplate Example Template 51 | --------------------------- 52 | 53 | We maintain an example template for a new Python project here: 54 | https://github.com/rambler-digital-solutions/scaraplate-example-template 55 | 56 | You may use it as a starting point for creating your own scaraplate template. 57 | Of course it doesn't have to be for a Python project: the cookiecutter 58 | template might be for anything. A Python project is just an example. 59 | 60 | Creating a new project from the template 61 | ++++++++++++++++++++++++++++++++++++++++ 62 | 63 | :: 64 | 65 | $ git clone https://github.com/rambler-digital-solutions/scaraplate-example-template.git 66 | $ scaraplate rollup ./scaraplate-example-template ./myproject 67 | `myproject1/.scaraplate.conf` file doesn't exist, continuing with an empty context... 68 | `project_dest` must equal to "myproject" 69 | project_dest [myproject]: 70 | project_monorepo_name []: 71 | python_package [myproject]: 72 | metadata_name [myproject]: 73 | metadata_author: Kostya Esmukov 74 | metadata_author_email: kostya@esmukov.net 75 | metadata_description: My example project 76 | metadata_long_description [file: README.md]: 77 | metadata_url [https://github.com/rambler-digital-solutions/myproject]: 78 | coverage_fail_under [100]: 90 79 | mypy_enabled [1]: 80 | Done! 81 | $ tree -a myproject 82 | myproject 83 | ├── .editorconfig 84 | ├── .gitignore 85 | ├── .scaraplate.conf 86 | ├── MANIFEST.in 87 | ├── Makefile 88 | ├── README.md 89 | ├── mypy.ini 90 | ├── setup.cfg 91 | ├── setup.py 92 | ├── src 93 | │   └── myproject 94 | │   └── __init__.py 95 | └── tests 96 | ├── __init__.py 97 | └── test_metadata.py 98 | 99 | 3 directories, 12 files 100 | 101 | The example template also contains a ``project_monorepo_name`` variable 102 | which simplifies creating subprojects in monorepos (e.g. a single git 103 | repository for multiple projects). In this case scaraplate should be 104 | applied to the inner projects: 105 | 106 | :: 107 | 108 | $ scaraplate rollup ./scaraplate-example-template ./mymonorepo/innerproject 109 | `mymonorepo/innerproject/.scaraplate.conf` file doesn't exist, continuing with an empty context... 110 | `project_dest` must equal to "innerproject" 111 | project_dest [innerproject]: 112 | project_monorepo_name []: mymonorepo 113 | python_package [mymonorepo_innerproject]: 114 | metadata_name [mymonorepo-innerproject]: 115 | metadata_author: Kostya Esmukov 116 | metadata_author_email: kostya@esmukov.net 117 | metadata_description: My example project in a monorepo 118 | metadata_long_description [file: README.md]: 119 | metadata_url [https://github.com/rambler-digital-solutions/mymonorepo]: 120 | coverage_fail_under [100]: 90 121 | mypy_enabled [1]: 122 | Done! 123 | $ tree -a mymonorepo 124 | mymonorepo 125 | └── innerproject 126 | ├── .editorconfig 127 | ├── .gitignore 128 | ├── .scaraplate.conf 129 | ├── MANIFEST.in 130 | ├── Makefile 131 | ├── README.md 132 | ├── mypy.ini 133 | ├── setup.cfg 134 | ├── setup.py 135 | ├── src 136 | │   └── mymonorepo_innerproject 137 | │   └── __init__.py 138 | └── tests 139 | ├── __init__.py 140 | └── test_metadata.py 141 | 142 | 4 directories, 12 files 143 | 144 | Updating a project from the template 145 | ++++++++++++++++++++++++++++++++++++ 146 | 147 | :: 148 | 149 | $ scaraplate rollup ./scaraplate-example-template ./myproject --no-input 150 | Continuing with the following context from the `myproject/.scaraplate.conf` file: 151 | {'_template': 'scaraplate-example-template', 152 | 'coverage_fail_under': '90', 153 | 'metadata_author': 'Kostya Esmukov', 154 | 'metadata_author_email': 'kostya@esmukov.net', 155 | 'metadata_description': 'My example project', 156 | 'metadata_long_description': 'file: README.md', 157 | 'metadata_name': 'myproject', 158 | 'metadata_url': 'https://github.com/rambler-digital-solutions/myproject', 159 | 'mypy_enabled': '1', 160 | 'project_dest': 'myproject', 161 | 'project_monorepo_name': '', 162 | 'python_package': 'myproject'} 163 | Done! 164 | 165 | 166 | .. _cookiecutter_context_types: 167 | 168 | Cookiecutter Context Types 169 | -------------------------- 170 | 171 | .. automodule:: scaraplate.cookiecutter 172 | :members: __doc__ 173 | 174 | 175 | .. autoclass:: scaraplate.cookiecutter.CookieCutterContext 176 | :show-inheritance: 177 | :members: 178 | 179 | .. automethod:: __init__ 180 | 181 | 182 | Built-in Cookiecutter Context Types 183 | ----------------------------------- 184 | 185 | .. autoclass:: scaraplate.cookiecutter.ScaraplateConf 186 | :show-inheritance: 187 | 188 | 189 | .. autoclass:: scaraplate.cookiecutter.SetupCfg 190 | :show-inheritance: 191 | 192 | 193 | Template Maintenance 194 | -------------------- 195 | 196 | Given that scaraplate provides ability to update the already created 197 | projects from the updated templates, it's worth discussing the maintenance 198 | of a scaraplate template. 199 | 200 | Removing a template variable 201 | ++++++++++++++++++++++++++++ 202 | 203 | Template variables could be used as :ref:`feature flags ` 204 | to gradually introduce some changes in the templates which some target 205 | projects might not use (yet) by disabling the flag. 206 | 207 | But once the migration is complete, you might want to remove the no longer 208 | needed variable. 209 | 210 | Fortunately this is very simple: just stop using it in the template and 211 | remove it from ``cookiecutter.json``. On the next ``scaraplate rollup`` 212 | the removed variable will be automatically removed from 213 | the :ref:`cookiecutter context file `. 214 | 215 | Adding a new template variable 216 | ++++++++++++++++++++++++++++++ 217 | 218 | The process for adding a new variable is the same as for removing one: 219 | just add it to the ``cookiecutter.json`` and you can start using it in 220 | the template. 221 | 222 | If the next ``scaraplate rollup`` is run with ``--no-input``, the new 223 | variable will have the default value as specified in ``cookiecutter.json``. 224 | If you need a different value, you have 2 options: 225 | 226 | 1. Run ``scraplate rollup`` without the ``--no-input`` flag so the value 227 | for the new variable could be asked interactively. 228 | 2. Manually add the value to 229 | the :ref:`cookiecutter context section ` 230 | so the next ``rollup`` could pick it up. 231 | 232 | Restructuring files 233 | +++++++++++++++++++ 234 | 235 | Scaraplate strategies intentionally don't provide support for anything 236 | more complex than a simple file-to-file change. It means that a scaraplate 237 | template cannot: 238 | 239 | 1. Delete or move files in the target project; 240 | 2. Take multiple files and union them. 241 | 242 | The reason is simple: such operations are always the one-time ones so it 243 | is just easier to perform them manually once than to maintain that logic 244 | in the template. 245 | 246 | 247 | Patterns 248 | -------- 249 | 250 | This section contains some patterns which might be helpful for 251 | creating and maintaining a scaraplate template. 252 | 253 | .. _feature_flags: 254 | 255 | Feature flags 256 | +++++++++++++ 257 | 258 | Let's say you have a template which you have applied to dozens of your 259 | projects. 260 | 261 | And now you want to start gradually introducing a new feature, let it 262 | be a new linter. 263 | 264 | You probably would not want to start using the new thing everywhere at once. 265 | Instead, usually you start with one or two projects, gain experience 266 | and then start rolling it up on the other projects. 267 | 268 | For that you can use template variables as feature flags. 269 | The :ref:`example template ` contains 270 | a ``mypy_enabled`` variable which demonstrates this concept. Basically 271 | it is a regular cookiecutter variable, which can take different values 272 | in the target projects and thus affect the template by enabling or disabling 273 | the new feature. 274 | 275 | Include files 276 | +++++++++++++ 277 | 278 | Consider ``Makefile``. On one hand, you would definitely want to have some 279 | make targets to come from the template; on the other hand, you might need 280 | to introduce custom make targets in some projects. Coming up with a scaraplate 281 | strategy which could merge such a file would be quite difficult. 282 | 283 | Fortunately, ``Makefile`` allows to include other files. So the solution 284 | is quite trivial: have ``Makefile`` synced from the template (with 285 | the :class:`scaraplate.strategies.Overwrite` strategy), and include 286 | a ``Makefile.inc`` file from there which will not be overwritten by the template. 287 | This concept is demonstrated in the :ref:`example template `. 288 | 289 | Manual merging 290 | ++++++++++++++ 291 | 292 | Sometimes you need to merge some files which might be modified in the target 293 | projects and for which there's no suitable strategy yet. In this case 294 | you can use :class:`scaraplate.strategies.TemplateHash` strategy as 295 | a temporary solution: it would overwrite the file each time a new 296 | git commit is added to the template, but keep the file unchanged since 297 | the last rollup of the same template commit. 298 | 299 | The :ref:`example template ` uses this 300 | approach for ``setup.py``. 301 | 302 | Create files conditionally 303 | ++++++++++++++++++++++++++ 304 | 305 | :doc:`Cookiecutter hooks ` can be used 306 | to post-process the generated :ref:`temporary project `. 307 | For example, you might want to skip some files from the template 308 | depending on the variables. 309 | 310 | The :ref:`example template ` contains 311 | an example hook which deletes ``mypy.ini`` file when the ``mypy_enabled`` 312 | variable is not set to ``1``. 313 | -------------------------------------------------------------------------------- /tests/test_rollup.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import json 3 | import os 4 | from pathlib import Path 5 | from typing import Dict, Optional 6 | from unittest.mock import ANY, sentinel 7 | 8 | import pytest 9 | 10 | from scaraplate.config import ScaraplateYamlOptions, ScaraplateYamlStrategies 11 | from scaraplate.cookiecutter import ScaraplateConf 12 | from scaraplate.rollup import ( 13 | InvalidScaraplateTemplateError, 14 | get_project_dest, 15 | get_strategy, 16 | get_target_project_cookiecutter_context, 17 | get_template_root_and_dir, 18 | rollup, 19 | ) 20 | 21 | 22 | @contextlib.contextmanager 23 | def with_working_directory(target_dir: Path): 24 | cur = os.getcwd() 25 | os.chdir(target_dir) 26 | try: 27 | yield 28 | finally: 29 | os.chdir(cur) 30 | 31 | 32 | @pytest.mark.parametrize("apply_count", [1, 2]) 33 | def test_rollup_fuzzy(tempdir_path, apply_count, init_git_and_commit): 34 | template_path = tempdir_path / "template" 35 | target_project_path = tempdir_path / "test" 36 | 37 | # Prepare template 38 | cookiecutter_path = template_path / "{{cookiecutter.project_dest}}" 39 | cookiecutter_path.mkdir(parents=True) 40 | (cookiecutter_path / "README.md").write_text( 41 | "{{ cookiecutter.project_dest }} mock!" 42 | ) 43 | (cookiecutter_path / "setup.py").write_text("#!/usr/bin/env python\n") 44 | (cookiecutter_path / "setup.py").chmod(0o755) 45 | (cookiecutter_path / "sense_vars").write_text("{{ cookiecutter|jsonify }}\n") 46 | (cookiecutter_path / ".scaraplate.conf").write_text( 47 | """[cookiecutter_context] 48 | {%- for key, value in cookiecutter.items()|sort %} 49 | {%- if key not in ('_output_dir',) %} 50 | {{ key }} = {{ value }} 51 | {%- endif %} 52 | {%- endfor %} 53 | """ 54 | ) 55 | (template_path / "cookiecutter.json").write_text('{"project_dest": "test"}') 56 | (template_path / "scaraplate.yaml").write_text( 57 | """ 58 | default_strategy: scaraplate.strategies.Overwrite 59 | strategies_mapping: 60 | setup.py: scaraplate.strategies.TemplateHash 61 | """ 62 | ) 63 | init_git_and_commit(template_path) 64 | 65 | # Apply template (possibly multiple times) 66 | for i in range(apply_count): 67 | rollup( 68 | template_dir=str(template_path), 69 | target_project_dir=str(target_project_path), 70 | no_input=True, 71 | ) 72 | 73 | assert "test mock!" == (target_project_path / "README.md").read_text() 74 | assert 0o755 == (0o777 & (target_project_path / "setup.py").stat().st_mode) 75 | 76 | with open((target_project_path / "sense_vars"), "rt") as f: 77 | assert json.load(f) == { 78 | "_output_dir": ANY, 79 | "_repo_dir": ANY, 80 | "_template": "template", 81 | "project_dest": "test", 82 | } 83 | 84 | 85 | def test_add_remove_template_var(tempdir_path, init_git_and_commit): 86 | template_path = tempdir_path / "template" 87 | target_project_path = tempdir_path / "test" 88 | 89 | # Prepare template 90 | cookiecutter_path = template_path / "{{cookiecutter.project_dest}}" 91 | cookiecutter_path.mkdir(parents=True) 92 | (cookiecutter_path / ".scaraplate.conf").write_text( 93 | """[cookiecutter_context] 94 | {%- for key, value in cookiecutter.items()|sort %} 95 | {%- if key not in ('_output_dir',) %} 96 | {{ key }} = {{ value }} 97 | {%- endif %} 98 | {%- endfor %} 99 | """ 100 | ) 101 | (template_path / "cookiecutter.json").write_text( 102 | '{"project_dest": "test", "removed_var": 42}' 103 | ) 104 | (template_path / "scaraplate.yaml").write_text( 105 | """ 106 | default_strategy: scaraplate.strategies.Overwrite 107 | strategies_mapping: {} 108 | """ 109 | ) 110 | init_git_and_commit(template_path) 111 | 112 | rollup( 113 | template_dir=str(template_path), 114 | target_project_dir=str(target_project_path), 115 | no_input=True, 116 | ) 117 | assert (target_project_path / ".scaraplate.conf").read_text() == ( 118 | """[cookiecutter_context] 119 | _repo_dir = template 120 | _template = template 121 | project_dest = test 122 | removed_var = 42 123 | """ 124 | ) 125 | 126 | # Remove `removed_var` and add `added_var` 127 | (template_path / "cookiecutter.json").write_text( 128 | '{"project_dest": "test", "added_var": 24}' 129 | ) 130 | rollup( 131 | template_dir=str(template_path), 132 | target_project_dir=str(target_project_path), 133 | no_input=True, 134 | ) 135 | assert (target_project_path / ".scaraplate.conf").read_text() == ( 136 | """[cookiecutter_context] 137 | _repo_dir = template 138 | _template = template 139 | added_var = 24 140 | project_dest = test 141 | """ 142 | ) 143 | 144 | 145 | @pytest.mark.parametrize("create_empty", [False, True]) 146 | def test_invalid_template_is_raised_for_missing_cookiecutter_context( 147 | create_empty, tempdir_path, init_git_and_commit 148 | ): 149 | template_path = tempdir_path / "template" 150 | target_project_path = tempdir_path / "test" 151 | 152 | # Prepare template 153 | cookiecutter_path = template_path / "{{cookiecutter.project_dest}}" 154 | cookiecutter_path.mkdir(parents=True) 155 | if create_empty: 156 | (cookiecutter_path / ".scaraplate.conf").write_text("") 157 | (template_path / "cookiecutter.json").write_text('{"project_dest": "test"}') 158 | (template_path / "scaraplate.yaml").write_text( 159 | """ 160 | default_strategy: scaraplate.strategies.Overwrite 161 | strategies_mapping: {} 162 | """ 163 | ) 164 | init_git_and_commit(template_path) 165 | 166 | with pytest.raises(InvalidScaraplateTemplateError) as excinfo: 167 | rollup( 168 | template_dir=str(template_path), 169 | target_project_dir=str(target_project_path), 170 | no_input=True, 171 | ) 172 | assert "cookiecutter context file `.scaraplate.conf` " in str(excinfo.value) 173 | 174 | 175 | def test_extra_context(tempdir_path, init_git_and_commit): 176 | template_path = tempdir_path / "template" 177 | target_project_path = tempdir_path / "test" 178 | 179 | # Prepare template 180 | cookiecutter_path = template_path / "{{cookiecutter.project_dest}}" 181 | cookiecutter_path.mkdir(parents=True) 182 | (cookiecutter_path / "sense_vars").write_text("{{ cookiecutter|jsonify }}\n") 183 | (cookiecutter_path / ".scaraplate.conf").write_text( 184 | """[cookiecutter_context] 185 | {%- for key, value in cookiecutter.items()|sort %} 186 | {%- if key not in ('_output_dir',) %} 187 | {{ key }} = {{ value }} 188 | {%- endif %} 189 | {%- endfor %} 190 | """ 191 | ) 192 | (template_path / "cookiecutter.json").write_text( 193 | '{"project_dest": "test", "key1": null, "key2": null}' 194 | ) 195 | (template_path / "scaraplate.yaml").write_text( 196 | """ 197 | default_strategy: scaraplate.strategies.Overwrite 198 | strategies_mapping: {} 199 | """ 200 | ) 201 | init_git_and_commit(template_path) 202 | 203 | # Initial rollup 204 | rollup( 205 | template_dir=str(template_path), 206 | target_project_dir=str(target_project_path), 207 | no_input=True, 208 | extra_context={"key1": "initial1", "key2": "initial2"}, 209 | ) 210 | with open((target_project_path / "sense_vars"), "rt") as f: 211 | assert json.load(f) == { 212 | "_output_dir": ANY, 213 | "_repo_dir": ANY, 214 | "_template": "template", 215 | "project_dest": "test", 216 | "key1": "initial1", 217 | "key2": "initial2", 218 | } 219 | 220 | # A second rollup with a different context 221 | rollup( 222 | template_dir=str(template_path), 223 | target_project_dir=str(target_project_path), 224 | no_input=True, 225 | extra_context={"key1": "second1", "key2": "second2"}, 226 | ) 227 | with open((target_project_path / "sense_vars"), "rt") as f: 228 | assert json.load(f) == { 229 | "_output_dir": ANY, 230 | "_repo_dir": ANY, 231 | "_template": "template", 232 | "project_dest": "test", 233 | "key1": "second1", 234 | "key2": "second2", 235 | } 236 | 237 | 238 | def test_rollup_with_jinja2_mapping(tempdir_path, init_git_and_commit): 239 | template_path = tempdir_path / "template" 240 | target_project_path = tempdir_path / "test" 241 | 242 | # Prepare template 243 | cookiecutter_path = template_path / "{{cookiecutter.project_dest}}" 244 | cookiecutter_path.mkdir(parents=True) 245 | (cookiecutter_path / "sense_vars").write_text("{{ cookiecutter|jsonify }}\n") 246 | (cookiecutter_path / ".scaraplate.conf").write_text( 247 | """[cookiecutter_context] 248 | {%- for key, value in cookiecutter.items()|sort %} 249 | {%- if key not in ('_output_dir',) %} 250 | {{ key }} = {{ value }} 251 | {%- endif %} 252 | {%- endfor %} 253 | """ 254 | ) 255 | (template_path / "cookiecutter.json").write_text( 256 | '{"project_dest": "test", "file1": null, "file2": "boop"}' 257 | ) 258 | (template_path / "scaraplate.yaml").write_text( 259 | """ 260 | default_strategy: scaraplate.strategies.SortedUniqueLines 261 | strategies_mapping: 262 | '{{ cookiecutter.file1 }}.txt': scaraplate.strategies.IfMissing 263 | '{{ cookiecutter.file2 }}.txt': scaraplate.strategies.Overwrite 264 | """ 265 | ) 266 | (cookiecutter_path / "{{ cookiecutter.file1 }}.txt").write_text("template!") 267 | (cookiecutter_path / "{{ cookiecutter.file2 }}.txt").write_text("template!") 268 | init_git_and_commit(template_path) 269 | 270 | # Initial rollup 271 | rollup( 272 | template_dir=str(template_path), 273 | target_project_dir=str(target_project_path), 274 | no_input=True, 275 | extra_context={"file1": "beep"}, 276 | ) 277 | assert "template!" == (target_project_path / "beep.txt").read_text() 278 | assert "template!" == (target_project_path / "boop.txt").read_text() 279 | (target_project_path / "beep.txt").write_text("target!") 280 | (target_project_path / "boop.txt").write_text("target!") 281 | 282 | # A second rollup (beep.txt should not be changed) 283 | rollup( 284 | template_dir=str(template_path), 285 | target_project_dir=str(target_project_path), 286 | no_input=True, 287 | extra_context={"file1": "beep"}, 288 | ) 289 | assert "target!" == (target_project_path / "beep.txt").read_text() 290 | assert "template!" == (target_project_path / "boop.txt").read_text() 291 | 292 | 293 | def test_get_project_dest(tempdir_path: Path) -> None: 294 | target = tempdir_path / "myproject" 295 | with with_working_directory(tempdir_path): 296 | assert "myproject" == get_project_dest(Path("myproject")) 297 | 298 | target.mkdir() 299 | with with_working_directory(target): 300 | assert "myproject" == get_project_dest(Path(".")) 301 | 302 | 303 | def test_get_template_root_and_dir(tempdir_path: Path) -> None: 304 | target = tempdir_path / "myproject" 305 | target.mkdir() 306 | 307 | with with_working_directory(tempdir_path): 308 | assert (tempdir_path, "myproject") == get_template_root_and_dir( 309 | Path("myproject") 310 | ) 311 | assert (tempdir_path, "myproject") == get_template_root_and_dir( 312 | tempdir_path / "myproject" 313 | ) 314 | 315 | with with_working_directory(target): 316 | assert (tempdir_path, "myproject") == get_template_root_and_dir(Path(".")) 317 | 318 | 319 | @pytest.mark.parametrize( 320 | "contents, expected_context", 321 | [ 322 | (None, {}), 323 | ("", {}), 324 | ( 325 | """ 326 | [cookiecutter_context] 327 | metadata_author = Usermodel @ Rambler&Co 328 | coverage_fail_under = 90 329 | project_monorepo_name = 330 | """, 331 | { 332 | "metadata_author": "Usermodel @ Rambler&Co", 333 | "coverage_fail_under": "90", 334 | "project_monorepo_name": "", 335 | }, 336 | ), 337 | ], 338 | ) 339 | def test_get_target_project_cookiecutter_context( 340 | tempdir_path: Path, contents: Optional[str], expected_context: Dict 341 | ) -> None: 342 | if contents is not None: 343 | (tempdir_path / ".scaraplate.conf").write_text(contents) 344 | 345 | scaraplate_yaml_options = ScaraplateYamlOptions( 346 | git_remote_type=None, cookiecutter_context_type=ScaraplateConf 347 | ) 348 | 349 | assert expected_context == get_target_project_cookiecutter_context( 350 | tempdir_path, scaraplate_yaml_options 351 | ) 352 | 353 | 354 | def test_get_strategy(): 355 | scaraplate_yaml_strategies = ScaraplateYamlStrategies( 356 | default_strategy=sentinel.default, 357 | strategies_mapping={ 358 | "Jenkinsfile": sentinel.jenkinsfile, 359 | "some/nested/setup.py": sentinel.nested_setup_py, 360 | "src/*/__init__.py": sentinel.glob_init, 361 | }, 362 | ) 363 | 364 | assert sentinel.default is get_strategy(scaraplate_yaml_strategies, Path("readme")) 365 | assert sentinel.jenkinsfile is get_strategy( 366 | scaraplate_yaml_strategies, Path("Jenkinsfile") 367 | ) 368 | assert sentinel.nested_setup_py is get_strategy( 369 | scaraplate_yaml_strategies, Path("some/nested/setup.py") 370 | ) 371 | assert sentinel.glob_init is get_strategy( 372 | scaraplate_yaml_strategies, Path("src/my_project/__init__.py") 373 | ) 374 | -------------------------------------------------------------------------------- /src/scaraplate/automation/git.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import datetime 3 | import logging 4 | import os 5 | import tempfile 6 | import uuid 7 | from pathlib import Path 8 | from typing import Dict, Iterator, List, Optional 9 | from urllib.parse import urlparse, urlunparse 10 | 11 | from pkg_resources import get_distribution 12 | 13 | from scaraplate.automation.base import ProjectVCS, TemplateVCS 14 | from scaraplate.config import get_scaraplate_yaml_options 15 | from scaraplate.template import TemplateMeta, _call_git, get_template_meta_from_git 16 | 17 | __all__ = ("GitCloneProjectVCS", "GitCloneTemplateVCS") 18 | 19 | logger = logging.getLogger("scaraplate") 20 | 21 | 22 | def scaraplate_version() -> str: 23 | return get_distribution("scaraplate").version 24 | 25 | 26 | class GitCloneTemplateVCS(TemplateVCS): 27 | """A ready to use :class:`.TemplateVCS` implementation which: 28 | 29 | - Uses git 30 | - Clones a git repo with the template to a temporary directory 31 | (which is cleaned up afterwards) 32 | - Allows to specify an inner dir inside the git repo as the template 33 | root (which is useful for monorepos) 34 | """ 35 | 36 | def __init__(self, template_path: Path, template_meta: TemplateMeta) -> None: 37 | self._template_path = template_path 38 | self._template_meta = template_meta 39 | 40 | @property 41 | def dest_path(self) -> Path: 42 | return self._template_path 43 | 44 | @property 45 | def template_meta(self) -> TemplateMeta: 46 | return self._template_meta 47 | 48 | @classmethod 49 | @contextlib.contextmanager 50 | def clone( 51 | cls, 52 | clone_url: str, 53 | *, 54 | clone_ref: Optional[str] = None, 55 | monorepo_inner_path: Optional[Path] = None, 56 | ) -> Iterator["GitCloneTemplateVCS"]: 57 | """Provides an instance of this class by issuing ``git clone`` 58 | to a tempdir when entering the context manager. Returns a context 59 | manager object which after ``__enter__`` returns an instance 60 | of this class. 61 | 62 | :param clone_url: Any valid ``git clone`` url. 63 | :param clone_ref: Git ref to checkout after clone 64 | (i.e. branch or tag name). 65 | :param monorepo_inner_path: Path to the root dir of template 66 | relative to the root of the repo. If ``None``, the root of 67 | the repo will be used as the root of template. 68 | """ 69 | 70 | with tempfile.TemporaryDirectory() as tmpdir_name: 71 | tmpdir_path = Path(tmpdir_name).resolve() 72 | template_path = tmpdir_path / "scaraplate_template" 73 | template_path.mkdir() 74 | 75 | git = Git.clone( 76 | clone_url, 77 | target_path=template_path, 78 | ref=clone_ref, 79 | # We need to strip credentials from the clone_url, 80 | # because otherwise urls generated for TemplateMeta 81 | # would contain them, and we don't want that. 82 | strip_credentials_from_remote=True, 83 | ) 84 | template_path = git.cwd 85 | 86 | if monorepo_inner_path is not None: 87 | template_path = template_path / monorepo_inner_path 88 | 89 | scaraplate_yaml_options = get_scaraplate_yaml_options(template_path) 90 | template_meta = get_template_meta_from_git( 91 | template_path, git_remote_type=scaraplate_yaml_options.git_remote_type 92 | ) 93 | if clone_ref is not None: 94 | assert clone_ref == template_meta.head_ref 95 | 96 | yield cls(template_path, template_meta) 97 | 98 | 99 | class GitCloneProjectVCS(ProjectVCS): 100 | """A ready to use :class:`.ProjectVCS` implementation which: 101 | 102 | - Uses git 103 | - Clones a git repo with the project to a temporary directory 104 | (which is cleaned up afterwards) 105 | - Allows to specify an inner dir inside the git repo as the project 106 | root (which is useful for monorepos) 107 | - Implements :meth:`.ProjectVCS.commit_changes` as 108 | ``git commit`` + ``git push``. 109 | """ 110 | 111 | def __init__( 112 | self, 113 | project_path: Path, 114 | git: "Git", 115 | *, 116 | changes_branch: str, 117 | commit_author: str, 118 | commit_message_template: str, 119 | ) -> None: 120 | self._project_path = project_path 121 | self._git = git 122 | self.changes_branch = changes_branch 123 | self.commit_author = commit_author 124 | self.commit_message_template = commit_message_template 125 | self.update_time = datetime.datetime.now() 126 | 127 | @property 128 | def dest_path(self) -> Path: 129 | return self._project_path 130 | 131 | def is_dirty(self) -> bool: 132 | return self._git.is_dirty() 133 | 134 | def commit_changes(self, template_meta: TemplateMeta) -> None: 135 | assert self.is_dirty() 136 | 137 | remote_branch = self._git.remote_ref(self.changes_branch) 138 | 139 | # Create a definitely not existing local branch: 140 | local_branch = f"{self.changes_branch}{uuid.uuid4()}" 141 | self._git.checkout_branch(local_branch) 142 | 143 | self._git.commit_all( 144 | self.format_commit_message(template_meta=template_meta), 145 | author=self.commit_author, 146 | ) 147 | 148 | if not self._git.is_existing_ref(remote_branch): 149 | self._git.push(self.changes_branch) 150 | else: 151 | # A branch with updates already exists in the remote. 152 | 153 | if self._git.is_same_commit(remote_branch, f"{local_branch}^1"): 154 | # The `changes_branch` is the same as the clone branch, 155 | # so essentially the created commit forms a linear history. 156 | # No need for any diffs here, we just need to push that. 157 | self._git.push(self.changes_branch) 158 | else: 159 | # The two branches have diverged, we need to compare them: 160 | changes: bool = not self._git.are_one_commit_diffs_equal( 161 | local_branch, remote_branch 162 | ) 163 | 164 | if changes: 165 | # We could've used force push here, but instead we delete 166 | # the branch first, because in GitLab it would also close 167 | # the existing MR (if any), and we want that instead of 168 | # silently updating the old MR. 169 | self._git.push_branch_delete(self.changes_branch) 170 | self._git.push(self.changes_branch) 171 | else: 172 | logger.info( 173 | "scaraplate did update the project, but there's " 174 | "an already existing branch in remote which diff " 175 | "is equal to the just produced changes" 176 | ) 177 | 178 | # Now we should ensure that a Pull Request exists for 179 | # the `self.changes_branch`, but this class is designed to be agnostic 180 | # from concrete git remotes, so it should be done in a child class. 181 | 182 | def format_commit_message(self, *, template_meta: TemplateMeta) -> str: 183 | return self.commit_message_template.format( 184 | # TODO retrieve path from self.clone_url and pass it here too? 185 | # (but careful: that clone_url might contain credentials). 186 | update_time=self.update_time, 187 | scaraplate_version=scaraplate_version(), 188 | template_meta=template_meta, 189 | ) 190 | 191 | @classmethod 192 | @contextlib.contextmanager 193 | def clone( 194 | cls, 195 | clone_url: str, 196 | *, 197 | clone_ref: Optional[str] = None, 198 | monorepo_inner_path: Optional[Path] = None, 199 | changes_branch: str, 200 | commit_author: str, 201 | commit_message_template: str = ( 202 | "Scheduled template update ({update_time:%Y-%m-%d})\n" 203 | "\n" 204 | "* scaraplate version: {scaraplate_version}\n" 205 | "* template commit: {template_meta.commit_url}\n" 206 | "* template ref: {template_meta.head_ref}\n" 207 | ), 208 | ) -> Iterator["GitCloneProjectVCS"]: 209 | """Provides an instance of this class by issuing ``git clone`` 210 | to a tempdir when entering the context manager. Returns a context 211 | manager object which after ``__enter__`` returns an instance 212 | of this class. 213 | 214 | :param clone_url: Any valid ``git clone`` url. 215 | :param clone_ref: Git ref to checkout after clone 216 | (i.e. branch or tag name). 217 | :param monorepo_inner_path: Path to the root dir of project 218 | relative to the root of the repo. If ``None``, the root of 219 | the repo will be used as the root of project. 220 | :param changes_branch: The branch name where the changes should be 221 | pushed in the remote. Might be the same as ``clone_ref``. 222 | Note that this branch is never force-pushed. If upon push 223 | the branch already exists in remote and its one-commit diff 224 | is different from the one-commit diff of the just created 225 | local branch, then the remote branch will be deleted and 226 | the local branch will be pushed to replace the previous one. 227 | :param commit_author: Author name to use for ``git commit``, e.g. 228 | ``John Doe ``. 229 | :param commit_message_template: :meth:`str.format` template 230 | which is used to produce a commit message when committing 231 | the changes. Available format variables are: 232 | 233 | - ``update_time`` [:class:`datetime.datetime`] -- the time 234 | of update 235 | - ``scaraplate_version`` [:class:`str`] -- scaraplate package 236 | version 237 | - ``template_meta`` [:class:`.TemplateMeta`] -- template meta 238 | returned by :meth:`.TemplateVCS.template_meta` 239 | """ 240 | 241 | with tempfile.TemporaryDirectory() as tmpdir_name: 242 | tmpdir_path = Path(tmpdir_name).resolve() 243 | project_path = tmpdir_path / "scaraplate_project" 244 | project_path.mkdir() 245 | 246 | git = Git.clone(clone_url, target_path=project_path, ref=clone_ref) 247 | project_path = git.cwd 248 | 249 | if monorepo_inner_path is not None: 250 | project_path = project_path / monorepo_inner_path 251 | 252 | yield cls( 253 | project_path, 254 | git, 255 | changes_branch=changes_branch, 256 | commit_author=commit_author, 257 | commit_message_template=commit_message_template, 258 | ) 259 | 260 | 261 | class Git: 262 | def __init__(self, cwd: Path, remote: str = "origin") -> None: 263 | self.cwd = cwd 264 | self.remote = remote 265 | 266 | def remote_ref(self, ref: str) -> str: 267 | return f"{self.remote}/{ref}" 268 | 269 | def checkout_branch(self, branch: str) -> None: 270 | self._git(["checkout", "-b", branch]) 271 | 272 | def commit_all(self, commit_message: str, author: Optional[str] = None) -> None: 273 | self._git(["add", "--all"]) 274 | extra: List[str] = [] 275 | if author is not None: 276 | extra = ["--author", author] 277 | self._git( 278 | ["commit", "-m", commit_message, *extra], 279 | env={ 280 | # git would fail if there's no `user.email` in the local 281 | # git config, even if `--author` is specified. 282 | "EMAIL": "scaraplate@localhost", 283 | "GIT_AUTHOR_EMAIL": "scaraplate@localhost", 284 | "GIT_AUTHOR_NAME": "scaraplate", 285 | "GIT_COMMITTER_EMAIL": "scaraplate@localhost", 286 | "GIT_COMMITTER_NAME": "scaraplate", 287 | "USERNAME": "scaraplate", 288 | }, 289 | ) 290 | 291 | def is_dirty(self) -> bool: 292 | return bool(self._git(["status", "--porcelain"])) 293 | 294 | def is_existing_ref(self, ref: str) -> bool: 295 | try: 296 | self._git(["rev-parse", "--verify", ref]) 297 | except RuntimeError: 298 | return False 299 | else: 300 | return True 301 | 302 | def is_same_commit(self, ref1: str, ref2: str) -> bool: 303 | commit1 = self._git(["rev-parse", "--verify", ref1]) 304 | commit2 = self._git(["rev-parse", "--verify", ref2]) 305 | return commit1 == commit2 306 | 307 | def are_one_commit_diffs_equal(self, ref1: str, ref2: str) -> bool: 308 | diff1 = self._git(["diff", f"{ref1}^1..{ref1}"]) 309 | diff2 = self._git(["diff", f"{ref2}^1..{ref2}"]) 310 | return diff1 == diff2 311 | 312 | def push_branch_delete(self, branch: str) -> None: 313 | self._git(["push", "--delete", self.remote, branch]) 314 | 315 | def push(self, ref: str) -> None: 316 | # https://stackoverflow.com/a/4183856 317 | self._git(["push", self.remote, f"HEAD:{ref}"]) 318 | 319 | def _git(self, args: List[str], *, env: Optional[Dict[str, str]] = None) -> str: 320 | return _call_git(args, cwd=self.cwd, env=env) 321 | 322 | @classmethod 323 | def clone( 324 | cls, 325 | clone_url: str, 326 | *, 327 | target_path: Path, 328 | ref: Optional[str] = None, 329 | strip_credentials_from_remote: bool = False, 330 | ) -> "Git": 331 | remote = "origin" 332 | clone_url_without_creds = strip_credentials_from_git_remote(clone_url) 333 | 334 | args = ["clone", clone_url] 335 | 336 | if ref is not None: 337 | # git-clone(1) explicitly mentions that both branches and tags 338 | # are allowed in the `--branch`. 339 | args.extend(["--branch", ref]) 340 | 341 | _call_git(args, cwd=target_path) 342 | 343 | actual_items_in_target_path = os.listdir(target_path) 344 | if len(actual_items_in_target_path) != 1: 345 | raise RuntimeError( 346 | f"Expected `git clone` to create exactly one directory. " 347 | f"Directories in the target: {actual_items_in_target_path}" 348 | ) 349 | 350 | (cloned_dir,) = actual_items_in_target_path 351 | target_path = target_path / cloned_dir 352 | 353 | if strip_credentials_from_remote: 354 | _call_git( 355 | ["remote", "set-url", remote, clone_url_without_creds], cwd=target_path 356 | ) 357 | 358 | return cls(cwd=target_path, remote=remote) 359 | 360 | 361 | def strip_credentials_from_git_remote(remote: str) -> str: 362 | parsed = urlparse(remote) 363 | if not parsed.scheme: 364 | # Not a URL (probably an SSH remote) 365 | return remote 366 | assert parsed.hostname is not None 367 | clean = parsed._replace(netloc=parsed.hostname) 368 | return urlunparse(clean) 369 | --------------------------------------------------------------------------------