├── tsrc ├── test │ ├── __init__.py │ ├── cli │ │ ├── __init__.py │ │ ├── test_main.py │ │ ├── test_cli.py │ │ ├── test_shallow_clones.py │ │ ├── test_apply_manifest.py │ │ ├── test_log.py │ │ └── test_env_setter.py │ ├── helpers │ │ ├── __init__.py │ │ ├── manifest_file.py │ │ ├── cli.py │ │ ├── message_recorder_ext.py │ │ └── test_git_server.py │ ├── test_workspace_config.py │ ├── conftest.py │ ├── test_config.py │ ├── test_file_system.py │ ├── test_executor.py │ ├── test_groups.py │ └── test_git_status.py ├── __init__.py ├── __main__.py ├── config_status_rc.py ├── status_header_dm.py ├── utils.py ├── config_data.py ├── switch.py ├── cli │ ├── apply_manifest.py │ ├── env_setter.py │ ├── init.py │ ├── main.py │ ├── log.py │ └── manifest.py ├── config.py ├── groups_and_constraints_data.py ├── cleaner.py ├── file_system_operator.py ├── workspace_config.py ├── errors.py ├── manifest_common_data.py ├── groups_to_find.py ├── dump_manifest_args_final_output.py ├── repo_grabber.py ├── config_tools.py ├── dump_manifest_args_source_mode.py ├── local_manifest.py ├── local_future_manifest.py ├── remote_setter.py ├── dump_manifest_helper.py ├── file_system.py ├── pcs_repo.py ├── local_tmp_bare_repos.py ├── dump_manifest_args_data.py ├── groups.py ├── git_remote.py ├── config_status.py ├── dump_manifest_args_update_source.py └── dump_manifest_args.py ├── MANIFEST.in ├── .skyspell-ignore ├── custom_theme └── img │ ├── favicon.ico │ └── tsrc-logo.png ├── docs ├── extra.css ├── ref │ ├── sync.md │ ├── workspace-config.md │ ├── cli.md │ └── manifest-config.md ├── guide │ ├── workspace-config.md │ ├── fixed-refs.md │ ├── groups.md │ ├── remotes.md │ ├── ci.md │ ├── fs.md │ ├── manifest.md │ └── foreach.md ├── contrib │ ├── issues.md │ └── dev.md ├── index.md └── faq.md ├── .coveragerc ├── .flake8 ├── .gitignore ├── .copier-answers.yml ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── linters.yml │ ├── doc.yml │ └── tests.yml ├── THANKS ├── tbump.toml ├── mypy.ini ├── tasks.py ├── mkdocs.yml ├── pyproject.toml ├── LICENSE └── README.md /tsrc/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tsrc/test/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tsrc/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "3.0.1" 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include THANKS 3 | -------------------------------------------------------------------------------- /tsrc/__main__.py: -------------------------------------------------------------------------------- 1 | from .cli.main import main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /.skyspell-ignore: -------------------------------------------------------------------------------- 1 | docs/extra.css 2 | *.ico 3 | *.png 4 | *.lock 5 | THANKS 6 | -------------------------------------------------------------------------------- /tsrc/test/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | """ Contains code that is only used by tests. """ 2 | -------------------------------------------------------------------------------- /custom_theme/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/your-tools/tsrc/HEAD/custom_theme/img/favicon.ico -------------------------------------------------------------------------------- /docs/extra.css: -------------------------------------------------------------------------------- 1 | .rst-content dl:not(.docutils) dt{ 2 | border-top: none; 3 | background: none; 4 | } 5 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = tsrc 3 | 4 | omit = 5 | tsrc/test/* 6 | .venv/* 7 | 8 | branch = True 9 | -------------------------------------------------------------------------------- /custom_theme/img/tsrc-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/your-tools/tsrc/HEAD/custom_theme/img/tsrc-logo.png -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = 3 | build 4 | max-line-length = 100 5 | max-complexity = 10 6 | ignore = E203 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.egg-info 3 | .cache 4 | .coverage* 5 | .ci 6 | .tox 7 | htmlcov 8 | dist/ 9 | build/ 10 | site/ 11 | 12 | .mypy_cache/ 13 | 14 | .venv/ 15 | -------------------------------------------------------------------------------- /.copier-answers.yml: -------------------------------------------------------------------------------- 1 | # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY 2 | _commit: v1.1.3-9-gfdd6bb7 3 | _src_path: git+https://git.sr.ht/~dmerej/copier-python 4 | author_email: dimitri@dmerej.info 5 | author_name: Dimitri Merejkowsky 6 | package_import_name: tsrc 7 | project_description: Manage groups of git repositories 8 | project_name: tsrc 9 | project_version: 2.7.0 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a new bug report 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Environment 11 | 12 | * Output of `tsrc version`: ... 13 | * Operating system: ... 14 | 15 | ### Command you ran 16 | 17 | ```console 18 | $ ... 19 | ``` 20 | 21 | ### Actual output 22 | 23 | ```text 24 | ... 25 | ``` 26 | 27 | ### Expected result 28 | -------------------------------------------------------------------------------- /THANKS: -------------------------------------------------------------------------------- 1 | Here's a list of everyone who contributed to this project, one way or an other: 2 | 3 | Alexandre Bossard 4 | Arnaud Gelas 5 | Cédric Gestes 6 | Philippe Daouadi 7 | Dimitri Merejkowsky 8 | Théo Delrieu 9 | Jakob Heuser 10 | Tronje Krabbe 11 | Matthew Lovell 12 | Atte Pellikka 13 | Johann Chang 14 | Albert De La Fuente Vigliotti 15 | Greg Dubicki 16 | 17 | 18 | If you make a contribution, feel free to make a pull request to add your name 19 | here :) 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | This may include new command-line syntax or changes in the schemas of the configuration files 15 | -------------------------------------------------------------------------------- /tsrc/config_status_rc.py: -------------------------------------------------------------------------------- 1 | """Config Status Return Code 2 | 3 | When change is registered to some configuration item, 4 | it does not mean, that there will not be any issue. 5 | 6 | Here is some common Enums of what can be used 7 | as return code. Extend it to your liking. 8 | """ 9 | 10 | from enum import Enum, unique 11 | 12 | 13 | @unique 14 | class ConfigStatusReturnCode(Enum): 15 | SUCCESS = 0 16 | IGNORE = 1 17 | CANCEL = 2 18 | NOT_FOUND = 3 19 | REVERT = 4 20 | ERROR = 5 21 | -------------------------------------------------------------------------------- /tsrc/status_header_dm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Status Header Display Mode 3 | 4 | Keeps information about what data can be displayed 5 | in the status header of some commands 6 | 7 | Currently it is implemented into: 8 | * status 9 | * manifest 10 | """ 11 | 12 | from enum import Flag 13 | 14 | 15 | class StatusHeaderDisplayMode(Flag): 16 | NONE = 0 17 | URL = 1 18 | BRANCH = 2 19 | CONFIG_CHANGE = 4 20 | """CONFIG_CHAGE: this flag should be set automatically, 21 | upon successful call for change in config.""" 22 | -------------------------------------------------------------------------------- /tsrc/test/cli/test_main.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from tsrc.cli.main import main 4 | 5 | 6 | def test_without_args() -> None: 7 | with pytest.raises(SystemExit) as e: 8 | main([]) 9 | assert e.value.code != 0 10 | 11 | 12 | def test_help() -> None: 13 | with pytest.raises(SystemExit) as e: 14 | main(["-h"]) 15 | assert e.value.code == 0 16 | 17 | 18 | def test_version() -> None: 19 | with pytest.raises(SystemExit) as e: 20 | main(["--version"]) 21 | assert e.value.code == 0 22 | -------------------------------------------------------------------------------- /tsrc/utils.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from typing import List 3 | 4 | import cli_ui as ui 5 | 6 | 7 | def erase_last_line() -> None: 8 | terminal_size = shutil.get_terminal_size() 9 | ui.info(" " * terminal_size.columns, end="\r") 10 | 11 | 12 | def len_of_cli_ui(ui_tokens: List[ui.Token]) -> int: 13 | len_: int = 0 14 | for i in ui_tokens: 15 | if isinstance(i, str): 16 | len_ += len(i) + 1 17 | 18 | if len_ > 0: 19 | len_ -= 1 20 | return len_ 21 | 22 | 23 | def align_left(l_just: int, l_str: str) -> List[ui.Token]: 24 | str_: str = "" 25 | if l_just == 1: 26 | str_ = " " 27 | if l_just > 1: 28 | str_ = " ".ljust(l_just) 29 | return [str_ + l_str] 30 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: linters 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | tags: v* 7 | pull_request: 8 | 9 | jobs: 10 | run_linters: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | 16 | - uses: actions/checkout@v3 17 | 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: "3.10" 22 | 23 | - name: Prepare project for development 24 | run: | 25 | python -m pip install poetry 26 | python -m poetry install 27 | 28 | - name: Run linters 29 | run: python -m poetry run invoke lint 30 | 31 | - name: Check that doc builds without warnings 32 | run: python -m poetry run mkdocs build 33 | -------------------------------------------------------------------------------- /tbump.toml: -------------------------------------------------------------------------------- 1 | github_url = "https://github.com/your-tools/tsrc" 2 | 3 | [version] 4 | current = "3.0.1" 5 | regex = ''' 6 | (?P\d+) 7 | \. 8 | (?P\d+) 9 | \. 10 | (?P\d+) 11 | ( 12 | - 13 | (?Palpha|beta|r) 14 | (?P\d+) 15 | )? 16 | ''' 17 | 18 | [git] 19 | tag_template = "v{new_version}" 20 | message_template = "Bump to {new_version}" 21 | 22 | [[file]] 23 | src = "pyproject.toml" 24 | search = 'version = "{current_version}"' 25 | 26 | 27 | [[file]] 28 | src = "tsrc/__init__.py" 29 | search = "__version__ =" 30 | 31 | 32 | [[before_push]] 33 | name = "Check Changelog" 34 | cmd = "grep -q {new_version} docs/changelog.md" 35 | 36 | [[after_push]] 37 | name = "Publish project on Pypi" 38 | cmd = "poetry publish --build" 39 | -------------------------------------------------------------------------------- /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: doc 2 | on: 3 | push: 4 | branches: 5 | - main 6 | 7 | jobs: 8 | deploy_documentation: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | 13 | - name: Set up Python 14 | uses: actions/setup-python@v4 15 | with: 16 | python-version: "3.10" 17 | 18 | - name: Prepare project for development 19 | run: | 20 | python -m pip install poetry 21 | python -m poetry install 22 | 23 | - name: Build documentation 24 | run: | 25 | poetry run mkdocs build 26 | 27 | - name: Deploy to GitHub pages 28 | uses: JamesIves/github-pages-deploy-action@v4.6.3 29 | with: 30 | token: ${{ secrets.GH_PAT }} 31 | branch: gh-pages 32 | folder: site 33 | -------------------------------------------------------------------------------- /tsrc/config_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | Config Data 3 | 4 | Is designed to help identify (ConfigUpdateType) 5 | config chage/update type 6 | and to keep new data for update (ConfigUpdateData) 7 | 8 | All this is so when config is going to be 9 | updated, it can be done in one fell swoop 10 | """ 11 | 12 | from dataclasses import dataclass 13 | from enum import Enum, unique 14 | from typing import List, Optional 15 | 16 | 17 | @unique 18 | class ConfigUpdateType(Enum): 19 | NONE = 0 20 | MANIFEST_BRANCH = 1 21 | REPO_GROUPS = 2 22 | # add other update types when needed 23 | 24 | 25 | @dataclass(frozen=True) 26 | class ConfigUpdateData: 27 | manifest_branch: Optional[str] = None 28 | repo_groups: Optional[List[str]] = None # not yet supported 29 | # add more configuration data when supported 30 | -------------------------------------------------------------------------------- /tsrc/test/test_workspace_config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from tsrc.workspace_config import WorkspaceConfig 4 | 5 | 6 | def test_save(tmp_path: Path) -> None: 7 | """Check that workspace config can be written 8 | and read. 9 | 10 | Note: the writing is done by `tsrc init`, all other 11 | commands simply read the file. 12 | """ 13 | config = WorkspaceConfig( 14 | manifest_url="https://gitlab.example", 15 | manifest_branch="stable", 16 | manifest_branch_0="stable", 17 | shallow_clones=True, 18 | repo_groups=["default", "a-team"], 19 | clone_all_repos=False, 20 | singular_remote=None, 21 | ) 22 | persistent_path = tmp_path / "config.yml" 23 | config.save_to_file(persistent_path) 24 | actual = WorkspaceConfig.from_file(persistent_path) 25 | assert actual == config 26 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | files = tsrc/**/*.py 3 | allow_incomplete_defs = false 4 | allow_subclassing_any = false 5 | allow_untyped_calls = false 6 | allow_untyped_decorators = false 7 | allow_untyped_defs = false 8 | check_untyped_defs = true 9 | enable_error_code = ignore-without-code 10 | ignore_missing_imports = false 11 | no_implicit_optional = true 12 | pretty = true 13 | show_error_codes = true 14 | warn_redundant_casts = true 15 | warn_return_any = true 16 | warn_unused_configs = true 17 | warn_unused_ignores = true 18 | 19 | [mypy-colored_traceback] 20 | ignore_missing_imports = true 21 | 22 | [mypy-path] 23 | ignore_missing_imports = true 24 | 25 | [mypy-pygit2] 26 | ignore_missing_imports = true 27 | 28 | [mypy-schema] 29 | ignore_missing_imports = true 30 | 31 | [mypy-gitlab] 32 | ignore_missing_imports = true 33 | 34 | [mypy-github3] 35 | ignore_missing_imports = true 36 | 37 | [mypy-ruamel] 38 | ignore_missing_imports = true 39 | -------------------------------------------------------------------------------- /tsrc/switch.py: -------------------------------------------------------------------------------- 1 | """Support for switch in manifest 2 | 3 | 'switch' can hold part of new default configuration 4 | and such should only be activated by option '--switch' 5 | when activated: 6 | A) and present: 7 | it should completely overwrite current configuration. 8 | B) but not present: 9 | the default (empty) configuration should be used 10 | 11 | this is particulary usefull when switching 12 | to new Manifest branch, no need to care about 13 | current configuration anymore. 14 | 15 | NOTE: original handling of configuration 16 | should be adhered when no '--switch' is provided 17 | """ 18 | 19 | from typing import Any, List, Optional 20 | 21 | 22 | class Switch: 23 | def __init__(self, switch_config: Any) -> None: 24 | self._config: Optional[Any] = None 25 | self._groups: Optional[List[Any]] = None 26 | if switch_config: 27 | self._config = switch_config.get("config") 28 | if self._config: 29 | self._groups = self._config.get("groups") 30 | -------------------------------------------------------------------------------- /tsrc/cli/apply_manifest.py: -------------------------------------------------------------------------------- 1 | """ Entry point for `tsrc apply-manifest`. """ 2 | 3 | import argparse 4 | from pathlib import Path 5 | 6 | from tsrc.cli import ( 7 | add_num_jobs_arg, 8 | add_workspace_arg, 9 | get_num_jobs, 10 | get_workspace, 11 | repos_from_config, 12 | ) 13 | from tsrc.manifest import load_manifest 14 | 15 | 16 | def configure_parser(subparser: argparse._SubParsersAction) -> None: 17 | parser = subparser.add_parser("apply-manifest") 18 | parser.add_argument("manifest_path", help="path to the local manifest", type=Path) 19 | add_workspace_arg(parser) 20 | add_num_jobs_arg(parser) 21 | parser.set_defaults(run=run) 22 | 23 | 24 | def run(args: argparse.Namespace) -> None: 25 | manifest = load_manifest(args.manifest_path) 26 | num_jobs = get_num_jobs(args) 27 | workspace = get_workspace(args) 28 | workspace.repos = repos_from_config(manifest, workspace.config) 29 | workspace.clone_missing(num_jobs=num_jobs) 30 | workspace.set_remotes(num_jobs=num_jobs) 31 | workspace.perform_filesystem_operations(manifest=manifest) 32 | -------------------------------------------------------------------------------- /docs/ref/sync.md: -------------------------------------------------------------------------------- 1 | # Sync algorithm 2 | 3 | You may have noticed that `tsrc sync` does not just calls `git pull` on every repository. 4 | 5 | Here's the algorithm that is used: 6 | 7 | 8 | * Run `git fetch --tags --prune` 9 | * Check if the repository is on a branch 10 | * Check if the currently checked out branch matches the one configured 11 | in the manifest. If it does not, the repository is clean and the 12 | `--no-correct-branch` flag is NOT set, the branch is changed to the 13 | configured one. 14 | * Check if the repository is dirty 15 | * Try and run a fast-forward merge 16 | 17 | Note that: 18 | 19 | * `git fetch` is always called so that local refs are up-to-date 20 | * `tsrc` will simply print an error and move on to the next repository if the 21 | fast-forward merge is not possible. That's because `tsrc` cannot guess 22 | what the correct action is, so it prefers doing nothing. It's up 23 | to the user to run something like `git merge` or `git rebase`. 24 | * in case the repository is on an incorrect branch, the fast-forward merge will 25 | still be attempted, but an error message will be show in the end 26 | 27 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | # type: ignore 2 | from invoke import call, task 3 | 4 | SOURCES = "tsrc" 5 | 6 | 7 | @task 8 | def black(c, check=False): 9 | print("Running black") 10 | cmd = f"black {SOURCES}" 11 | if check: 12 | cmd += " --check" 13 | c.run(cmd) 14 | 15 | 16 | @task 17 | def isort(c, check=False): 18 | print("Running isort") 19 | cmd = f"isort {SOURCES}" 20 | if check: 21 | cmd += " --check" 22 | c.run(cmd) 23 | 24 | 25 | @task 26 | def flake8(c): 27 | print("Running flake8") 28 | c.run(f"flake8 {SOURCES}") 29 | 30 | 31 | @task 32 | def mypy(c, machine_readable=False): 33 | print("Running mypy") 34 | cmd = "mypy" 35 | if machine_readable: 36 | cmd += " --no-pretty" 37 | else: 38 | cmd += " --color-output --pretty" 39 | c.run(cmd) 40 | 41 | 42 | @task 43 | def test(c): 44 | print("Running pytest") 45 | c.run("pytest -n auto", pty=True) 46 | 47 | 48 | @task( 49 | pre=[ 50 | call(black, check=True), 51 | call(isort, check=True), 52 | call(flake8), 53 | call(mypy), 54 | ] 55 | ) 56 | def lint(c): 57 | pass 58 | -------------------------------------------------------------------------------- /tsrc/config.py: -------------------------------------------------------------------------------- 1 | """ Parse tsrc config files """ 2 | 3 | from pathlib import Path 4 | from typing import Any, Dict, NewType 5 | 6 | import ruamel.yaml 7 | from schema import Schema, SchemaError 8 | 9 | from tsrc.errors import InvalidConfigError 10 | 11 | Config = NewType("Config", Dict[str, Any]) 12 | 13 | 14 | def parse_config(file_path: Path, *, schema: Schema) -> Config: 15 | """Parse a config given a file path and a schema.""" 16 | 17 | # Note: we try and wrap any raised exception into a generic 18 | # InvalidConfigError error, so that error messages always contains 19 | # the path of the file that caused the error. 20 | try: 21 | contents = file_path.read_text() 22 | except OSError as os_error: 23 | raise InvalidConfigError(file_path, os_error) 24 | try: 25 | yaml = ruamel.yaml.YAML(typ="safe", pure=True) 26 | parsed = yaml.load(contents) 27 | except ruamel.yaml.error.YAMLError as yaml_error: 28 | raise InvalidConfigError(file_path, yaml_error) 29 | try: 30 | schema.validate(parsed) 31 | except SchemaError as schema_error: 32 | raise InvalidConfigError(file_path, schema_error) 33 | return Config(parsed) 34 | -------------------------------------------------------------------------------- /tsrc/groups_and_constraints_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dataclass for Groups, 'include_regex', 'exclude_regex' 3 | 'singular_remote' 4 | 5 | everything that can reduce Repos should have single 6 | place and that place should be here 7 | """ 8 | 9 | import argparse 10 | from dataclasses import dataclass 11 | from typing import List, Optional 12 | 13 | 14 | @dataclass(frozen=True) 15 | class GroupsAndConstraints: 16 | groups: Optional[List[str]] = None # just what was provided via cmd 17 | # not to be mistaken with Group class 18 | singular_remote: str = "" # NOTE possibly unused by now 19 | include_regex: str = "" 20 | exclude_regex: str = "" 21 | 22 | 23 | def get_group_and_constraints_data(args: argparse.Namespace) -> GroupsAndConstraints: 24 | 25 | # NOTE: does not obtains 'singular_remote' 26 | groups: Optional[List[str]] = None 27 | if args.groups: 28 | groups = args.groups 29 | include_regex: str = "" 30 | if args.include_regex: 31 | include_regex = args.include_regex 32 | exclude_regex: str = "" 33 | if args.exclude_regex: 34 | exclude_regex = args.exclude_regex 35 | return GroupsAndConstraints( 36 | groups=groups, include_regex=include_regex, exclude_regex=exclude_regex 37 | ) 38 | -------------------------------------------------------------------------------- /tsrc/cleaner.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List 3 | 4 | import cli_ui as ui 5 | 6 | from tsrc.executor import Outcome, Task 7 | from tsrc.repo import Repo 8 | 9 | 10 | class Cleaner(Task[Repo]): 11 | def __init__( 12 | self, 13 | workspace_path: Path, 14 | *, 15 | do_hard_clean: bool = False, 16 | ) -> None: 17 | self.workspace_path = workspace_path 18 | self.do_hard_clean = do_hard_clean 19 | 20 | def describe_item(self, item: Repo) -> str: 21 | return item.dest 22 | 23 | def describe_process_start(self, item: Repo) -> List[ui.Token]: 24 | return ["Cleaning", item.dest] 25 | 26 | def describe_process_end(self, item: Repo) -> List[ui.Token]: 27 | return [ui.green, "ok", ui.reset, item.dest] 28 | 29 | def process(self, index: int, count: int, repo: Repo) -> Outcome: 30 | """ 31 | Clean each repo so it will be ready for next 'sync' 32 | """ 33 | self.info_count(index, count, "Cleaning", repo.dest) 34 | 35 | repo_path = self.workspace_path / repo.dest 36 | self.run_git(repo_path, "clean", "-f", "-d") 37 | if self.do_hard_clean is True: 38 | self.run_git(repo_path, "clean", "-f", "-X", "-d") 39 | 40 | return Outcome.empty() 41 | -------------------------------------------------------------------------------- /docs/ref/workspace-config.md: -------------------------------------------------------------------------------- 1 | # Workspace configuration 2 | 3 | The workspace configuration lies in `/.tsrc/config.yml`. It is 4 | created by `tsrc init` then read by `tsrc sync` and other commands. It can 5 | be freely edited by hand. 6 | 7 | Here's an example: 8 | 9 | ```yaml 10 | manifest_url: git@acme.corp:manifest.git 11 | manifest_branch: master 12 | shallow_clones: false 13 | repo_groups: 14 | - default 15 | clone_all_repos: false 16 | singular_remote: 17 | ``` 18 | 19 | 20 | * `manifest_url`: an git URL containing a `manifest.yml` file 21 | * `manifest_branch`: the branch to use when updating the local manifest (e.g, the first step of `tsrc sync`) 22 | * `shallow_clones`: whether to use only shallow clones when cloning missing repositories 23 | * `repo_groups`: the list of groups to use - every mentioned group must be present in the `manifest.yml` file (see above) 24 | * `clone_all_repos`: whether to ignore groups entirely and clone every repository from the manifest instead 25 | * `singular_remote`: if set to ``, behaves as if `tsrc sync` and 26 | `tsrc init` were called with `--singular-remote ` option. See the 27 | [Using remotes guide](../guide/remotes.md) for details. If `tsrc sync -r 28 | ` is used, it will take precedence over the file configuration 29 | parameter. 30 | -------------------------------------------------------------------------------- /docs/guide/workspace-config.md: -------------------------------------------------------------------------------- 1 | # Editing workspace configuration 2 | 3 | ## Creation 4 | 5 | The configuration file created by `tsrc init` contains the whole list 6 | of available settings, with their default value, and is 7 | located at ``. 8 | 9 | Note that if you use command-line options when using `tsrc init`, those 10 | will be written in the `.tsrc/config.yml`. 11 | 12 | For instance: 13 | 14 | ``` 15 | tsrc init git@github.com:dmerejkowsky/dummy-manifest 16 | ``` 17 | 18 | generates this file: 19 | 20 | ```yaml 21 | manifest_url: git@github.com:dmerejkowsky/dummy-manifest 22 | manifest_branch: master 23 | repo_groups: [] 24 | shallow_clones: false 25 | clone_all_repos: false 26 | singular_remote: 27 | ``` 28 | 29 | But 30 | 31 | ``` 32 | tsrc init git@github.com:dmerejkowsky/dummy-manifest --branch main 33 | ``` 34 | 35 | generates this instead: 36 | 37 | 38 | ```yaml 39 | manifest_url: git@github.com:dmerejkowsky/dummy-manifest 40 | manifest_branch: main 41 | repo_groups: [] 42 | shallow_clones: false 43 | clone_all_repos: false 44 | singular_remote: 45 | ``` 46 | 47 | ## Editing 48 | 49 | You can edit the workspace configuration as you please, for instance 50 | if you need to switch the manifest branch. 51 | 52 | If you do so, note that your changes will be taken into account 53 | next time you run `tsrc sync`. 54 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | tags: v* 7 | pull_request: 8 | 9 | jobs: 10 | run_tests: 11 | 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] 17 | os: [ubuntu-latest, macos-latest, windows-latest] 18 | 19 | steps: 20 | 21 | - uses: actions/checkout@v3 22 | 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | - name: Prepare project for development 29 | run: | 30 | python -m pip install poetry 31 | python -m poetry install 32 | 33 | - name: Setup tests 34 | run: | 35 | # tests run git commands, and they need 36 | # a proper git identity for that 37 | git config --global user.email "test@tsrc-tests.com" 38 | git config --global user.name "Tasty Test" 39 | 40 | # Note: for some reason pytest --cov fails on Windows 41 | - name: Run tests (unix) 42 | if: ${{ matrix.os != 'windows-latest' }} 43 | run: poetry run pytest --cov . --cov-report term --numprocesses auto 44 | 45 | - name: Run tests (windows) 46 | if: ${{ matrix.os == 'windows-latest' }} 47 | run: poetry run pytest --numprocesses auto 48 | -------------------------------------------------------------------------------- /tsrc/test/conftest.py: -------------------------------------------------------------------------------- 1 | """ Fixtures for tsrc testing. """ 2 | 3 | import os 4 | from pathlib import Path 5 | from typing import Any, Iterator 6 | 7 | import pytest 8 | from cli_ui.tests import MessageRecorder 9 | 10 | from tsrc.test.helpers.cli import tsrc_cli # noqa: F401 11 | from tsrc.test.helpers.git_server import git_server # noqa: F401 12 | from tsrc.test.helpers.message_recorder_ext import MessageRecorderExt 13 | from tsrc.workspace import Workspace 14 | 15 | 16 | @pytest.fixture(scope="session", autouse=True) 17 | def set_test_env() -> None: 18 | os.environ["TSRC_TESTING"] = "true" 19 | 20 | 21 | @pytest.fixture() 22 | def tmp_path(tmpdir: Any) -> Path: 23 | """Convert py.path.Local() to Path() objects.""" 24 | return Path(tmpdir.strpath) 25 | 26 | 27 | @pytest.fixture 28 | def workspace_path(tmp_path: Path) -> Path: 29 | res = tmp_path / "work" 30 | res.mkdir() 31 | return res 32 | 33 | 34 | @pytest.fixture 35 | def workspace(workspace_path: Path) -> Workspace: 36 | return Workspace(workspace_path) 37 | 38 | 39 | @pytest.fixture() 40 | def message_recorder() -> Iterator[MessageRecorder]: 41 | res = MessageRecorder() 42 | res.start() 43 | yield res 44 | res.stop() 45 | 46 | 47 | @pytest.fixture 48 | def message_recorder_ext(request: Any) -> Iterator[MessageRecorderExt]: 49 | recorder = MessageRecorderExt() 50 | recorder.start() 51 | yield recorder 52 | recorder.stop() 53 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | strict: true 2 | 3 | site_name: tsrc documentation 4 | repo_url: https://github.com/your-tools/tsrc/ 5 | 6 | theme: 7 | name: material 8 | custom_dir: custom_theme 9 | favicon: img/tsrc-logo.png 10 | 11 | 12 | markdown_extensions: 13 | - codehilite 14 | - markdown.extensions.admonition #for !!! notice 15 | - markdown.extensions.def_list 16 | - pymdownx.superfences 17 | 18 | - toc: 19 | permalink: True 20 | separator: "_" 21 | 22 | nav: 23 | - Home: index.md 24 | - Getting Started: getting-started.md 25 | 26 | - Guides: 27 | - Editing the manifest safely: guide/manifest.md 28 | - Editing workspace configuration: guide/workspace-config.md 29 | - Using groups: guide/groups.md 30 | - Using several remotes: guide/remotes.md 31 | - Using fixed git references: guide/fixed-refs.md 32 | - Performing file system operations: guide/fs.md 33 | - Running a command for each repo in the workspace: guide/foreach.md 34 | - Using tsrc with continuous integration: guide/ci.md 35 | 36 | - Reference: 37 | - Command line usage: ref/cli.md 38 | - Sync algorithm: ref/sync.md 39 | - Manifest configuration: ref/manifest-config.md 40 | - Workspace configuration: ref/workspace-config.md 41 | 42 | - Contributing: 43 | - Using the issue tracker: contrib/issues.md 44 | - Suggesting changes: contrib/dev.md 45 | - Code manifesto: code-manifesto.md 46 | 47 | - FAQ: faq.md 48 | 49 | - Changelog: changelog.md 50 | -------------------------------------------------------------------------------- /tsrc/test/cli/test_cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from multiprocessing import cpu_count 3 | from typing import List 4 | 5 | import pytest 6 | 7 | from tsrc.cli import add_num_jobs_arg, get_num_jobs 8 | 9 | 10 | class TestNumJobsParsing: 11 | def parse_args(self, args: List[str]) -> int: 12 | parser = argparse.ArgumentParser() 13 | add_num_jobs_arg(parser) 14 | parsed = parser.parse_args(args) 15 | return get_num_jobs(parsed) 16 | 17 | def test_defaults_to_all_cpus(self) -> None: 18 | actual = self.parse_args([]) 19 | assert actual == cpu_count() 20 | 21 | def test_auto_uses_all_cpus(self) -> None: 22 | actual = self.parse_args(["--jobs", "auto"]) 23 | assert actual == cpu_count() 24 | 25 | def test_specify_num_jobs_explicitly(self) -> None: 26 | actual = self.parse_args(["--jobs", "3"]) 27 | assert actual == 3 28 | 29 | def test_using_env_variable( 30 | self, 31 | monkeypatch: pytest.MonkeyPatch, 32 | ) -> None: 33 | monkeypatch.setenv("TSRC_PARALLEL_JOBS", "5") 34 | actual = self.parse_args([]) 35 | 36 | assert actual == 5 37 | 38 | def test_overriding_env_variable_from_command_line( 39 | self, 40 | monkeypatch: pytest.MonkeyPatch, 41 | ) -> None: 42 | monkeypatch.setenv("TSRC_PARALLEL_JOBS", "5") 43 | actual = self.parse_args(["--jobs", "6"]) 44 | 45 | assert actual == 6 46 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.isort] 2 | profile = "black" 3 | 4 | [tool.poetry] 5 | name = "tsrc" 6 | version = "3.0.1" 7 | description = "Manage groups of git repositories" 8 | authors = ["Dimitri Merejkowsky "] 9 | readme = "README.md" 10 | license = "BSD-3-Clause" 11 | repository = "https://github.com/your-tools/tsrc" 12 | documentation = "https://your-tools.github.io/tsrc" 13 | 14 | [tool.poetry.urls] 15 | Changelog = "https://your-tools.github.io/tsrc/changelog/" 16 | Issues = "https://github.com/your-tools/tsrc/issues" 17 | 18 | [tool.poetry.dependencies] 19 | # Note: keep this in sync with .github/workflows/tests.yml 20 | python = "^3.8.1" 21 | 22 | cli-ui = "^0.11.0" 23 | colored_traceback = "^0.3.0" 24 | ruamel-yaml = "^0.18.5" 25 | schema = "^0.7.1" 26 | mypy_extensions = "^1.0.0" 27 | 28 | [tool.poetry.dev-dependencies] 29 | # Task runner 30 | invoke = "^2.2" 31 | 32 | # Tests 33 | pytest = "^7.4" 34 | pytest-cov = "^4.1" 35 | pytest-xdist = "^3.5.0" 36 | pygit2 = "^1.13" 37 | 38 | # Linters 39 | black = "^24.8" 40 | flake8 = "^7.1.1" 41 | flake8-bugbear = "^24.4.0" 42 | flake8-comprehensions = "^3.15.0" 43 | pep8-naming = "^0.14.1" 44 | isort = "^5.13.2" 45 | types-mock = "^0.1.5" 46 | mypy = "^1.11.1" 47 | 48 | # Docs 49 | mkdocs = "^1.5" 50 | mkdocs-material = "^9.5" 51 | 52 | [tool.poetry.scripts] 53 | tsrc = "tsrc.cli.main:main" 54 | 55 | [build-system] 56 | requires = ["poetry-core>=1.0.0"] 57 | build-backend = "poetry.core.masonry.api" 58 | -------------------------------------------------------------------------------- /tsrc/file_system_operator.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List 3 | 4 | import cli_ui as ui 5 | 6 | from tsrc.errors import Error 7 | from tsrc.executor import Outcome, Task 8 | from tsrc.file_system import FileSystemOperation 9 | from tsrc.repo import Repo 10 | 11 | 12 | class FileSystemOperator(Task[FileSystemOperation]): 13 | """Implement file system operations to be run once every missing 14 | repo has been cloned, like copying files or creating symlinks. 15 | 16 | """ 17 | 18 | def __init__(self, workspace_path: Path, repos: List[Repo]) -> None: 19 | self.workspace_path = workspace_path 20 | self.repos = repos 21 | 22 | def describe_item(self, item: FileSystemOperation) -> str: 23 | return item.describe(self.workspace_path) 24 | 25 | def describe_process_start(self, item: FileSystemOperation) -> List[ui.Token]: 26 | return [] 27 | 28 | def describe_process_end(self, item: FileSystemOperation) -> List[ui.Token]: 29 | return [] 30 | 31 | def process(self, index: int, count: int, item: FileSystemOperation) -> Outcome: 32 | # Note: we don't want to run this Task in parallel, just in case 33 | # the order of filesystem operations matters, so we can always 34 | # return an empty Outcome 35 | description = item.describe(self.workspace_path) 36 | self.info_count(index, count, description) 37 | try: 38 | item.perform(self.workspace_path) 39 | except OSError as e: 40 | raise Error(str(e)) 41 | return Outcome.empty() 42 | -------------------------------------------------------------------------------- /docs/contrib/issues.md: -------------------------------------------------------------------------------- 1 | # Using the issue tracker 2 | 3 | Reporting bugs and requesting new features is done one the [tsrc issue tracker on GitHub](https://github.com/your-tools/tsrc/issues). 4 | 5 | ## Reporting bugs 6 | 7 | If you are reporting a bug, please provide the following information: 8 | 9 | * `tsrc` version 10 | * Details about your environment (operating system, Python version) 11 | * The exact command you run 12 | * The full output 13 | 14 | Doing so will ensure we can investigate your bug right away. 15 | 16 | 17 | ## Suggesting new features 18 | 19 | If you think `tsrc` is lacking a feature, please provide the following information: 20 | 21 | * What exactly is your use case? 22 | * Do you need a new command-line option or even a new command? 23 | * Do you need changes in the configuration files? 24 | 25 | Note that changing`tsrc` behavior can get tricky. 26 | 27 | First off, we want to avoid *data loss* following a `tsrc` command above 28 | 29 | Second, we want to keep `tsrc` behavior as least surprising as possible, so that 30 | it can be used without having to read (too much of) documentation. 31 | 32 | To that end, and keeping in mind `tsrc` needs to accommodate a large 33 | variety of use cases, we want to keep the code: 34 | 35 | * easy to read and, 36 | * easy to maintain, 37 | * and very well tested. 38 | 39 | The best way to achieve all of this is to *keep it simple*. 40 | 41 | This means we'll be very cautious before implementing a new feature, so 42 | don't hesitate to open an issue for discussion before jumping into the 43 | development of a new feature. 44 | 45 | 46 | -------------------------------------------------------------------------------- /tsrc/test/cli/test_shallow_clones.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from cli_ui.tests import MessageRecorder 4 | 5 | from tsrc.git import is_shallow 6 | from tsrc.test.helpers.cli import CLI 7 | from tsrc.test.helpers.git_server import GitServer 8 | 9 | 10 | def assert_shallow_clone(workspace_path: Path, repo: str) -> None: 11 | repo_path = workspace_path / repo 12 | assert is_shallow(repo_path) 13 | 14 | 15 | def test_shallow_clones( 16 | tsrc_cli: CLI, git_server: GitServer, workspace_path: Path 17 | ) -> None: 18 | git_server.add_repo("foo/bar") 19 | git_server.add_repo("spam/eggs") 20 | git_server.push_file("foo/bar", "bar.txt", contents="this is bar") 21 | 22 | manifest_url = git_server.manifest_url 23 | tsrc_cli.run("init", "--shallow", manifest_url) 24 | assert_shallow_clone(workspace_path, "foo/bar") 25 | assert_shallow_clone(workspace_path, "spam/eggs") 26 | 27 | git_server.add_repo("foo/baz") 28 | tsrc_cli.run("sync") 29 | assert_shallow_clone(workspace_path, "foo/baz") 30 | 31 | 32 | def test_shallow_with_fix_ref( 33 | tsrc_cli: CLI, 34 | git_server: GitServer, 35 | workspace_path: Path, 36 | message_recorder: MessageRecorder, 37 | ) -> None: 38 | git_server.add_repo("foo") 39 | initial_sha1 = git_server.get_sha1("foo") 40 | git_server.push_file("foo", "one.c") 41 | git_server.manifest.set_repo_sha1("foo", initial_sha1) 42 | 43 | manifest_url = git_server.manifest_url 44 | tsrc_cli.run_and_fail("init", "--shallow", manifest_url) 45 | assert message_recorder.find("Cannot use --shallow with a fixed sha1") 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Kontrol SAS 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /tsrc/test/helpers/manifest_file.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List 3 | 4 | import ruamel.yaml 5 | 6 | """helper function(s) follows: 7 | these functions do not take part on testing by itself alone""" 8 | 9 | 10 | def ad_hoc_deep_manifest_manifest_branch( 11 | workspace_path: Path, 12 | branch: str, 13 | ) -> None: 14 | manifest_path = workspace_path / "manifest" / "manifest.yml" 15 | manifest_path.parent.mkdir(parents=True, exist_ok=True) 16 | yaml = ruamel.yaml.YAML(typ="rt") 17 | parsed = yaml.load(manifest_path.read_text()) 18 | for _, value in parsed.items(): 19 | if isinstance(value, List): 20 | for x in value: 21 | if isinstance(x, ruamel.yaml.comments.CommentedMap): 22 | if "dest" in x and x["dest"] == "manifest": 23 | x.insert(2, "branch", branch) 24 | 25 | with open(manifest_path, "w") as file: 26 | yaml.dump(parsed, file) 27 | 28 | 29 | def ad_hoc_deep_manifest_manifest_url( 30 | workspace_path: Path, 31 | url: str, 32 | ) -> None: 33 | manifest_path = workspace_path / "manifest" / "manifest.yml" 34 | manifest_path.parent.mkdir(parents=True, exist_ok=True) 35 | yaml = ruamel.yaml.YAML(typ="rt") 36 | parsed = yaml.load(manifest_path.read_text()) 37 | for _, value in parsed.items(): 38 | if isinstance(value, List): 39 | for x in value: 40 | if isinstance(x, ruamel.yaml.comments.CommentedMap): 41 | if "dest" in x and x["dest"] == "manifest": 42 | x["url"] = url 43 | 44 | with open(manifest_path, "w") as file: 45 | yaml.dump(parsed, file) 46 | -------------------------------------------------------------------------------- /tsrc/test/test_config.py: -------------------------------------------------------------------------------- 1 | import textwrap 2 | from pathlib import Path 3 | from unittest import mock 4 | 5 | import pytest 6 | import ruamel.yaml 7 | import schema 8 | 9 | from tsrc.config import parse_config 10 | from tsrc.errors import InvalidConfigError 11 | 12 | 13 | def test_invalid_syntax(tmp_path: Path) -> None: 14 | foo_yml = tmp_path / "foo.yml" 15 | foo_yml.write_text( 16 | textwrap.dedent( 17 | """ 18 | foo: 19 | bar: 20 | baz: [ 21 | 22 | baz: 42 23 | """ 24 | ) 25 | ) 26 | with pytest.raises(InvalidConfigError) as e: 27 | dummy_schema = mock.Mock() 28 | parse_config(foo_yml, schema=dummy_schema) 29 | raised_error = e.value 30 | assert raised_error.config_path == foo_yml 31 | assert isinstance(raised_error.cause, ruamel.yaml.error.YAMLError) 32 | 33 | 34 | def test_invalid_schema(tmp_path: Path) -> None: 35 | foo_yml = tmp_path / "foo.yml" 36 | foo_yml.write_text( 37 | textwrap.dedent( 38 | """ 39 | foo: 40 | bar: 42 41 | """ 42 | ) 43 | ) 44 | foo_schema = schema.Schema({"foo": {"bar": str}}) 45 | with pytest.raises(InvalidConfigError) as e: 46 | parse_config(foo_yml, schema=foo_schema) 47 | assert isinstance(e.value.cause, schema.SchemaError) 48 | 49 | 50 | def test_use_pure_python_types(tmp_path: Path) -> None: 51 | """Check that parse_config() returns pure Python dicts, 52 | not an OrderedDict or yaml's CommentedMap 53 | """ 54 | foo_yml = tmp_path / "foo.yml" 55 | foo_yml.write_text("foo: 42\n") 56 | foo_schema = schema.Schema({"foo": int}) 57 | parsed = parse_config(foo_yml, schema=foo_schema) 58 | assert parsed.__class__ == dict 59 | -------------------------------------------------------------------------------- /tsrc/test/helpers/cli.py: -------------------------------------------------------------------------------- 1 | """ Helper to call tsrc's main() function. 2 | 3 | Used by the `tsrc_cli` fixture. 4 | """ 5 | 6 | import os 7 | from pathlib import Path 8 | from typing import Any, List, Type 9 | 10 | import cli_ui as ui 11 | import pytest 12 | 13 | from tsrc.cli.main import testable_main 14 | from tsrc.errors import Error 15 | 16 | 17 | def append_jobs_option(*args: str) -> List[str]: 18 | jobs = os.environ.get("TSRC_TEST_JOBS") 19 | if jobs: 20 | action, *rest = args 21 | if action not in ["apply-manifest", "version"]: 22 | return [action, "-j", jobs, *rest] 23 | return [*args] 24 | 25 | 26 | class CLI: 27 | def __init__(self) -> None: 28 | self.workspace_path = Path(os.getcwd()) 29 | 30 | def run(self, *args: str) -> None: 31 | tsrc_args = append_jobs_option(*args) 32 | ui.info(">", ui.bold, "tsrc", *tsrc_args) 33 | testable_main(tsrc_args) 34 | 35 | def run_and_fail(self, *args: str) -> Error: 36 | jobs = os.environ.get("TSRC_TEST_JOBS") 37 | if jobs: 38 | info_args = [*args, "-j", jobs] 39 | else: 40 | info_args = [*args] 41 | ui.info(">", ui.bold, "tsrc", *info_args, end="") 42 | ui.info(ui.red, " (expecting failure)") 43 | with pytest.raises(Error) as e: 44 | testable_main(args) 45 | return e.value 46 | 47 | def run_and_fail_with(self, error: Type[Error], *args: str) -> Error: 48 | actual_error = self.run_and_fail(*args) 49 | assert isinstance(actual_error, error) 50 | return actual_error 51 | 52 | 53 | @pytest.fixture 54 | def tsrc_cli(workspace_path: Path, monkeypatch: Any) -> CLI: 55 | monkeypatch.chdir(workspace_path) 56 | res = CLI() 57 | return res 58 | -------------------------------------------------------------------------------- /docs/contrib/dev.md: -------------------------------------------------------------------------------- 1 | # Suggesting changes 2 | 3 | All the development happens on [GitHub](https://github.com/your-tools/tsrc). 4 | 5 | You are free to open a pull request for anything you want to change on `tsrc`. 6 | 7 | In particular, pull requests that implement a prototype for a new 8 | feature are welcome, having "real code" to look at can provide useful 9 | insight, even if the code is not merged after all. 10 | 11 | That being said, if you want your pull request to be merged, we'll 12 | ask that: 13 | 14 | * The code follows the indications from the [code manifesto](../code-manifesto.md) 15 | * All existing linters pass 16 | * All existing tests run 17 | * The new feature comes with appropriate tests 18 | * The Git History is easy to review 19 | 20 | See the [GitHub actions workflows](https://github.com/your-tools/tsrc/blob/main/.github/workflows) 21 | to see what exactly what commands are run and the Python versions we 22 | support. 23 | 24 | Also, if relevant, you will need to: 25 | 26 | * update the changelog (in `docs/changelog.md`) 27 | * update the documentation if required 28 | 29 | 30 | Finally, feel free to add your name in the `THANKS` file ;) 31 | 32 | ## Checking your changes 33 | 34 | * Install latest [poetry](https://python-poetry.org) version. 35 | * Install development and documentation dependencies: 36 | 37 | ```console 38 | $ poetry install 39 | ``` 40 | 41 | * Run linters and tests: 42 | 43 | ```console 44 | $ poetry run invoke lint 45 | $ poetry run pytest -n auto 46 | ``` 47 | 48 | 49 | ## Adding documentation 50 | 51 | * Follow the steps from the above section to setup your python environment 52 | * Launch the development server locally: 53 | 54 | ```bash 55 | $ poetry run mkdocs serve 56 | ``` 57 | 58 | * Edit the markdown files from the `docs/` folder and review the changes in your browser 59 | * Finally, submit your changes by opening a pull request on GitHub 60 | -------------------------------------------------------------------------------- /tsrc/workspace_config.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from dataclasses import asdict, dataclass, fields 3 | from pathlib import Path 4 | from typing import Any, List, Optional 5 | 6 | import ruamel.yaml 7 | 8 | 9 | @dataclass 10 | class WorkspaceConfig: 11 | """Persistent configuration of the workspace. 12 | 13 | Stored in /.tsrc/config.yml, and can be 14 | edited by hand to use a different set of groups 15 | for instance. 16 | """ 17 | 18 | manifest_url: str 19 | manifest_branch: str 20 | manifest_branch_0: str 21 | repo_groups: List[str] 22 | 23 | shallow_clones: bool = False 24 | clone_all_repos: bool = False 25 | 26 | singular_remote: Optional[str] = None 27 | 28 | def __init__(self, **kwargs: Any) -> None: 29 | # only set those that are present 30 | names = {f.name for f in fields(self)} 31 | for key, value in kwargs.items(): 32 | if key in names: 33 | setattr(self, key, value) 34 | 35 | @classmethod 36 | def from_file(cls, cfg_path: Path) -> "WorkspaceConfig": 37 | yaml = ruamel.yaml.YAML(typ="rt") 38 | parsed = yaml.load(cfg_path.read_text()) 39 | if not parsed.get("manifest_branch_0"): 40 | """compatibility fix for older version. 41 | usefull when transitioning with Workspace initialized 42 | by older version""" 43 | parsed["manifest_branch_0"] = parsed.get("manifest_branch") 44 | parsed = OrderedDict(sorted(parsed.items())) 45 | return cls(**parsed) 46 | 47 | def save_to_file(self, cfg_path: Path) -> None: 48 | cfg_path.parent.mkdir(parents=True, exist_ok=True) 49 | yaml = ruamel.yaml.YAML(typ="rt") 50 | yaml.register_class(Path) 51 | as_dict = asdict(self) 52 | with cfg_path.open("w") as fp: 53 | yaml.dump(as_dict, fp) 54 | -------------------------------------------------------------------------------- /tsrc/errors.py: -------------------------------------------------------------------------------- 1 | """ Custom exceptions """ 2 | 3 | from pathlib import Path 4 | from typing import Any 5 | 6 | from tsrc.manifest_common_data import ManifestsTypeOfData 7 | 8 | DOC_URL = "https://your-tools.github.io/tsrc" 9 | 10 | 11 | class Error(Exception): 12 | """Base class for our own errors.""" 13 | 14 | def __init__(self, *args: Any) -> None: 15 | super().__init__(self, *args) 16 | self.message = " ".join(str(x) for x in args) 17 | 18 | def __str__(self) -> str: 19 | return self.message 20 | 21 | def __repr__(self) -> str: 22 | return f"<{self.__class__.__name__}>" 23 | 24 | 25 | class InvalidConfigError(Error): 26 | def __init__(self, config_path: Path, cause: Exception) -> None: 27 | self.config_path = config_path 28 | self.cause = cause 29 | super().__init__(self.detailed_message) 30 | 31 | @property 32 | def detailed_message(self) -> str: 33 | res = f"{self.config_path}: {self.cause}" 34 | res += "\n" 35 | res += f"See {DOC_URL} for details" 36 | return res 37 | 38 | def __str__(self) -> str: 39 | return self.detailed_message 40 | 41 | 42 | class LoadManifestSchemaError(Error): 43 | def __init__(self, mtod: ManifestsTypeOfData) -> None: 44 | if mtod == ManifestsTypeOfData.DEEP: 45 | msg = "Failed to get Deep Manifest" 46 | elif mtod == ManifestsTypeOfData.FUTURE: 47 | msg = "Failed to get Future Manifest" 48 | else: 49 | msg = "Failed to get Manifest" 50 | super().__init__(msg) 51 | 52 | 53 | class LoadManifestSwitchConfigGroupsError(Error): 54 | def __init__(self) -> None: 55 | msg = "Manifest's Switch Config Groups does not match Groups" 56 | super().__init__(msg) 57 | 58 | 59 | class MissingRepoError(Error): 60 | def __init__(self, dest: str): 61 | super().__init__(f"No repo found in '{dest}'. Please run `tsrc sync`") 62 | -------------------------------------------------------------------------------- /tsrc/test/cli/test_apply_manifest.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from pathlib import Path 3 | 4 | import ruamel.yaml 5 | 6 | from tsrc.test.helpers.cli import CLI 7 | from tsrc.test.helpers.git_server import GitServer 8 | 9 | 10 | def test_apply_manifest_adds_new_repo( 11 | tsrc_cli: CLI, git_server: GitServer, workspace_path: Path 12 | ) -> None: 13 | """Scenario: 14 | 15 | * Create a manifest with one repo 16 | * Create a workspace using `tsrc init` 17 | * Copy the manifest file somewhere in the workspace 18 | * Create a new repo on the server 19 | * Edit the copied manifest to contain the new repo, including 20 | a file system copy 21 | * Run `tsrc apply-manifest /path/to/copied_manifest` 22 | * Check that the new repo is cloned 23 | * Check that the copy is performed 24 | 25 | """ 26 | git_server.add_repo("foo") 27 | tsrc_cli.run("init", git_server.manifest_url) 28 | 29 | cloned_manifest_path = workspace_path / ".tsrc/manifest/manifest.yml" 30 | copied_manifest_path = workspace_path / "manifest.yml" 31 | shutil.copy(cloned_manifest_path, copied_manifest_path) 32 | 33 | bar_url = git_server.add_repo("bar", add_to_manifest=False) 34 | git_server.push_file("bar", "src") 35 | add_repo_to_manifest_with_copy(copied_manifest_path, "bar", bar_url) 36 | 37 | tsrc_cli.run("apply-manifest", str(copied_manifest_path)) 38 | 39 | assert (workspace_path / "bar").exists(), "bar repo should have been cloned" 40 | assert ( 41 | workspace_path / "dest" 42 | ).exists(), "file system operations should have been performed" 43 | 44 | 45 | def add_repo_to_manifest_with_copy(manifest_path: Path, dest: str, url: str) -> None: 46 | yaml = ruamel.yaml.YAML() 47 | data = yaml.load(manifest_path.read_text()) 48 | repos = data["repos"] 49 | to_add = {"dest": dest, "url": url, "copy": [{"file": "src", "dest": "dest"}]} 50 | repos.append(to_add) 51 | with manifest_path.open("w") as fileobj: 52 | yaml.dump(data, fileobj) 53 | -------------------------------------------------------------------------------- /tsrc/manifest_common_data.py: -------------------------------------------------------------------------------- 1 | from enum import Enum, unique 2 | from typing import List 3 | 4 | import cli_ui as ui 5 | 6 | 7 | @unique 8 | class ManifestsTypeOfData(Enum): 9 | LOCAL = 1 10 | DEEP = 2 11 | DEEP_ON_UPDATE = 3 # do not put warning about missing element 12 | DEEP_BLOCK = 4 13 | FUTURE = 5 14 | SAVED = 6 # manifest created by '--save-to' 15 | 16 | 17 | def get_mtod_str(tod: ManifestsTypeOfData) -> str: 18 | if tod == ManifestsTypeOfData.LOCAL: 19 | return "Local Manifest" 20 | if tod == ManifestsTypeOfData.DEEP: 21 | return "Deep Manifest" 22 | if tod == ManifestsTypeOfData.DEEP_ON_UPDATE: 23 | return "Deep Manifest on UPDATE" 24 | if tod == ManifestsTypeOfData.DEEP_BLOCK: 25 | return "Deep Manifest's block" 26 | if tod == ManifestsTypeOfData.FUTURE: 27 | return "Future Manifest" 28 | if tod == ManifestsTypeOfData.SAVED: 29 | return "Saved Manifest" 30 | 31 | 32 | def mtod_can_ignore_remotes() -> List[ManifestsTypeOfData]: 33 | rl: List[ManifestsTypeOfData] = [ 34 | # only for LOCAL Manifest the missing remote 35 | # cannot be ignored. 36 | ManifestsTypeOfData.DEEP, 37 | ManifestsTypeOfData.DEEP_ON_UPDATE, 38 | ManifestsTypeOfData.DEEP_BLOCK, 39 | ManifestsTypeOfData.FUTURE, 40 | ManifestsTypeOfData.SAVED, 41 | ] 42 | return rl 43 | 44 | 45 | def mtod_get_main_color(tod: ManifestsTypeOfData) -> ui.Token: 46 | # for Local Manifest (using for Manifest's Marker color) 47 | if tod == ManifestsTypeOfData.LOCAL: 48 | return ui.reset 49 | 50 | # for Deep Manifest (for: 'dest' color, MM color) 51 | if tod == ManifestsTypeOfData.DEEP: 52 | return ui.purple 53 | 54 | # for Deep Manifest block (for: square brackets color) 55 | if tod == ManifestsTypeOfData.DEEP_BLOCK: 56 | return ui.brown 57 | 58 | # for Future Manifest (for 'dest' color, MM color) 59 | if tod == ManifestsTypeOfData.FUTURE: 60 | return ui.cyan 61 | 62 | return ui.reset # we should never reach it 63 | -------------------------------------------------------------------------------- /tsrc/groups_to_find.py: -------------------------------------------------------------------------------- 1 | """ 2 | Groups To Find 3 | 4 | Keeps track of what groups was requested 5 | and what groups was found 6 | 7 | This can help in a way to have just one class 8 | to transfer data with into the other functions. 9 | 10 | And perform easy calculation to see if some 11 | requested group(s) ('self.groups') does not match, 12 | so the exception can be raised properly 13 | """ 14 | 15 | from typing import List, Tuple, Union 16 | 17 | 18 | class GroupsToFind: 19 | def __init__( 20 | self, groups: Union[List[str], None], ignore_missing_groups: bool = False 21 | ) -> None: 22 | self.groups = groups 23 | self.ignore_missing_groups = ignore_missing_groups 24 | self.found_groups: List[str] = [] 25 | 26 | def found_some(self) -> bool: 27 | if self.found_groups: 28 | return True 29 | return False 30 | 31 | def found_this(self, this_group: str) -> None: 32 | """mark single group as found""" 33 | if self.groups: 34 | if this_group not in self.found_groups: 35 | self.found_groups.append(this_group) 36 | 37 | def found_these(self, this_found_groups: List[str]) -> None: 38 | """mark entire list of groups as found""" 39 | if self.found_groups: 40 | # just eliminate duplicates in the list 41 | self.found_groups = list(set(self.found_groups + this_found_groups)) 42 | else: 43 | self.found_groups = this_found_groups 44 | 45 | def was_found(self, this_group: str) -> bool: 46 | """check only single group whether it was found""" 47 | if this_group in self.found_groups: 48 | return True 49 | return False 50 | 51 | def all_found(self) -> Tuple[bool, List[str]]: 52 | """checks if we have found all groups""" 53 | if self.groups: 54 | missing_groups = list(set(self.groups).difference(self.found_groups)) 55 | else: 56 | return True, [] 57 | if missing_groups: 58 | return False, missing_groups 59 | return True, [] 60 | -------------------------------------------------------------------------------- /docs/guide/fixed-refs.md: -------------------------------------------------------------------------------- 1 | # Using fixed git references 2 | 3 | By default, `tsrc sync` synchronize projects using *branches names*. 4 | 5 | Usually, one would use the same branch name for several git repositories, like this: 6 | 7 | ```yaml 8 | repos: 9 | - dest: foo 10 | url: git@gitlab.acme.com/your-team/foo 11 | branch: main 12 | 13 | - dest: bar 14 | url: git@gitlab.acme.com/your-team/bar 15 | branch: main 16 | ``` 17 | 18 | The assumption here is that `foo` and `bar` evolve "at the same time", so when the 19 | `main` branch of `foo` is updated, the `main` branch of `bar` much change too. 20 | 21 | Sometimes though, this will not be the case. For instance, the `main` branch of the 22 | `bar` repo needs a *specific, fixed version* of `foo` in order to work. 23 | 24 | ## Using a tag 25 | 26 | One way to solve this is to push a v1.0 tag in the `foo` repository, and change 27 | the manifest too look like this: 28 | 29 | 30 | ```diff 31 | repos: 32 | - dest: foo 33 | url: git@gitlab.acme.com/your-team/foo 34 | - branch: main 35 | + tag: v1.0 36 | ``` 37 | 38 | ## Using a sha1 39 | 40 | An other way is to put the SHA1 of the relevant git commit in the `foo` repository in the 41 | manifest: 42 | 43 | 44 | ```diff 45 | repos: 46 | - dest: foo 47 | url: git@gitlab.acme.com/your-team/foo 48 | branch: main 49 | + sha1: ad2b68539c78e749a372414165acdf2a1bb68203 50 | ``` 51 | 52 | ## Cloning repos using fixed refs 53 | 54 | * If the repo is configured with a tag, `tsrc` will call `git clone 55 | --branch ` (which is valid) 56 | * Otherwise, `tsrc` will call `git clone`, followed by `git reset --hard ` 57 | 58 | This is because you cannot tell git to use an arbitrary git reference as 59 | start branch when cloning (tags are fine, but sha1s are not). 60 | 61 | This also explain why you need both `branch` and `sha1` in the 62 | configuration. 63 | 64 | 65 | ## Synchronizing repos using fixed refs 66 | 67 | Here's what `tsrc sync` will do when trying to synchronize a repo 68 | configured with a fixed ref: 69 | 70 | * Run `git fetch --tags --prune` 71 | * Check if the repository is clean 72 | * If so, run `git reset --hard ` 73 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | tsrc logo 2 | 3 | # tsrc - managing groups of git repositories 4 | 5 | 6 | ## What it does 7 | 8 | `tsrc` is a command-line tool that helps you manage groups of git repositories. 9 | 10 | It works by listing the repositories in a file called `manifest.yml` that looks like this: 11 | 12 | ```yaml 13 | repos: 14 | - dest: foo 15 | url: git@example.com:foo.git 16 | 17 | - dest: bar 18 | url: git@example.com:bar.git 19 | ``` 20 | 21 | You can then use: 22 | 23 | * `tsrc init ` to create a *workspace* containing 24 | the `foo` and `bar` repository 25 | 26 | * `tsrc sync` to synchronize all repos in the workspace. 27 | 28 | * ... and many more commands. Run `tsrc help` to list them, or read the [command line reference](ref/cli.md) 29 | 30 | ## Tutorial 31 | 32 | Interested in using `tsrc` in your own organization? 33 | 34 | Proceed to the [getting started tutorial](getting-started.md)! 35 | 36 | 37 | ## Guides 38 | 39 | Once you've learn how to setup tsrc for your organization, feel free to 40 | read the following guides - tsrc supports a variety of use cases beyond 41 | just listing git repositories to be cloned or synchronized and are 42 | described here: 43 | 44 | * [Editing the manifest safely](guide/manifest.md) 45 | * [Editing workspace configuration](guide/workspace-config.md) 46 | * [Using groups](guide/groups.md) 47 | * [Using several remotes](guide/remotes.md) 48 | * [Using fixed git references](guide/fixed-refs.md) 49 | * [Performing file system operations](guide/fs.md) 50 | * [Running a command for each repo in the workspace](guide/foreach.md) 51 | * [Using tsrc with continuous integration](guide/ci.md) 52 | 53 | ### Reference 54 | 55 | * [Command line interface](ref/cli.md) 56 | * [Sync algorithm](ref/sync.md) 57 | * [Manifest configuration](ref/manifest-config.md) 58 | * [Workspace configuration](ref/workspace-config.md) 59 | 60 | ## Contributing 61 | 62 | * [Using the issue tracker](contrib/issues.md) 63 | * [Suggesting changes](contrib/dev.md) 64 | * [Code Manifesto](./code-manifesto.md) 65 | 66 | ## Useful links 67 | 68 | * [FAQ](./faq.md) 69 | * [Changelog](./changelog.md) 70 | -------------------------------------------------------------------------------- /docs/guide/groups.md: -------------------------------------------------------------------------------- 1 | # Using groups 2 | 3 | Sometimes it can be necessary to create groups of repositories, especially if the number 4 | of repositories grows and if you have people in different teams work on different repositories. 5 | 6 | ## Defining groups in the manifest 7 | 8 | The first step is to edit the `manifest.yml` file to describe the groups. Here's an 9 | example. 10 | 11 | ```yaml 12 | repos: 13 | - {url: git@gitlab.local:acme/one, dest: one} 14 | - {url: git@gitlab.local:acme/two, dest: two} 15 | - {url: git@gitlab.local:acme/three, dest: three} 16 | 17 | groups: 18 | default: 19 | repos: [] 20 | g1: 21 | repos: 22 | - one 23 | - two 24 | g2: 25 | repos: 26 | - three 27 | ``` 28 | 29 | Here we define a `g1` group that contains repositories named `one` and `two`, 30 | and a `g2` group that contains the repository named `three`. 31 | 32 | ## Using groups in `tsrc init` 33 | 34 | If you only need the repositories in the `g1` group you can run: 35 | 36 | ``` 37 | tsrc init git@gitlab.local:acme/manifest --group g1 38 | ``` 39 | 40 | ## Filtering repositories in groups with regular expressions 41 | 42 | You can utilize inclusive regular expression with the `-i`-flag and 43 | exclusive regular expression with the `-e`-flag. This allows you to filter 44 | repositories within a group or a set of groups for the given action. 45 | 46 | 47 | To include all repositories in the group g1 matching "config" and excluding "template", 48 | you can do the following: 49 | 50 | ``` 51 | tsrc init git@gitlab.local:acme/manifest --group g1 -i config -e template 52 | ``` 53 | 54 | 55 | ## Updating workspace configuration 56 | 57 | Alternatively, you can edit the `.tsrc/config.yml` file, like this: 58 | 59 | ```yaml 60 | manifest_url: git@gitlab.local:acme/manifest.git 61 | manifest_branch: master 62 | repo_groups: 63 | - g1 # <- specify the list of groups to use 64 | ``` 65 | 66 | You can use this technique to change the groups used in a given workspace - 67 | the above method using `init` only works to *create* new workspaces. 68 | 69 | The config file contains other configuration options, which are described 70 | in the [workspace configuration documentation](../ref/workspace-config.md). 71 | -------------------------------------------------------------------------------- /tsrc/test/test_file_system.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from tsrc.errors import Error 7 | from tsrc.file_system import safe_link 8 | 9 | 10 | def test_can_create_symlink_when_source_does_not_exist(tmp_path: Path) -> None: 11 | source = tmp_path / "source" 12 | target = tmp_path / "target" 13 | target.touch() 14 | safe_link(source=source, target=target) 15 | assert source.exists() 16 | assert source.resolve() == target.resolve() 17 | 18 | 19 | def test_can_create_symlink_pointing_to_directory(tmp_path: Path) -> None: 20 | source = tmp_path / "source" 21 | target = tmp_path / "target" 22 | target.mkdir(parents=True) 23 | safe_link(source=source, target=target) 24 | 25 | assert source.exists() 26 | assert source.resolve() == target.resolve() 27 | 28 | 29 | def test_cannot_create_symlink_when_source_is_a_file(tmp_path: Path) -> None: 30 | source = tmp_path / "source" 31 | target = tmp_path / "target" 32 | source.touch() 33 | with pytest.raises(Error) as e: 34 | safe_link(source=source, target=target) 35 | assert "is not a link" in e.value.message 36 | 37 | 38 | def test_can_update_broken_symlink(tmp_path: Path) -> None: 39 | source = tmp_path / "source" 40 | target = tmp_path / "target" 41 | os.symlink(target, source) 42 | 43 | new_target = tmp_path / "new_target" 44 | new_target.touch() 45 | safe_link(source=source, target=new_target) 46 | 47 | assert source.exists() 48 | assert source.resolve() == new_target.resolve() 49 | 50 | 51 | def test_can_update_existing_symlink(tmp_path: Path) -> None: 52 | source = tmp_path / "source" 53 | target = tmp_path / "target" 54 | target.touch() 55 | os.symlink(target, source) 56 | 57 | new_target = tmp_path / "new_target" 58 | safe_link(source=source, target=new_target) 59 | 60 | new_target.touch() 61 | assert source.exists() 62 | assert source.resolve() == new_target.resolve() 63 | 64 | 65 | def test_do_nothing_if_symlink_has_the_correct_target(tmp_path: Path) -> None: 66 | source = tmp_path / "source" 67 | target = tmp_path / "target" 68 | target.touch() 69 | os.symlink(target, source) 70 | 71 | safe_link(source=source, target=target) 72 | 73 | assert source.exists() 74 | assert source.resolve() == target.resolve() 75 | -------------------------------------------------------------------------------- /tsrc/test/test_executor.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import cli_ui as ui 4 | 5 | from tsrc.errors import Error 6 | from tsrc.executor import ( 7 | Outcome, 8 | Task, 9 | process_items, 10 | process_items_parallel, 11 | process_items_sequence, 12 | ) 13 | 14 | 15 | class Kaboom(Error): 16 | def __init__(self) -> None: 17 | self.message = "Kaboom" 18 | 19 | 20 | class FakeTask(Task[str]): 21 | """This is a fake Task that can be used for testing. 22 | 23 | Note that it will raise an instance of the Kaboom exception 24 | when processing an item whose value is "failing" 25 | 26 | """ 27 | 28 | def __init__(self) -> None: 29 | pass 30 | 31 | def describe_process_start(self, item: str) -> List[ui.Token]: 32 | return ["Frobnicating", item] 33 | 34 | def describe_process_end(self, item: str) -> List[ui.Token]: 35 | return [item, "ok"] 36 | 37 | def process(self, index: int, count: int, item: str) -> Outcome: 38 | if item == "failing": 39 | raise Kaboom() 40 | return Outcome.empty() 41 | 42 | def describe_item(self, item: str) -> str: 43 | return item 44 | 45 | 46 | def test_sequence_nothing() -> None: 47 | task = FakeTask() 48 | items: List[str] = [] 49 | actual = process_items_sequence(items, task) 50 | assert not actual 51 | 52 | 53 | def test_sequence_happy() -> None: 54 | task = FakeTask() 55 | actual = process_items_sequence(["foo", "bar"], task) 56 | assert not actual["foo"].error 57 | assert not actual["bar"].error 58 | 59 | 60 | def test_sequence_sad() -> None: 61 | task = FakeTask() 62 | actual = process_items(["foo", "failing", "bar"], task) 63 | assert actual.errors["failing"].message == "Kaboom" 64 | 65 | 66 | def test_parallel_nothing() -> None: 67 | task = FakeTask() 68 | items: List[str] = [] 69 | actual = process_items_parallel(items, task, num_jobs=2) 70 | assert not actual 71 | 72 | 73 | def test_parallel_happy() -> None: 74 | task = FakeTask() 75 | ui.info("Frobnicating 4 items with two workers") 76 | actual = process_items_parallel(["foo", "bar", "baz", "quux"], task, num_jobs=2) 77 | ui.info("Done") 78 | for outcome in actual.values(): 79 | assert outcome.success() 80 | 81 | 82 | def test_parallel_sad() -> None: 83 | task = FakeTask() 84 | actual = process_items(["foo", "bar", "failing", "baz", "quux"], task, num_jobs=2) 85 | errors = actual.errors 86 | assert errors["failing"].message == "Kaboom" 87 | -------------------------------------------------------------------------------- /tsrc/cli/env_setter.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from tsrc.git import GitStatus 4 | from tsrc.repo import Repo 5 | from tsrc.workspace import Workspace 6 | 7 | 8 | class EnvSetter: 9 | def __init__(self, workspace: Workspace): 10 | self.workspace = workspace 11 | 12 | def get_env_for_repo(self, repo: Repo) -> Dict[str, str]: 13 | workspace_vars = get_workspace_vars(self.workspace) 14 | 15 | repo_vars = get_repo_vars(repo) 16 | 17 | repo_path = self.workspace.root_path / repo.dest 18 | status = GitStatus(repo_path) 19 | status.update() 20 | status_vars = get_status_vars(status) 21 | 22 | res = {} 23 | res.update(repo_vars) 24 | res.update(status_vars) 25 | res.update(workspace_vars) 26 | return res 27 | 28 | 29 | def get_workspace_vars(workspace: Workspace) -> Dict[str, str]: 30 | res = {} 31 | res["TSRC_WORKSPACE_PATH"] = str(workspace.root_path.resolve()) 32 | res["TSRC_MANIFEST_URL"] = workspace.config.manifest_url 33 | res["TSRC_MANIFEST_BRANCH"] = workspace.config.manifest_branch 34 | return res 35 | 36 | 37 | def get_repo_vars(repo: Repo) -> Dict[str, str]: 38 | res = {} 39 | res["TSRC_PROJECT_DEST"] = repo.dest 40 | if repo.branch: 41 | res["TSRC_PROJECT_MANIFEST_BRANCH"] = repo.branch 42 | res["TSRC_PROJECT_CLONE_URL"] = repo.clone_url 43 | if repo.sha1: 44 | res["TSRC_PROJECT_SHA1"] = repo.sha1 45 | if repo.tag: 46 | res["TSRC_PROJECT_TAG"] = repo.tag 47 | if repo.shallow: 48 | res["TSRC_PROJECT_SHALLOW"] = "true" 49 | for remote in repo.remotes: 50 | key = "TSRC_PROJECT_REMOTE_" + remote.name.upper() 51 | value = remote.url 52 | res[key] = value 53 | return res 54 | 55 | 56 | def get_status_vars(status: GitStatus) -> Dict[str, str]: 57 | res = {} 58 | res["TSRC_PROJECT_STATUS_UNTRACKED"] = str(status.untracked) 59 | res["TSRC_PROJECT_STATUS_ADDED"] = str(status.added) 60 | res["TSRC_PROJECT_STATUS_STAGED"] = str(status.staged) 61 | res["TSRC_PROJECT_STATUS_NOT_STAGED"] = str(status.not_staged) 62 | res["TSRC_PROJECT_STATUS_BEHIND"] = str(status.behind) 63 | res["TSRC_PROJECT_STATUS_AHEAD"] = str(status.ahead) 64 | if status.branch: 65 | res["TSRC_PROJECT_STATUS_BRANCH"] = status.branch 66 | if status.sha1: 67 | res["TSRC_PROJECT_STATUS_SHA1"] = status.sha1 68 | if status.tag: 69 | res["TSRC_PROJECT_STATUS_TAG"] = status.tag 70 | if status.dirty: 71 | res["TSRC_PROJECT_STATUS_DIRTY"] = "true" 72 | return res 73 | -------------------------------------------------------------------------------- /tsrc/dump_manifest_args_final_output.py: -------------------------------------------------------------------------------- 1 | # FinalOutput (ModeFlag|AllPaths) 2 | import argparse 3 | import os 4 | 5 | from tsrc.dump_manifest_args_data import ( 6 | DumpManifestOperationDetails, 7 | FinalOutputModeFlag, 8 | ) 9 | 10 | 11 | class FinalOutput: 12 | 13 | def __init__( 14 | self, args: argparse.Namespace, dmod: DumpManifestOperationDetails 15 | ) -> None: 16 | self.args = args 17 | self.dmod = dmod 18 | 19 | def get_final_output_modes_and_paths(self) -> DumpManifestOperationDetails: 20 | 21 | # on '--preview' 22 | self._take_care_of__preview() 23 | 24 | # on '--save_to' 25 | self._take_care_of__save_to() 26 | 27 | return self.dmod 28 | 29 | def _take_care_of__preview(self) -> None: 30 | if self.args.just_preview is True: 31 | # no output path in this case 32 | self.dmod.final_output_mode.append(FinalOutputModeFlag.PREVIEW) 33 | 34 | def _take_care_of__save_to(self) -> None: 35 | if self.args.save_to: 36 | if self.args.save_to.is_dir() is True: 37 | self.args.save_to = self.args.save_to / "manifest.yml" 38 | elif ( 39 | os.path.dirname(self.args.save_to) 40 | and os.path.isdir(os.path.dirname(self.args.save_to)) # noqa: W503 41 | is False # noqa: W503 42 | ): 43 | raise Exception( 44 | f"'SAVE_TO' directory structure must exists, however '{os.path.dirname(self.args.save_to)}' does not" # noqa: E501 45 | ) 46 | if self.args.save_to.is_file() is True: 47 | if ( 48 | self.args.use_force is False 49 | and FinalOutputModeFlag.PREVIEW # noqa: W503 50 | not in self.dmod.final_output_mode 51 | ): 52 | raise Exception( 53 | f"'SAVE_TO' file exist, use '--force' to overwrite existing file, or use '--update-on {self.args.save_to}' instead" # noqa: E501 54 | ) 55 | else: 56 | 57 | # set data only in regard of output 58 | self.dmod.final_output_mode.append(FinalOutputModeFlag.OVERWRITE) 59 | self.dmod.final_output_path_list.save_to_path = self.args.save_to 60 | 61 | else: 62 | 63 | # set data only in regard of output 64 | self.dmod.final_output_mode.append(FinalOutputModeFlag.NEW) 65 | self.dmod.final_output_path_list.save_to_path = self.args.save_to 66 | -------------------------------------------------------------------------------- /tsrc/test/helpers/message_recorder_ext.py: -------------------------------------------------------------------------------- 1 | """ 2 | this code extend the one in: 3 | 'https://github.com/your-tools/python-cli-ui' 4 | 5 | and it is 'find_next' function that is added 6 | (implemented here) 7 | 8 | patch was provided to original source in a form 9 | of [pull request #115](https://github.com/your-tools/python-cli-ui/pull/115) 10 | """ 11 | 12 | import re 13 | from typing import Any, Iterator, Optional 14 | 15 | import cli_ui 16 | import pytest 17 | 18 | 19 | class MessageRecorderExt: 20 | """Helper class to tests emitted messages""" 21 | 22 | def __init__(self) -> None: 23 | cli_ui._MESSAGES = [] 24 | self.idx_find_next: int = 0 25 | 26 | def start(self) -> None: 27 | """Start recording messages""" 28 | cli_ui.CONFIG["record"] = True 29 | 30 | def stop(self) -> None: 31 | """Stop recording messages""" 32 | cli_ui.CONFIG["record"] = False 33 | cli_ui._MESSAGES = [] 34 | 35 | def reset(self) -> None: 36 | """Reset the list""" 37 | cli_ui._MESSAGES = [] 38 | 39 | def find(self, pattern: str) -> Optional[str]: 40 | """Find a message in the list of recorded message 41 | 42 | :param pattern: regular expression pattern to use 43 | when looking for recorded message 44 | """ 45 | regexp = re.compile(pattern) 46 | for idx, message in enumerate(cli_ui._MESSAGES): 47 | if re.search(regexp, message): 48 | if isinstance(message, str): 49 | self.idx_find_next = idx + 1 50 | return message 51 | return None 52 | 53 | def find_right_after(self, pattern: str) -> Optional[str]: 54 | """Same as 'find', but only check the message that is right after 55 | the one found last time. if no message was found before, the 1st 56 | message in buffer is checked 57 | 58 | This is particulary usefull if we want to match only consecutive message. 59 | Calling this function can be repeated for further consecutive message match. 60 | """ 61 | if len(cli_ui._MESSAGES) > self.idx_find_next: 62 | regexp = re.compile(pattern) 63 | message = cli_ui._MESSAGES[self.idx_find_next] 64 | if re.search(regexp, message): 65 | if isinstance(message, str): 66 | self.idx_find_next += 1 67 | return message 68 | return None 69 | 70 | 71 | @pytest.fixture 72 | def message_recorder_ext(request: Any) -> Iterator[MessageRecorderExt]: 73 | recorder = MessageRecorderExt() 74 | recorder.start() 75 | yield recorder 76 | recorder.stop() 77 | -------------------------------------------------------------------------------- /tsrc/repo_grabber.py: -------------------------------------------------------------------------------- 1 | """ 2 | Repo Grabber 3 | 4 | allows 'dump-manifest' to paralelise GIT operations 5 | on single Path of possible Repo. 6 | """ 7 | 8 | from pathlib import Path 9 | from typing import List, Optional, Union 10 | 11 | import cli_ui as ui 12 | 13 | from tsrc.executor import Outcome, Task 14 | from tsrc.git import GitStatus, is_git_repository 15 | from tsrc.git_remote import GitRemote 16 | from tsrc.repo import Repo 17 | 18 | 19 | class RepoGrabber(Task[Repo]): 20 | """ 21 | Implements a Task that check and obtain Repo from Path 22 | """ 23 | 24 | def __init__(self, common_path: Union[List[str], None]) -> None: 25 | self.common_path = common_path 26 | self.repos: List[Repo] = [] # these are our output data 27 | 28 | def describe_item(self, item: Repo) -> str: 29 | return item.dest 30 | 31 | def describe_process_start(self, item: Repo) -> List[ui.Token]: 32 | return [item.dest] 33 | 34 | def describe_process_end(self, item: Repo) -> List[ui.Token]: 35 | return [ui.green, "ok", ui.reset, item.dest] 36 | 37 | def process(self, index: int, count: int, repo: Repo) -> Outcome: 38 | 39 | # we need actual Path (as Workspace Path may not be present here) 40 | repo_path: Optional[Path] = repo._grabbed_from_path 41 | if repo_path: 42 | if is_git_repository(repo_path) is False: 43 | return Outcome.empty() 44 | 45 | # obtain local GIT data 46 | gits = GitStatus(repo_path) 47 | gits.update() 48 | 49 | # obtain remote GIT data as well 50 | gitr = GitRemote(repo_path, repo.branch) 51 | gitr.update() 52 | if not gitr.remotes: 53 | # report missing remotes as such manifest will have litle meaning 54 | # in case we will want to use it later for synchronization 55 | ui.warning(f"No remote found for: '{repo.dest}' (path: '{repo_path}')") 56 | 57 | # we are now ready to create full Repo 58 | self.repos.append( 59 | Repo( 60 | dest=repo.dest, 61 | branch=gits.branch, 62 | keep_branch=True, # save empty branch if it is empty 63 | is_default_branch=False, 64 | orig_branch=gits.branch, 65 | sha1=gits.sha1, 66 | sha1_full=gits.sha1_full, 67 | tag=gits.tag, 68 | remotes=gitr.remotes, 69 | _grabbed_ahead=gits.ahead, 70 | _grabbed_behind=gits.behind, 71 | ) 72 | ) 73 | 74 | return Outcome.empty() 75 | -------------------------------------------------------------------------------- /docs/guide/remotes.md: -------------------------------------------------------------------------------- 1 | # Using several remotes 2 | 3 | When you specify a repository in the manifest with 4 | just an URL, `tsrc` assumes you want a remote named 5 | origin: 6 | 7 | ```yaml 8 | repos: 9 | - dest: foo 10 | url: git@gitlab.acme.com/your-team/foo 11 | 12 | - dest: bar 13 | url: git@gitlab.acme.com/your-team/bar 14 | ``` 15 | 16 | But sometimes you need several remotes. Let's see a few use cases. 17 | 18 | ## Mirroring open-source projects 19 | 20 | If you want some repos in your organization to be open source, you may need: 21 | 22 | * a remote named 'origin' containing for the private repository on your GitLab instance 23 | * a remote named 'github' for the public repository on GitHub 24 | 25 | In that case, you can use an alternative syntax: 26 | 27 | ```yaml 28 | repos: 29 | # foo is open source and thus needs two remotes: 30 | - dest: foo 31 | remotes: 32 | - name: origin 33 | url: git@gitlab.acme.com/your-team/foo 34 | - name: github 35 | url: git@github.com/your-team/foo 36 | 37 | # bar is closed source and thus only needs the 38 | # default, 'origin' remote: 39 | - dest: bar 40 | url: gitlab.acme.com/your-team/bar 41 | ``` 42 | 43 | After this change, when running `tsrc init` or `tsrc sync`, both the `origin` and `github` 44 | remotes will be created in the `foo` repo if they don't exist, and both 45 | remotes will be fetched when using `tsrc sync`. 46 | 47 | ## Using a VPN 48 | 49 | Sometimes you will need two remotes, because depending the physical location of 50 | your developers, they need to use either: 51 | 52 | * a 'normal' remote, when they are in the office 53 | * a 'vpn' remote, when they are working at home 54 | 55 | In that case, you can create a manifest looking like this: 56 | 57 | ```yaml 58 | repos: 59 | - dest: foo 60 | remotes: 61 | - name: origin 62 | url: git@gitlab.local/your-team/foo 63 | - name: vpn 64 | url: git@myvpn.com/gitlab/your-team/foo 65 | 66 | - dest: bar 67 | remotes: 68 | - name: origin 69 | url: git@gitlab.local/your-team/bar 70 | - name: vpn 71 | url: git@myvpn.com/gitlab/your-team/bar 72 | ``` 73 | 74 | Developers can then use the `-r, --singular-remote` option to either use the `origin` or `vpn` when 75 | running `tsrc init` (to create a workspace), or `tsrc sync` (to synchronize it), depending on 76 | their physical location: 77 | 78 | ```bash 79 | # Init the workspace using the 'vpn' remote 80 | $ tsrc init -r vpn 81 | # Bring back the computer in the office 82 | # Synchronize using the 'origin' remote: 83 | $ tsrc sync -r origin 84 | ``` 85 | 86 | !!!note 87 | When using this option, `tsrc` expects the remote to be present in the manifest for *all* repositories. 88 | 89 | -------------------------------------------------------------------------------- /tsrc/config_tools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Config Tools 3 | 4 | Contains all config-change related functions 5 | that can be performed on config 6 | 7 | The idea is to reach full command control 8 | of the config, so no manual editing needs to be performed 9 | 10 | Functions of this class should return only 'bool' 11 | and/or relevant data. It should not display anything anywhere 12 | """ 13 | 14 | from typing import List 15 | 16 | from tsrc.config_data import ConfigUpdateData, ConfigUpdateType 17 | from tsrc.config_status_rc import ConfigStatusReturnCode 18 | from tsrc.git_remote import remote_branch_exist 19 | from tsrc.workspace import Workspace 20 | 21 | 22 | class ConfigTools: 23 | def __init__( 24 | self, 25 | workspace: Workspace, 26 | ) -> None: 27 | self.workspace = workspace 28 | 29 | def commit_config_update( 30 | self, 31 | cfgud: ConfigUpdateData, 32 | cfguts: List[ConfigUpdateType], 33 | ) -> None: 34 | """once all updates are done, 35 | calling commit is in order""" 36 | 37 | for this_type in cfguts: 38 | if this_type == ConfigUpdateType.MANIFEST_BRANCH and cfgud.manifest_branch: 39 | self.workspace.config.manifest_branch = cfgud.manifest_branch 40 | # here add more type match option when implemented 41 | 42 | # now write to config (all changes at once) 43 | self.workspace.config.save_to_file(self.workspace.cfg_path) 44 | 45 | def update_manifest_branch( 46 | self, 47 | new_branch: str, 48 | ) -> ConfigStatusReturnCode: 49 | if self.workspace.config.manifest_branch == new_branch: 50 | return ConfigStatusReturnCode.CANCEL 51 | 52 | if self.workspace.config.manifest_branch_0 != new_branch: 53 | rc_is_on_remote = remote_branch_exist( 54 | self.workspace.config.manifest_url, 55 | new_branch, 56 | ) 57 | if rc_is_on_remote == 0: 58 | return ConfigStatusReturnCode.SUCCESS 59 | else: 60 | return ConfigStatusReturnCode.NOT_FOUND 61 | else: 62 | return ConfigStatusReturnCode.REVERT 63 | 64 | def local_update_manifest_branch(self) -> None: 65 | """ 66 | Q: Why we should not accept 67 | manifest branch change to branch that does not 68 | exists remotely, while it is in local 69 | (Workspace) Manifest's repository? 70 | A: because Future Manifest will not work in such case. 71 | Q: What about Deep Manifest? 72 | A: Deep Manifest will work when we chage branch 73 | of Manifest repo by 'git' command. 74 | Deep Manifest does not care about 75 | Manifest's configured branch in config 76 | """ 77 | pass 78 | -------------------------------------------------------------------------------- /tsrc/dump_manifest_args_source_mode.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from pathlib import Path 4 | from typing import Tuple 5 | 6 | from tsrc.cli import get_workspace_with_repos 7 | from tsrc.dump_manifest_args_data import DumpManifestOperationDetails, SourceModeEnum 8 | from tsrc.errors import Error 9 | 10 | 11 | class SourceMode: 12 | 13 | def __init__( 14 | self, args: argparse.Namespace, dmod: DumpManifestOperationDetails 15 | ) -> None: 16 | self.args = args 17 | self.dmod = dmod 18 | 19 | def get_source_mode_and_path( 20 | self, 21 | ) -> Tuple[DumpManifestOperationDetails, argparse.Namespace]: 22 | 23 | self._respect_workspace_path() 24 | 25 | self._decide_source_mode() 26 | 27 | self._get_workspace_if_needed() 28 | 29 | if not self.dmod.workspace: 30 | self._get_workspace_optionally() 31 | 32 | return self.dmod, self.args 33 | 34 | def _respect_workspace_path(self) -> None: 35 | # when Workspace path is provided by '-w', we have to consider 36 | # it as root path when relative path is provided for RAW dump 37 | if ( 38 | self.args.raw_dump_path 39 | and self.args.workspace_path # noqa: W503 40 | and not os.path.isabs(self.args.raw_dump_path) # noqa: W503 41 | ): 42 | self.args.raw_dump_path = Path( 43 | os.path.join(self.args.workspace_path, self.args.raw_dump_path) 44 | ) 45 | 46 | def _decide_source_mode(self) -> None: 47 | if self.args.raw_dump_path: 48 | self.dmod.source_mode = SourceModeEnum.RAW_DUMP 49 | self.dmod.source_path = self.args.raw_dump_path 50 | else: 51 | # right now there are no more 'Source MODEs' implemented 52 | # therefore any other then RAW MODE is Workspace MODE 53 | self.dmod.source_mode = SourceModeEnum.WORKSPACE_DUMP 54 | 55 | def _get_workspace_if_needed(self) -> None: 56 | # determine if Workspace is required 57 | if not self.args.raw_dump_path or ( 58 | self.args.raw_dump_path and self.args.do_update is True 59 | ): 60 | # it will throw Error if there is no Workspace 61 | self.dmod.workspace = get_workspace_with_repos(self.args) 62 | 63 | def _get_workspace_optionally(self) -> None: 64 | # do not throw and Error if Workspace is not found 65 | if self.args.raw_dump_path and ( 66 | self.args.skip_manifest is True or self.args.only_manifest is True 67 | ): 68 | try: 69 | self.dmod.workspace = get_workspace_with_repos(self.args) 70 | except Exception as e: 71 | if isinstance(e, Error): 72 | pass 73 | else: 74 | raise e 75 | -------------------------------------------------------------------------------- /tsrc/cli/init.py: -------------------------------------------------------------------------------- 1 | """ Entry point for `tsrc init`. """ 2 | 3 | import argparse 4 | from pathlib import Path 5 | 6 | import cli_ui as ui 7 | 8 | from tsrc.cli import ( 9 | add_groups_arg, 10 | add_num_jobs_arg, 11 | add_workspace_arg, 12 | get_num_jobs, 13 | repos_from_config, 14 | ) 15 | from tsrc.errors import Error 16 | from tsrc.local_manifest import LocalManifest 17 | from tsrc.workspace import Workspace 18 | from tsrc.workspace_config import WorkspaceConfig 19 | 20 | 21 | def configure_parser(subparser: argparse._SubParsersAction) -> None: 22 | parser = subparser.add_parser("init") 23 | add_workspace_arg(parser) 24 | parser.add_argument("manifest_url", help="git url containing the manifest file") 25 | parser.add_argument( 26 | "--branch", 27 | help="use this branch for the manifest", 28 | dest="manifest_branch", 29 | ) 30 | parser.add_argument( 31 | "--shallow", 32 | action="store_true", 33 | help="use shallow clones", 34 | dest="shallow_clones", 35 | ) 36 | parser.add_argument( 37 | "-r", 38 | "--singular-remote", 39 | help="only use this remote when cloning repositories", 40 | ) 41 | 42 | parser.add_argument( 43 | "--clone-all-repos", 44 | action="store_true", 45 | help="clone all repos from the manifest, regardless of the groups", 46 | ) 47 | add_groups_arg(parser) 48 | add_num_jobs_arg(parser) 49 | parser.set_defaults(run=run) 50 | 51 | 52 | def run(args: argparse.Namespace) -> None: 53 | workspace_path = args.workspace_path or Path.cwd() 54 | num_jobs = get_num_jobs(args) 55 | 56 | cfg_path = workspace_path / ".tsrc" / "config.yml" 57 | 58 | if cfg_path.exists(): 59 | raise Error(f"Workspace already configured. `{cfg_path}` already exists") 60 | 61 | ui.info_1("Configuring workspace in", ui.bold, workspace_path) 62 | 63 | clone_path = workspace_path / ".tsrc/manifest" 64 | local_manifest = LocalManifest(clone_path) 65 | local_manifest.init(url=args.manifest_url, branch=args.manifest_branch) 66 | manifest_branch = local_manifest.current_branch() 67 | 68 | workspace_config = WorkspaceConfig( 69 | manifest_url=args.manifest_url, 70 | manifest_branch=manifest_branch, 71 | manifest_branch_0=manifest_branch, 72 | clone_all_repos=args.clone_all_repos, 73 | repo_groups=args.groups or [], 74 | shallow_clones=args.shallow_clones, 75 | singular_remote=args.singular_remote, 76 | ) 77 | workspace_config.save_to_file(cfg_path) 78 | 79 | workspace = Workspace(workspace_path) 80 | manifest = workspace.get_manifest() 81 | workspace.repos = repos_from_config(manifest, workspace_config) 82 | workspace.clone_missing(num_jobs=num_jobs) 83 | workspace.set_remotes(num_jobs=num_jobs) 84 | workspace.perform_filesystem_operations() 85 | ui.info_2("Workspace initialized") 86 | ui.info_2("Configuration written in", ui.bold, workspace.cfg_path) 87 | -------------------------------------------------------------------------------- /tsrc/test/test_groups.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from tsrc.groups import GroupList, GroupNotFound, UnknownGroupElement 4 | 5 | 6 | def test_happy_grouping() -> None: 7 | group_list = GroupList(elements=["a", "b", "c"]) 8 | group_list.add("default", ["a", "b"]) 9 | group_list.add("other", ["c"], includes=["default"]) 10 | actual = group_list.get_elements(groups=["other"]) 11 | assert actual == ["a", "b", "c"] 12 | 13 | 14 | def test_remove_duplicates() -> None: 15 | group_list = GroupList(elements=["a", "b", "c", "z"]) 16 | group_list.add("one", ["a", "z"]) 17 | group_list.add("two", ["b", "z"]) 18 | group_list.add("all", ["c"], includes=["one", "two"]) 19 | actual = group_list.get_elements(groups=["all"]) 20 | assert actual == ["a", "z", "b", "c"] 21 | 22 | 23 | def test_unknown_element() -> None: 24 | group_list = GroupList(elements=["a", "b", "c"]) 25 | with pytest.raises(UnknownGroupElement) as e: 26 | group_list.add("invalid-group", ["no-such-element"]) 27 | assert e.value.group_name == "invalid-group" 28 | assert e.value.element == "no-such-element" 29 | 30 | 31 | def test_unknown_include() -> None: 32 | group_list = GroupList(elements=["a", "b", "c"]) 33 | group_list.add("default", ["a", "b"]) 34 | group_list.add("invalid-group", ["c"], includes=["no-such-group"]) 35 | with pytest.raises(GroupNotFound) as e: 36 | group_list.get_elements(groups=["invalid-group"]) 37 | assert e.value.parent_group is not None 38 | assert e.value.parent_group.name == "invalid-group" 39 | assert e.value.group_name == "no-such-group" 40 | 41 | 42 | def test_diamond() -> None: 43 | group_list = GroupList(elements=["a", "b", "c", "d"]) 44 | group_list.add("top", ["a"]) 45 | group_list.add("left", ["b"], includes=["top"]) 46 | group_list.add("right", ["c"], includes=["top"]) 47 | group_list.add("bottom", ["d"], includes=["left", "right"]) 48 | actual = group_list.get_elements(groups=["bottom"]) 49 | assert actual == ["a", "b", "c", "d"] 50 | 51 | 52 | def test_ping_pong() -> None: 53 | group_list = GroupList(elements=["a", "b"]) 54 | group_list.add("ping", ["a"], includes=["pong"]) 55 | group_list.add("pong", ["b"], includes=["ping"]) 56 | actual = group_list.get_elements(groups=["ping"]) 57 | assert actual == ["b", "a"] 58 | 59 | 60 | def test_circle() -> None: 61 | group_list = GroupList(elements=["a", "b", "c"]) 62 | group_list.add("a", ["a"], includes=["b"]) 63 | group_list.add("b", ["b"], includes=["c"]) 64 | group_list.add("c", ["c"], includes=["a"]) 65 | actual = group_list.get_elements(groups=["a"]) 66 | assert actual == ["c", "b", "a"] 67 | 68 | 69 | def test_unknown_group() -> None: 70 | group_list = GroupList(elements=["a", "b", "c"]) 71 | group_list.add("default", ["a", "b"]) 72 | with pytest.raises(GroupNotFound) as e: 73 | group_list.get_elements(groups=["no-such-group"]) 74 | assert e.value.parent_group is None 75 | assert e.value.group_name == "no-such-group" 76 | -------------------------------------------------------------------------------- /tsrc/local_manifest.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Optional 3 | 4 | from tsrc.git import get_current_branch, run_git 5 | from tsrc.manifest import Manifest, load_manifest, load_manifest_safe_mode 6 | from tsrc.manifest_common_data import ManifestsTypeOfData 7 | 8 | 9 | class LocalManifest: 10 | """Represent a manifest repository that has been cloned locally 11 | inside `/.tsrc/manifest`. 12 | 13 | Usage: 14 | 15 | >>> local_manifest = LocalManifest(Path(workspace / ".tsrc/manifest") 16 | 17 | # First, update the cloned repository using a remote git URL and a 18 | # branch: 19 | >>> manifest.update("git@acme.com/manifest.git", branch="devel") 20 | 21 | # Then, read the `manifest.yml` file from the clone repository: 22 | >>> manifest = local_manifest.get_manifest() 23 | 24 | """ 25 | 26 | def __init__(self, clone_path: Path) -> None: 27 | self.clone_path = clone_path 28 | 29 | def current_branch(self) -> str: 30 | return get_current_branch(self.clone_path) 31 | 32 | def init( 33 | self, 34 | url: str, 35 | *, 36 | branch: Optional[str], 37 | show_output: bool = True, 38 | show_cmd: bool = True, 39 | ) -> None: 40 | parent = self.clone_path.parent 41 | name = self.clone_path.name 42 | parent.mkdir(parents=True, exist_ok=True) 43 | cmd = ["clone", url] 44 | if branch: 45 | cmd += ["--branch", branch] 46 | cmd += [name] 47 | run_git( 48 | self.clone_path.parent, *cmd, show_output=show_output, show_cmd=show_cmd 49 | ) 50 | 51 | def get_manifest(self) -> Manifest: 52 | return load_manifest(self.clone_path / "manifest.yml") 53 | 54 | def get_manifest_safe_mode(self, mtod: ManifestsTypeOfData) -> Manifest: 55 | return load_manifest_safe_mode(self.clone_path / "manifest.yml", mtod) 56 | 57 | def update( 58 | self, url: str, *, branch: str, show_output: bool = True, show_cmd: bool = True 59 | ) -> None: 60 | run_git( 61 | self.clone_path, 62 | "remote", 63 | "set-url", 64 | "origin", 65 | url, 66 | show_output=show_output, 67 | show_cmd=show_cmd, 68 | ) 69 | run_git(self.clone_path, "fetch", show_output=show_output, show_cmd=show_cmd) 70 | run_git( 71 | self.clone_path, 72 | "checkout", 73 | "-B", 74 | branch, 75 | show_output=show_output, 76 | show_cmd=show_cmd, 77 | ) 78 | run_git( 79 | self.clone_path, 80 | "branch", 81 | branch, 82 | "--set-upstream-to", 83 | f"origin/{branch}", 84 | show_output=show_output, 85 | show_cmd=show_cmd, 86 | ) 87 | ref = f"origin/{branch}" 88 | run_git( 89 | self.clone_path, 90 | "reset", 91 | "--hard", 92 | ref, 93 | show_output=show_output, 94 | show_cmd=show_cmd, 95 | ) 96 | -------------------------------------------------------------------------------- /tsrc/local_future_manifest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Local Future Manifest 3 | 4 | Obtains information about Future Manifest 5 | by init or update Manifest repository 6 | to *local* directory. 7 | 8 | Local Future Manifest will be in: 9 | root_path / ".tsrc" / "future_manifest" 10 | 11 | This way we can see how Workspace will transform 12 | after the 'sync'. 13 | """ 14 | 15 | from typing import Dict, Tuple, Union 16 | 17 | import cli_ui as ui 18 | 19 | from tsrc.errors import LoadManifestSchemaError 20 | from tsrc.groups_to_find import GroupsToFind 21 | from tsrc.local_manifest import LocalManifest 22 | from tsrc.manifest import Manifest 23 | from tsrc.manifest_common import ManifestGetRepos 24 | from tsrc.manifest_common_data import ManifestsTypeOfData 25 | from tsrc.repo import Repo 26 | from tsrc.workspace import Workspace 27 | 28 | 29 | def get_local_future_manifests_manifest_and_repos( 30 | workspace: Workspace, 31 | gtf: GroupsToFind, 32 | must_find_all_groups: bool = False, 33 | use_same_future_manifest: bool = False, 34 | ) -> Tuple[ 35 | Union[Manifest, None], Union[Dict[str, Repo], None], bool, GroupsToFind, bool 36 | ]: 37 | # returns: lfm, lfm_repos, must_find_all_groups, gtf, report_skip_fm_update 38 | path = workspace.root_path / ".tsrc" / "future_manifest" 39 | path_to_m_file = path / "manifest.yml" 40 | report_skip_fm_update: bool = False 41 | 42 | # as Manifest.yml by itself does not have configuration, we need to check 43 | # Workspace config to apply some missing options 44 | clone_all_repos = False 45 | if workspace.config.clone_all_repos is True: 46 | clone_all_repos = True 47 | 48 | lfm = LocalManifest(path) 49 | if path.is_dir(): 50 | # if it is already present 51 | if use_same_future_manifest is False or not path_to_m_file.is_file(): 52 | lfm.update( 53 | url=workspace.config.manifest_url, 54 | branch=workspace.config.manifest_branch, 55 | show_output=False, 56 | show_cmd=False, 57 | ) 58 | else: 59 | report_skip_fm_update = True 60 | 61 | else: 62 | # first time use 63 | lfm.init( 64 | url=workspace.config.manifest_url, 65 | branch=workspace.config.manifest_branch, 66 | show_output=False, 67 | show_cmd=False, 68 | ) 69 | 70 | # read manifest file and obtain raw data 71 | # lfmm = lfm.get_manifest() 72 | try: 73 | lfmm = lfm.get_manifest_safe_mode(ManifestsTypeOfData.FUTURE) 74 | except LoadManifestSchemaError as lmse: 75 | ui.warning(lmse) 76 | return None, None, must_find_all_groups, gtf, False 77 | 78 | mgr = ManifestGetRepos(workspace, lfmm, True, clone_all_repos) 79 | 80 | # get repos that match 'groups' 81 | repos, must_find_all_groups, gtf = mgr.by_groups( 82 | gtf, must_find_all_groups=must_find_all_groups 83 | ) 84 | 85 | # repos: make Dict from List 86 | dict_repos: Dict[str, Repo] = {} 87 | for repo in repos: 88 | dict_repos[repo.dest] = repo 89 | 90 | return lfmm, dict_repos, must_find_all_groups, gtf, report_skip_fm_update 91 | -------------------------------------------------------------------------------- /tsrc/remote_setter.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List, Optional 3 | 4 | import cli_ui as ui 5 | 6 | from tsrc.executor import Outcome, Task 7 | from tsrc.git import run_git, run_git_captured 8 | from tsrc.git_remote import remote_urls_are_same 9 | from tsrc.repo import Remote, Repo 10 | 11 | 12 | class RemoteSetter(Task[Repo]): 13 | """ 14 | For each repository: 15 | 16 | * look for the remote configured in the manifest, 17 | * add any missing remote, 18 | * if a remote is found but with an incorrect URL, update its URL. 19 | 20 | """ 21 | 22 | def __init__(self, workspace_path: Path) -> None: 23 | self.workspace_path = workspace_path 24 | 25 | def describe_item(self, item: Repo) -> str: 26 | return item.dest 27 | 28 | def describe_process_start(self, item: Repo) -> List[ui.Token]: 29 | return ["Configuring remotes", item.dest] 30 | 31 | def describe_process_end(self, item: Repo) -> List[ui.Token]: 32 | return [ui.green, "ok", ui.reset, item.dest] 33 | 34 | def process(self, index: int, count: int, repo: Repo) -> Outcome: 35 | # Note: 36 | # When self.parallel is True we need to return a string describing 37 | # all the changes, otherwise, we can just call cli_ui.info() directly 38 | summary_lines = [] 39 | for remote in repo.remotes: 40 | existing_remote = self.get_remote(repo, remote.name) 41 | if existing_remote: 42 | if remote_urls_are_same(existing_remote.url, remote.url) is False: 43 | self.set_remote(repo, remote) 44 | summary_lines.append( 45 | f"{repo.dest}: remote '{remote.name}' set to '{remote.url}'" 46 | ) 47 | else: 48 | self.add_remote(repo, remote) 49 | summary_lines.append( 50 | f"{repo.dest}: added remote '{remote.name}' with url: '{remote.url}'" 51 | ) 52 | return Outcome.from_lines(summary_lines) 53 | 54 | def get_remote(self, repo: Repo, name: str) -> Optional[Remote]: 55 | full_path = self.workspace_path / repo.dest 56 | rc, url = run_git_captured(full_path, "remote", "get-url", name, check=False) 57 | if rc != 0: 58 | return None 59 | else: 60 | return Remote(name=name, url=url) 61 | 62 | def set_remote(self, repo: Repo, remote: Remote) -> None: 63 | full_path = self.workspace_path / repo.dest 64 | # fmt: off 65 | self.info_3( 66 | repo.dest + ":", "Update remote", ui.reset, 67 | ui.bold, remote.name, ui.reset, 68 | "to new url:", ui.brown, f"({remote.url})" 69 | ) 70 | # fmt: on 71 | run_git(full_path, "remote", "set-url", remote.name, remote.url) 72 | 73 | def add_remote(self, repo: Repo, remote: Remote) -> None: 74 | full_path = self.workspace_path / repo.dest 75 | # fmt: off 76 | self.info_3( 77 | repo.dest + ":", "Add remote", ui.reset, 78 | ui.bold, remote.name, ui.reset, ui.brown, f"({remote.url})" 79 | ) 80 | # fmt: on 81 | self.run_git(full_path, "remote", "add", remote.name, remote.url) 82 | -------------------------------------------------------------------------------- /docs/guide/ci.md: -------------------------------------------------------------------------------- 1 | # Using tsrc with Continuous Integration (CI) 2 | 3 | ## GitHub Actions 4 | 5 | Let suppose you have a private GitHub organization holding several private 6 | repositories and tsrc to synchronize them using the SSH protocol. Let suppose 7 | you want to use [GitHub Actions](https://docs.github.com/en/actions) to download 8 | the code source of your organization, compile it and run some non regression 9 | tests. What to write to achieve this with tsrc? 10 | 11 | ### Step 1: Your tsrc manifest 12 | 13 | Your tsrc `manifest.yml` looks something like this: 14 | 15 | ``` 16 | repos: 17 | - url: git@github.com:project1/foo 18 | dest: foo 19 | ``` 20 | 21 | The `git@` means SSH protocol. 22 | 23 | ### Step 2: Create your GitHub workflows file 24 | 25 | In your private GitHub repository holding the GitHub workflows files, create the 26 | folder `.github/workflows` and your yaml file with the desired name and the 27 | following content. For more information about GitHub actions syntax see this 28 | [video](https://youtu.be/R8_veQiYBjI): 29 | 30 | ``` 31 | name: tsrc with private github repos 32 | on: 33 | workflow_dispatch: 34 | branches: 35 | - main 36 | 37 | jobs: 38 | export_linux: 39 | runs-on: ubuntu-latest 40 | steps: 41 | - name: Installing tsrc tool 42 | run: | 43 | sudo apt-get update 44 | sudo apt-get install -y python3 45 | python -m pip install tsrc 46 | 47 | - name: Cloning private github repos 48 | run: | 49 | git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github.com/".insteadOf git@github.com: 50 | export WORKSPACE=$GITHUB_WORKSPACE/your_project 51 | mkdir -p $WORKSPACE 52 | cd $WORKSPACE 53 | tsrc init git@github.com:yourorganisation/manifest.git 54 | tsrc sync 55 | ``` 56 | 57 | This script will run on the latest Ubuntu Docker and triggers steps: 58 | - The first step named `Installing tsrc tool` allows to install python3 and then 59 | tsrc. 60 | - The second step named `Cloning private github repos` creates a folder named 61 | `your_project` for your workspace and call the initialization and 62 | synchronization of your repositories. 63 | 64 | The important command is: 65 | ``` 66 | git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github.com/".insteadOf git@github.com: 67 | ``` 68 | 69 | which allows to replace the SSH syntax by the HTTPs syntax on your GitHub repository names. 70 | 71 | ### Step 3: Create the GitHub secret 72 | 73 | For GitHub organization one member of the team has the responsibility to hold a 74 | `Personal access tokens` for the organization. Go https://github.com/settings/tokens 75 | and click on the button `Generate new token` then click on `repo` checkbox then click 76 | on the button `Generate token`. 77 | 78 | Now, this token shall be saved into an action secret named `ACCESS_TOKEN` inside 79 | the GitHub repository holding the GitHub workflows files. 80 | 81 | ### Step 4: Enjoy 82 | 83 | In the menu `Actions` of your repository you can trig the workflow. In this 84 | example we used `workflow_dispatch` to perform manual triggers. So click on the 85 | button to start the process. Once this step done with success, you can update 86 | your workflow yaml to complete your CI work: compilation of your project, run 87 | non regression tests, etc. 88 | -------------------------------------------------------------------------------- /tsrc/test/cli/test_log.py: -------------------------------------------------------------------------------- 1 | from cli_ui.tests import MessageRecorder 2 | 3 | from tsrc.test.helpers.cli import CLI 4 | from tsrc.test.helpers.git_server import GitServer 5 | 6 | 7 | def test_happy( 8 | tsrc_cli: CLI, git_server: GitServer, message_recorder: MessageRecorder 9 | ) -> None: 10 | """ 11 | Scenario: 12 | * Create a manifest with two repos, foo and bar 13 | * Initialize a workspace from this manifest 14 | * Create a tag named v0.1 on foo and bar 15 | * Run `tsrc log --from v0.1 16 | """ 17 | git_server.add_repo("foo") 18 | git_server.add_repo("spam") 19 | git_server.push_file("foo", "bar.txt", message="boring bar") 20 | git_server.tag("foo", "v0.1") 21 | git_server.tag("spam", "v0.1") 22 | manifest_url = git_server.manifest_url 23 | tsrc_cli.run("init", manifest_url) 24 | git_server.push_file("foo", "foo.txt", message="new foo!") 25 | tsrc_cli.run("sync") 26 | message_recorder.reset() 27 | 28 | tsrc_cli.run("log", "--from", "v0.1") 29 | 30 | assert message_recorder.find("new foo!") 31 | 32 | message_recorder.reset() 33 | tsrc_cli.run("log", "--from", "v0.1", "--to", "v0.1") 34 | assert not message_recorder.find("new foo!") 35 | 36 | 37 | def test_log_error(tsrc_cli: CLI, git_server: GitServer) -> None: 38 | """ 39 | Scenario: 40 | * Create a manifest with one repo, foo 41 | * Initialize a workspace from this manifest 42 | * Check that `tsrc log --from v0.1` fails (the `v0.1` tag does not exist) 43 | """ 44 | git_server.add_repo("foo") 45 | manifest_url = git_server.manifest_url 46 | tsrc_cli.run("init", manifest_url) 47 | 48 | tsrc_cli.run_and_fail("log", "--from", "v0.1") 49 | 50 | 51 | def test_use_given_group(tsrc_cli: CLI, git_server: GitServer) -> None: 52 | """ 53 | Scenario: 54 | * Create a manifest containing: 55 | * a group named 'group1' containing the repo 'foo' 56 | * a group named 'group2' containing the repo 'bar' 57 | * Initialize a workspace from this manifest using the 'group1' and 58 | 'group2' groups 59 | * Create a tag named v0.1 on foo 60 | * Run `tsrc --log --from v0.1 --group group1` 61 | """ 62 | 63 | git_server.add_group("group1", ["foo"]) 64 | git_server.add_group("group2", ["bar"]) 65 | 66 | manifest_url = git_server.manifest_url 67 | git_server.tag("foo", "v0.1") 68 | git_server.push_file("foo", "foo.txt", message="new foo!") 69 | 70 | tsrc_cli.run("init", manifest_url, "--groups", "group1", "group2") 71 | tsrc_cli.run("log", "--from", "v0.1", "--group", "group1") 72 | 73 | 74 | def test_missing_repos_from_given_group( 75 | tsrc_cli: CLI, git_server: GitServer, message_recorder: MessageRecorder 76 | ) -> None: 77 | """ 78 | Scenario: 79 | * Create a manifest with two disjoint groups, group1 and group2 80 | * For each repo, create v0.1 tag 81 | * Initialize a workspace from this manifest using group1 82 | * Run `tsrc log --from v0.1 --groups group1 group2` 83 | * Check it fails 84 | """ 85 | git_server.add_group("group1", ["foo"]) 86 | git_server.add_group("group2", ["bar"]) 87 | git_server.tag("foo", "v0.1") 88 | git_server.tag("bar", "v0.1") 89 | manifest_url = git_server.manifest_url 90 | tsrc_cli.run("init", manifest_url, "--group", "group1") 91 | 92 | message_recorder.reset() 93 | tsrc_cli.run_and_fail("log", "--from", "v0.1", "--groups", "group1", "group2") 94 | -------------------------------------------------------------------------------- /tsrc/cli/main.py: -------------------------------------------------------------------------------- 1 | """ Main tsrc entry point. """ 2 | 3 | import argparse 4 | import functools 5 | import os 6 | import sys 7 | from typing import Callable, Optional, Sequence 8 | 9 | import cli_ui as ui 10 | import colored_traceback 11 | 12 | from tsrc import __version__ 13 | from tsrc.cli import ( 14 | apply_manifest, 15 | dump_manifest, 16 | foreach, 17 | init, 18 | log, 19 | manifest, 20 | status, 21 | sync, 22 | ) 23 | from tsrc.errors import Error 24 | 25 | ArgsList = Optional[Sequence[str]] 26 | MainFunc = Callable[..., None] 27 | 28 | 29 | def main_wrapper(main_func: MainFunc) -> MainFunc: 30 | """Wraps main() entry point to better deal with errors.""" 31 | 32 | @functools.wraps(main_func) 33 | def wrapped(args: ArgsList = None) -> None: 34 | colored_traceback.add_hook() 35 | try: 36 | main_func(args=args) 37 | except Error as e: 38 | # "expected" failure, display it and exit note: we allow 39 | # Error instances to have an empty message. In that 40 | # case, do not print anything and assume relevant info has 41 | # already been printed. 42 | if e.message: # noqa: B306 43 | ui.error(e.message) # noqa: B306 44 | sys.exit(1) 45 | except KeyboardInterrupt: 46 | ui.warning("Interrupted by user, quitting") 47 | sys.exit(1) 48 | 49 | return wrapped 50 | 51 | 52 | def setup_ui(args: argparse.Namespace) -> None: 53 | """Configure the cli_ui package using options 54 | set on the command line and environment variables. 55 | 56 | """ 57 | verbose = False 58 | if os.environ.get("VERBOSE"): 59 | verbose = True 60 | if args.verbose: 61 | verbose = args.verbose 62 | ui.setup(verbose=verbose, quiet=args.quiet, color=args.color) 63 | 64 | 65 | @main_wrapper 66 | def main(args: ArgsList = None) -> None: 67 | """Main entry point.""" 68 | main_impl(args=args) 69 | 70 | 71 | def testable_main(args: ArgsList) -> None: 72 | """Same behavior as the main entrypoint, except we never 73 | hide backtraces when an exception is raised. 74 | 75 | """ 76 | main_impl(args=args) 77 | 78 | 79 | def main_impl(args: ArgsList = None) -> None: 80 | parser = argparse.ArgumentParser(prog="tsrc") 81 | parser.add_argument("--version", action="version", version="tsrc " + __version__) 82 | 83 | parser.add_argument("--verbose", help="show debug messages", action="store_true") 84 | parser.add_argument( 85 | "-q", "--quiet", help="only display warnings and errors", action="store_true" 86 | ) 87 | parser.add_argument( 88 | "--color", 89 | choices=["auto", "always", "never"], 90 | help="whether to enable colored output", 91 | ) 92 | 93 | actions_parser = parser.add_subparsers(help="available actions", dest="action") 94 | 95 | for module in ( 96 | apply_manifest, 97 | dump_manifest, 98 | foreach, 99 | init, 100 | log, 101 | manifest, 102 | status, 103 | sync, 104 | ): 105 | module.configure_parser(actions_parser) 106 | 107 | namespace = parser.parse_args(args=args) 108 | 109 | setup_ui(namespace) 110 | if not hasattr(namespace, "run"): 111 | parser.print_help() 112 | sys.exit(1) 113 | namespace.run(namespace) 114 | -------------------------------------------------------------------------------- /tsrc/test/cli/test_env_setter.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from tsrc.cli.env_setter import ( 4 | EnvSetter, 5 | get_repo_vars, 6 | get_status_vars, 7 | get_workspace_vars, 8 | ) 9 | from tsrc.git import GitStatus, run_git 10 | from tsrc.repo import Remote, Repo 11 | from tsrc.test.helpers.cli import CLI 12 | from tsrc.test.helpers.git_server import GitServer 13 | from tsrc.workspace import Workspace 14 | 15 | 16 | def test_set_project_dest_and_branch( 17 | tsrc_cli: CLI, git_server: GitServer, workspace_path: Path 18 | ) -> None: 19 | git_server.add_repo("foo") 20 | git_server.add_repo("bar", default_branch="devel") 21 | 22 | tsrc_cli.run("init", git_server.manifest_url) 23 | bar_path = workspace_path / "bar" 24 | run_git(bar_path, "checkout", "-b", "other") 25 | 26 | workspace = Workspace(workspace_path) 27 | manifest = workspace.get_manifest() 28 | env_setter = EnvSetter(workspace) 29 | 30 | workspace_vars = get_workspace_vars(workspace) 31 | assert workspace_vars["TSRC_MANIFEST_URL"] == git_server.manifest_url 32 | assert workspace_vars["TSRC_MANIFEST_BRANCH"] == "master" 33 | assert workspace_vars["TSRC_WORKSPACE_PATH"] == str(workspace_path) 34 | 35 | foo_repo = manifest.get_repo("foo") 36 | foo_env = env_setter.get_env_for_repo(foo_repo) 37 | # check that shared env is part of the result for foo 38 | assert foo_env["TSRC_MANIFEST_URL"] == git_server.manifest_url 39 | assert foo_env["TSRC_PROJECT_CLONE_URL"] == foo_repo.clone_url 40 | 41 | # check that bar and foo envs are different 42 | bar_repo = manifest.get_repo("bar") 43 | bar_env = env_setter.get_env_for_repo(bar_repo) 44 | assert bar_env["TSRC_PROJECT_DEST"] == "bar" 45 | assert bar_env["TSRC_PROJECT_MANIFEST_BRANCH"] == "devel" 46 | 47 | # check that git status is set 48 | assert bar_env["TSRC_PROJECT_STATUS_BRANCH"] == "other" 49 | 50 | 51 | def test_get_repo_vars() -> None: 52 | origin = Remote(name="origin", url="git@origin.tld") 53 | mirror = Remote(name="mirror", url="git@mirror.com") 54 | foo = Repo(dest="foo", remotes=[origin, mirror], branch="devel") 55 | 56 | actual = get_repo_vars(foo) 57 | assert actual["TSRC_PROJECT_DEST"] == "foo" 58 | assert actual["TSRC_PROJECT_MANIFEST_BRANCH"] == "devel" 59 | assert actual["TSRC_PROJECT_CLONE_URL"] == "git@origin.tld" 60 | assert actual["TSRC_PROJECT_REMOTE_ORIGIN"] == "git@origin.tld" 61 | assert actual["TSRC_PROJECT_REMOTE_MIRROR"] == "git@mirror.com" 62 | 63 | 64 | def test_git_status_vars(tmp_path: Path) -> None: 65 | git_status = GitStatus(tmp_path) 66 | git_status.untracked = 0 67 | git_status.added = 1 68 | git_status.staged = 2 69 | git_status.not_staged = 3 70 | git_status.behind = 4 71 | git_status.ahead = 5 72 | git_status.branch = "other" 73 | git_status.sha1 = "abcde43" 74 | git_status.tag = "some-tag" 75 | git_status.dirty = True 76 | 77 | actual = get_status_vars(git_status) 78 | 79 | assert actual["TSRC_PROJECT_STATUS_UNTRACKED"] == "0" 80 | assert actual["TSRC_PROJECT_STATUS_ADDED"] == "1" 81 | assert actual["TSRC_PROJECT_STATUS_STAGED"] == "2" 82 | assert actual["TSRC_PROJECT_STATUS_NOT_STAGED"] == "3" 83 | assert actual["TSRC_PROJECT_STATUS_BEHIND"] == "4" 84 | assert actual["TSRC_PROJECT_STATUS_AHEAD"] == "5" 85 | assert actual["TSRC_PROJECT_STATUS_BRANCH"] == "other" 86 | assert actual["TSRC_PROJECT_STATUS_SHA1"] == "abcde43" 87 | assert actual["TSRC_PROJECT_STATUS_TAG"] == "some-tag" 88 | assert actual["TSRC_PROJECT_STATUS_DIRTY"] == "true" 89 | -------------------------------------------------------------------------------- /tsrc/dump_manifest_helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | Manifest Dumper - Helpers 3 | 4 | helps Dumper to use unified dataclass 5 | that can be processed across various of cases 6 | and thus simplify already complex functions 7 | """ 8 | 9 | from dataclasses import dataclass 10 | from typing import Dict, List, Optional, Union 11 | 12 | import cli_ui as ui 13 | 14 | from tsrc.repo import Remote, Repo 15 | from tsrc.status_endpoint import CollectedStatuses, Status 16 | 17 | 18 | @dataclass(frozen=True) 19 | class ManifestRepoItem: 20 | branch: Optional[str] = None 21 | tag: Optional[str] = None 22 | sha1: Optional[str] = None 23 | empty: Optional[bool] = False 24 | ignore_submodules: Optional[bool] = False 25 | remotes: Optional[List[Remote]] = None 26 | groups_considered: Optional[bool] = False 27 | 28 | # positon-related 29 | ahead: int = 0 30 | behind: int = 0 31 | # TODO: implement test if required variables are set 32 | 33 | @property 34 | def clone_url(self) -> str: 35 | assert self.remotes 36 | return self.remotes[0].url 37 | 38 | 39 | class MRISHelpers: 40 | def __init__( 41 | self, 42 | statuses: Optional[CollectedStatuses] = None, 43 | w_repos: Optional[List[Repo]] = None, # Workspace's Repos 44 | repos: Optional[List[Repo]] = None, 45 | ) -> None: 46 | self.mris: Dict[str, ManifestRepoItem] = {} 47 | if bool(statuses) == bool(repos): 48 | return 49 | if statuses: 50 | self._statuses_to_mris(statuses, w_repos) 51 | if repos: 52 | self._repos_to_mris(repos) 53 | 54 | def _repo_to_mri( 55 | self, 56 | repo: Repo, 57 | ) -> ManifestRepoItem: 58 | return ManifestRepoItem( 59 | branch=repo.branch, 60 | tag=repo.tag, 61 | sha1=repo.sha1_full, 62 | # sha1=repo.sha1, 63 | ignore_submodules=repo.ignore_submodules, 64 | remotes=repo.remotes, 65 | ahead=repo._grabbed_ahead, 66 | behind=repo._grabbed_behind, 67 | ) 68 | 69 | def _repos_to_mris( 70 | self, 71 | repos: Union[List[Repo], None], 72 | ) -> None: 73 | if repos: 74 | for repo in repos: 75 | # skip empty Repo(s) 76 | if repo.branch or repo.tag or repo.sha1: 77 | self.mris[repo.dest] = self._repo_to_mri(repo) 78 | else: 79 | ui.warning(f"Skipping empty Repo: {repo.dest}") 80 | 81 | def _status_to_mri( 82 | self, 83 | status: Union[Status, Exception], 84 | w_repo: Repo, 85 | ) -> ManifestRepoItem: 86 | if isinstance(status, Status) and status.git.empty is False: 87 | return ManifestRepoItem( 88 | branch=status.git.branch, 89 | tag=status.git.tag, 90 | sha1=status.git.sha1_full, 91 | empty=status.git.empty, 92 | ignore_submodules=w_repo.ignore_submodules, 93 | remotes=w_repo.remotes, 94 | groups_considered=True, 95 | ahead=status.git.ahead, 96 | behind=status.git.behind, 97 | ) 98 | return ManifestRepoItem() 99 | 100 | def _statuses_to_mris( 101 | self, 102 | statuses: Union[CollectedStatuses, None], 103 | w_repos: Union[List[Repo], None], 104 | ) -> None: 105 | if statuses and w_repos: 106 | for repo in w_repos: 107 | dest = repo.dest 108 | if isinstance(statuses[dest], Status): 109 | self.mris[dest] = self._status_to_mri(statuses[dest], repo) 110 | -------------------------------------------------------------------------------- /docs/guide/fs.md: -------------------------------------------------------------------------------- 1 | # Performing file system operations 2 | 3 | ## Introduction 4 | 5 | When using `tsrc`, it is assumed that repositories are put in non-overlapping 6 | file system hierarchies, like this: 7 | 8 | ```text 9 | workspace/ 10 | project_1/ 11 | CMakeLists.txt 12 | foo.cpp 13 | bar.cpp 14 | project_2/ 15 | CMakeLists.txt 16 | spam.cpp 17 | eggs.cpp 18 | ``` 19 | 20 | Not like that, where `project_2` is inside a sub-directory of `project_1`: 21 | 22 | ```text 23 | workspace/ 24 | project_1/ 25 | CMakeLists.txt 26 | foo.cpp 27 | bar.cpp 28 | project_2/ 29 | CMakeLists.txt 30 | spam.cpp 31 | eggs.cpp 32 | ``` 33 | 34 | !!! note 35 | if you really need `project_2` to be a sub-directory of `project_1`, 36 | consider using *git submodules* instead. 37 | 38 | This is usually fine, except when `project_1` and `project_2` share some common configuration. 39 | 40 | For instance, you may want to use `clang-format` for both `project_1` and `project_2`. 41 | 42 | ## Copying a file 43 | 44 | One solution is to put the `.clang-format` configuration file in a repo named 45 | `common` and then tell `tsrc` to copy it at the root of the workspace: 46 | 47 | ```yaml 48 | repos: 49 | - dest: project_1 50 | url: git@acme.com:team/project_1 51 | 52 | - dest: project_2 53 | url: git@acme.com:team/project_2 54 | 55 | - dest: common 56 | url: git@acme.com:team/commont 57 | copy: 58 | - file: clang-format 59 | dest: .clang-format 60 | ``` 61 | 62 | ```bash 63 | $ tsrc sync 64 | => Cloning missing repos 65 | * (1/1) Cloning common 66 | Cloning into 'common'... 67 | ... 68 | => Performing filesystem operations 69 | * (1/1) Copy /path/to/work/common/clang-format -> /path/to/work/.clang-format 70 | ``` 71 | 72 | Notes: 73 | 74 | * `copy` only works with files, not directories. 75 | * The source path for a copy link is relative to associated repos destination, whereas 76 | the destination path of the copy is relative to the workspace root. 77 | 78 | ## Creating a symlink 79 | 80 | The above method works fine if the file does not change too often - if not, you may want to create 81 | a symbolic link instead: 82 | 83 | ```yaml 84 | repos: 85 | - dest: project_1 86 | url: git@acme.com:team/project_1 87 | 88 | - dest: project_2 89 | url: git@acme.com:team/project_2 90 | 91 | - dest: common 92 | url: git@acme.com:team/commont 93 | symlink: 94 | - source: .clang-format 95 | target: common/clang-format 96 | ``` 97 | 98 | ```bash 99 | $ tsrc sync 100 | => Cloning missing repos 101 | ... 102 | => Performing filesystem operations 103 | * (1/1) Lint /path/to/work/.clang-format -> common/.clang-format 104 | ``` 105 | 106 | Notes: 107 | 108 | * The source path for a symbolic link is relative to the top-level ``, whereas 109 | each target path is then relative to the associated source. (This path relationship 110 | is essentially identical to how `ln -s` works on the command line in Unix-like 111 | environments.) Multiple symlinks can be specified; each must specify a source and target. 112 | 113 | * Symlink creation is supported on all operating systems, but creation of NTFS symlinks on 114 | Windows requires that the current user have appropriate security policy permission 115 | (SeCreateSymbolicLinkPrivilege). By default, only administrators have that privilege set, 116 | although newer versions of Windows 10 support a Developer Mode that permits unprivileged 117 | accounts to create symlinks. Note that Cygwin running on Windows defaults to creating 118 | links via Windows shortcuts, which do *not* require any special privileges. 119 | (Cygwin's symlink behavior can be user controlled with the `winsymlinks` setting 120 | in the `CYGWIN` environment variable.) 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![image](https://img.shields.io/github/license/your-tools/tsrc.svg)](https://github.com/your-tools/tsrc/blob/main/LICENSE) 2 | 3 | [![image](https://github.com/your-tools/tsrc/workflows/tests/badge.svg)](https://github.com/your-tools/tsrc/actions) 4 | 5 | [![image](https://github.com/your-tools/tsrc/workflows/linters/badge.svg)](https://github.com/your-tools/tsrc/actions) 6 | 7 | [![image](https://img.shields.io/pypi/v/tsrc.svg)](https://pypi.org/project/tsrc/) 8 | 9 | [![image](https://img.shields.io/badge/deps%20scanning-pyup.io-green)](https://github.com/your-tools/tsrc/actions) 10 | 11 | # tsrc: manage groups of git repositories 12 | 13 | ## Overview 14 | 15 | tsrc is a command-line tool that helps you manage groups of several git 16 | repositories. 17 | 18 | It can be [seen in action on asciinema.org](https://asciinema.org/a/131625). 19 | 20 | ## Requirements 21 | 22 | Python **3.8** or later 23 | 24 | ## Installation 25 | 26 | Use `pipx` (recommended) or `pip` (ok, if you know what you're doing) to install. 27 | 28 | Please see the [installation docs](https://your-tools.github.io/tsrc/getting-started/#installing_tsrc) for more info. 29 | 30 | ## Usage Example 31 | 32 | - Create a *manifest* repository. (`git@example.org/manifest.git`) 33 | - Add a file named `manifest.yml` at the root of the *manifest* 34 | repository. 35 | 36 | `manifest.yml`: 37 | 38 | ```yaml 39 | repos: 40 | - url: git@example.com/foo.git 41 | dest: foo 42 | 43 | - url: git@example.com/bar.git 44 | dest: bar 45 | ``` 46 | 47 | It is convenient while optional to include the manifest repository itself in your `manifest.yml`. It will allow you to have a local copy of you manifest repository to easily make changes to it in the future. 48 | 49 | - commit your `manifest.yml` and push the changes to the manifest 50 | repository. 51 | - Create a new workspace with all the repositories listed in the 52 | manifest: 53 | 54 | ```console 55 | $ tsrc init git@git.local/manifest.git 56 | 57 | :: Configuring workspace in /path/to/work 58 | ... 59 | => Cloning missing repos 60 | * (1/2) foo 61 | ... 62 | * (2/2) bar 63 | ... 64 | : Configuring remotes 65 | Done ✓ 66 | ``` 67 | 68 | - Synchronize all the repositories in the workspace: 69 | 70 | ```console 71 | $ tsrc sync 72 | => Updating manifest 73 | ... 74 | :: Configuring remotes 75 | :: Synchronizing workspace 76 | * (1/2) foo 77 | => Fetching origin 78 | => Updating branch 79 | Already up to date 80 | * (2/2) bar 81 | => Updating branch 82 | Updating 29ac0e1..b635a43 83 | Fast-forward 84 | bar.txt | 1 + 85 | 1 file changed, 1 insertion(+) 86 | create mode 100644 bar.txt 87 | Done ✓ 88 | ``` 89 | 90 | ## Documentation 91 | 92 | For more details and examples, please refer to [tsrc documentation](https://your-tools.github.io/tsrc/). 93 | 94 | ## Release notes 95 | 96 | Detailed changes for each release are documented in the [changelog](https://your-tools.github.io/tsrc/changelog/). 97 | 98 | ## Contributing 99 | 100 | We welcome feedback, [bug reports](https://github.com/your-tools/tsrc/issues), and bug fixes in the form of [pull requests](https://github.com/your-tools/tsrc/pulls). 101 | 102 | Detailed instructions can be found [in the documentation](https://your-tools.github.io/tsrc). 103 | 104 | ## License 105 | 106 | tsrc is licensed under a [BSD 3-Clause license](https://github.com/your-tools/tsrc/blob/main/LICENSE). 107 | 108 | ## History 109 | 110 | This project was originally hosted on the [TankerHQ](https://github.com/TankerHQ) organization, which was [dmerejkowsky](https://github.com/dmerejkowsky)'s employer from 2016 to 2021. They kindly agreed to give back ownership of this project to Dimitri in 2021 - thanks! Dimitri later on shared this project even more by moving it to the [your-tools](https://github.com/your-tools) organization. 111 | -------------------------------------------------------------------------------- /tsrc/cli/log.py: -------------------------------------------------------------------------------- 1 | """ Entry point for `tsrc log`. """ 2 | 3 | import argparse 4 | from pathlib import Path 5 | from typing import List 6 | 7 | import cli_ui as ui 8 | 9 | from tsrc.cli import ( 10 | add_num_jobs_arg, 11 | add_repos_selection_args, 12 | add_workspace_arg, 13 | get_num_jobs, 14 | get_workspace_with_repos, 15 | ) 16 | from tsrc.errors import Error, MissingRepoError 17 | from tsrc.executor import Outcome, Task, process_items 18 | from tsrc.git import run_git_captured 19 | from tsrc.repo import Repo 20 | 21 | 22 | def configure_parser(subparser: argparse._SubParsersAction) -> None: 23 | parser = subparser.add_parser("log") 24 | add_workspace_arg(parser) 25 | add_repos_selection_args(parser) 26 | parser.add_argument( 27 | "--from", dest="from_ref", metavar="FROM", help="run `git log` from this ref" 28 | ) 29 | parser.add_argument( 30 | "--to", 31 | dest="to_ref", 32 | default="HEAD", 33 | help="run `git log` until this ref", 34 | ) 35 | add_num_jobs_arg(parser) 36 | parser.set_defaults(run=run) 37 | 38 | 39 | class LogCollector(Task[Repo]): 40 | def __init__(self, workspace_path: Path, *, from_ref: str, to_ref: str) -> None: 41 | self.workspace_path = workspace_path 42 | self.from_ref = from_ref 43 | self.to_ref = to_ref 44 | 45 | def describe_item(self, item: Repo) -> str: 46 | return item.dest 47 | 48 | def describe_process_start(self, item: Repo) -> List[ui.Token]: 49 | return [item.dest] 50 | 51 | def describe_process_end(self, item: Repo) -> List[ui.Token]: 52 | return [ui.green, "ok", ui.reset, item.dest] 53 | 54 | def process(self, index: int, count: int, repo: Repo) -> Outcome: 55 | # We just need to compute a summary here with the log between 56 | # self.from_ref and self.to_ref 57 | # 58 | # Note: make sure that when there is no diff between 59 | # self.from_ref and self.to_ref, the summary is empty, 60 | # so that the repo is not shown by OutcomeCollection.print_summary() 61 | repo_path = self.workspace_path / repo.dest 62 | if not repo_path.exists(): 63 | raise MissingRepoError(repo.dest) 64 | 65 | # The main reason for the `git log` command to fail is if `self.from_ref` or 66 | # `self.to_ref` references are not found for the repo, so check for this case 67 | # explicitly 68 | rc, _ = run_git_captured(repo_path, "rev-parse", self.from_ref, check=False) 69 | if rc != 0: 70 | raise Error(f"{self.from_ref} not found") 71 | rc, _ = run_git_captured(repo_path, "rev-parse", self.to_ref, check=False) 72 | if rc != 0: 73 | raise Error(f"{self.to_ref} not found") 74 | 75 | colors = ["green", "reset", "yellow", "reset", "bold blue", "reset"] 76 | log_format = "%m {}%h{} - {}%d{} %s {}<%an>{}" 77 | log_format = log_format.format(*("%C({})".format(x) for x in colors)) 78 | cmd = [ 79 | "log", 80 | "--color=always", 81 | f"--pretty=format:{log_format}", 82 | f"{self.from_ref}...{self.to_ref}", 83 | ] 84 | rc, out = run_git_captured(repo_path, *cmd, check=True) 85 | if out: 86 | lines = [repo.dest, "-" * len(repo.dest), out] 87 | return Outcome.from_lines(lines) 88 | else: 89 | return Outcome.empty() 90 | 91 | 92 | def run(args: argparse.Namespace) -> None: 93 | workspace = get_workspace_with_repos(args) 94 | num_jobs = get_num_jobs(args) 95 | from_ref = args.from_ref 96 | to_ref = args.to_ref 97 | repos = workspace.repos 98 | log_collector = LogCollector(workspace.root_path, from_ref=from_ref, to_ref=to_ref) 99 | collection = process_items(repos, log_collector, num_jobs=num_jobs) 100 | collection.print_summary() 101 | if collection.errors: 102 | ui.error("Error when collecting logs") 103 | collection.print_errors() 104 | raise LogCollectorFailed 105 | 106 | 107 | class LogCollectorFailed(Error): 108 | pass 109 | -------------------------------------------------------------------------------- /tsrc/file_system.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import os 3 | import shutil 4 | from dataclasses import dataclass 5 | from pathlib import Path 6 | 7 | import cli_ui as ui 8 | 9 | from tsrc.errors import Error 10 | 11 | 12 | class FileSystemOperation(metaclass=abc.ABCMeta): 13 | @abc.abstractmethod 14 | def describe(self, workspace_path: Path) -> str: 15 | pass 16 | 17 | @abc.abstractmethod 18 | def perform(self, workspace_path: Path) -> None: 19 | pass 20 | 21 | @abc.abstractmethod 22 | def get_repo(self) -> str: 23 | pass 24 | 25 | 26 | @dataclass(frozen=True) 27 | class Copy(FileSystemOperation): 28 | repo: str 29 | src: str 30 | dest: str 31 | 32 | def get_repo(self) -> str: 33 | return self.repo 34 | 35 | def describe(self, workspace_path: Path) -> str: 36 | src_path = workspace_path / self.repo / self.src 37 | dest_path = workspace_path / self.dest 38 | return f"Copy {src_path} -> {dest_path}" 39 | 40 | def perform(self, workspace_path: Path) -> None: 41 | src_path = workspace_path / self.repo / self.src 42 | dest_path = workspace_path / self.dest 43 | shutil.copy(src_path, dest_path) 44 | 45 | 46 | @dataclass(frozen=True) 47 | class Link(FileSystemOperation): 48 | repo: str 49 | source: str 50 | target: str 51 | 52 | def get_repo(self) -> str: 53 | return self.repo 54 | 55 | def describe(self, workspace_path: Path) -> str: 56 | source = workspace_path / self.source 57 | return f"Link {source} -> {self.target}" 58 | 59 | def perform(self, workspace_path: Path) -> None: 60 | source = workspace_path / self.source 61 | target = Path(self.target) 62 | safe_link(source=source, target=target) 63 | 64 | 65 | def safe_link(*, source: Path, target: Path) -> None: 66 | """Safely create a link in 'source' pointing to 'target'.""" 67 | # Not: we need to call both islink() and exist() to safely ensure 68 | # that the link exists: 69 | # 70 | # islink() exists() Description 71 | # ---------------------------------------------------------- 72 | # False False source does not currently exist : OK 73 | # False True source corresponds to a file or dir : Error! 74 | # True False broken symlink, need to remove 75 | # True True symlink points to a valid target, check target 76 | # ---------------------------------------------------------- 77 | make_link = check_link(source=source, target=target) 78 | if make_link: 79 | ui.info_3("Creating link", source, "->", target) 80 | 81 | os.symlink( 82 | os.path.normpath(target), 83 | os.path.normcase(source), 84 | target_is_directory=target.is_dir(), 85 | ) 86 | 87 | 88 | def check_link(*, source: Path, target: Path) -> bool: 89 | remove_link = False 90 | if source.exists() and not source.is_symlink(): 91 | raise Error("Specified symlink source exists but is not a link") 92 | if source.is_symlink(): 93 | if source.exists(): 94 | # symlink exists and points to some target 95 | current_target = Path(os.readlink(str(source))) 96 | if current_target.resolve() == target.resolve(): 97 | ui.info_3("Leaving existing link") 98 | return False 99 | else: 100 | ui.info_3("Replacing existing link") 101 | remove_link = True 102 | else: 103 | # symlink exists, but points to a non-existent target 104 | ui.info_3("Replacing broken link") 105 | remove_link = True 106 | if remove_link: 107 | os.unlink(source) 108 | return True 109 | 110 | 111 | def make_relative(in_path: Path) -> Path: 112 | # takes input Path and make it relative to current path 113 | # if it is not relative already 114 | if in_path.is_absolute() is True: 115 | in_path_dir = os.path.dirname(in_path) 116 | in_path_file = os.path.basename(in_path) 117 | in_path = Path(os.path.join(os.path.relpath(in_path_dir), in_path_file)) 118 | return in_path 119 | -------------------------------------------------------------------------------- /docs/ref/cli.md: -------------------------------------------------------------------------------- 1 | # Command line usage 2 | 3 | ## Important note 4 | 5 | We use the [argparse](https://docs.python.org/3/library/argparse.html) library to 6 | parse command line arguments, so the `--help` messages are always up-to-date, 7 | probably more so than this documentation :) 8 | 9 | ## General 10 | 11 | `tsrc` uses the same "subcommand" pattern as git does. 12 | 13 | Options common to all commands are placed right before the command name. 14 | 15 | Options after the command name only apply to this command. 16 | 17 | For instance: 18 | 19 | ```console 20 | $ tsrc --verbose sync 21 | $ tsrc init MANIFEST_URL 22 | ``` 23 | 24 | ## Goodies 25 | 26 | First, note that like `git`, tsrc will walk up the folders hierarchy 27 | looking for a `.tsrc` folder, which means you can run tsrc commands 28 | anywhere in your workspace, not just at the top. 29 | 30 | Second, almost all commands run the operation in parallel. For instance, 31 | `tsrc sync` by default will use as many jobs as the number of CPUs 32 | available on the current machine to synchronize the repos in your workspace. 33 | If this behavior is not desired, you can specify a greater (or lower) 34 | number of jobs using something like `tsrc sync -j2`, or disable the 35 | parallelism completely with `-j1`. You can also set the default number 36 | of jobs by using the `TSRC_PARALLEL_JOBS ` environment variable. 37 | 38 | ## Global options 39 | 40 | --verbose 41 | : show verbose messages 42 | 43 | -q, --quiet 44 | : hide everything except errors and warnings 45 | 46 | --color [always|never|auto] 47 | : control using color for messages (default 'auto', on if stdout is a terminal) 48 | 49 | ## Usage 50 | 51 | 52 | tsrc init MANIFEST_URL [--group GROUP1, GROUP2] [--singular-remote SINGULAR_REMOTE] 53 | : Initializes a new workspace. 54 | 55 | MANIFEST_URL should be a git URL containing a valid 56 | `manifest.yml` file. 57 | 58 | The `-g,--groups` option can be used to specify a list of groups 59 | to use when cloning repositories. 60 | 61 | The `-i` "inclusive regular expression" and `-e` "exclusive regular expression" options 62 | can be combined with the group option to filter for repositories within a group. `-i` takes 63 | precedence if both options are present. 64 | 65 | The `-s,--shallow` option can be used to make shallow clone of all repositories. 66 | 67 | If you want to add or remove a group in your workspace, you can 68 | edit the configuration file in `/.tsrc/config.yml` 69 | 70 | The `-r,--singular-remote` option can be used to set a fixed remote to use when cloning 71 | and syncing the repositories. If this flag is set, the remote from the manifest 72 | with the given name will be used for all repos. It is an error if a repo 73 | does not have this remote specified. 74 | 75 | 76 | tsrc foreach -- command --opt1 arg1 77 | : Runs `command --opt1 arg1` in every repository, and report failures 78 | at the end. 79 | 80 | Note the `--` token to separate options for `command` from options for 81 | `tsrc`. 82 | 83 | tsrc foreach -c 'command --opt1 arg1' 84 | : Ditto, but uses a shell (`/bin/sh` on Linux or macOS, `cmd.exe` on Windows). 85 | 86 | 87 | tsrc log --from FROM [--to TO] 88 | : Display a summary of all changes since `FROM` (should be a tag), 89 | to `TO` (defaulting to `master`). 90 | 91 | Note that if no changes are found, the repository will not be displayed at 92 | all. 93 | 94 | tsrc status 95 | : Displays a summary of the status of your workspace: 96 | 97 | * Shows dirty repositories 98 | * Shows repositories not on the expected branch 99 | 100 | tsrc sync [--no-correct-branch] 101 | : Updates all the repositories and shows a summary at the end. 102 | If any of the repositories is not on the configured branch, but it is clean 103 | and the `--no-correct-branch` flag is NOT set, then the branch is changed to 104 | the configured one and then the repository is updated. Otherwise that repository 105 | will not be not updated. 106 | 107 | tsrc version 108 | : Displays `tsrc` version number, along additional data if run from a git clone. 109 | 110 | tsrc apply-manifest PATH 111 | : Apply changes from the manifest file located at `PATH`. Useful to check changes 112 | in the manifest before publishing them to the manifest repository. 113 | -------------------------------------------------------------------------------- /docs/guide/manifest.md: -------------------------------------------------------------------------------- 1 | # Editing the manifest safely 2 | 3 | ## Introduction: when things go wrong 4 | 5 | Let's assume you've successfully implemented `tsrc` for your 6 | organization - now need to make sure to not break anyone's workflow. 7 | 8 | Let's see what could go wrong if you make mistakes while editing the 9 | manifest, using a branch called `broken` for the sake of the example). 10 | 11 | First, let's see what happens if you break the YAML syntax: 12 | 13 | ```diff 14 | commit 1633c5a6 (HEAD -> broken, origin/broken) 15 | 16 | Break the manifest syntax 17 | 18 | diff --git a/manifest.yml b/manifest.yml 19 | index fe74142..068c35e 100644 20 | --- a/manifest.yml 21 | +++ b/manifest.yml 22 | @@ -1,4 +1,4 @@ 23 | -repos: 24 | +repos 25 | - url: git@github.com:your-tools/bar.git 26 | dest: bar 27 | ``` 28 | 29 | After this change is push, anyone using the `broken` branch of the 30 | manifest will be faced with this kind of error message: 31 | 32 | ```bash 33 | $ tsrc sync 34 | ``` 35 | 36 | ```text 37 | => Updating manifest 38 | Reset branch 'broken' 39 | Your branch is up to date with 'origin/broken'. 40 | Branch 'broken' set up to track remote branch 'broken' from 'origin'. 41 | HEAD is now at 1633c5a Break the manifest syntax 42 | Error: /path/to/work/.tsrc/manifest/manifest.yml: mapping values are 43 | not allowed here : 44 | 45 | - url: git@gitlab.acme.com:your-team/foo 46 | ^ (line: 2) 47 | 48 | ``` 49 | 50 | Similarly, if you put an invalid URL in the manifest, like this: 51 | 52 | 53 | ```diff 54 | commit ccfb902 (HEAD -> broken, origin/broken) 55 | 56 | Use invalid URL for bar repo 57 | 58 | diff --git a/manifest.yml b/manifest.yml 59 | index fe74142..068c35e 100644 60 | --- a/manifest.yml 61 | +++ b/manifest.yml 62 | @@ -1,4 +1,4 @@ 63 | repos: 64 | - - url: git@gitlab.acme.com:your-team/bar 65 | + - url: git@gitlab.acme.com:your-team/invalid 66 | dest: bar 67 | ``` 68 | 69 | Users will get: 70 | 71 | ```bash 72 | $ tsrc sync 73 | ``` 74 | 75 | ```text 76 | :: Using workspace in /path/to/work 77 | => Updating manifest 78 | ... 79 | HEAD is now at ccfb902 Use invalid URL 80 | => Cloning missing repos 81 | => Configuring remotes 82 | * bar: Update remote origin to new url: (git@acme.com:your-team/invalid.git) 83 | ... 84 | => Synchronizing repos 85 | * (1/2) Synchronizing bar 86 | * Fetching origin 87 | ERROR: Repository not found. 88 | fatal: Could not read from remote repository. 89 | 90 | Please make sure you have the correct access rights 91 | and the repository exists. 92 | Error: fetch from 'origin' failed 93 | * (2/2) Synchronizing foo 94 | ... 95 | Error: Failed to synchronize the following repos: 96 | * bar : fetch from 'origin' failed 97 | ``` 98 | 99 | This will probably not be a huge problem for you, dear reader, 100 | because you know about tsrc's manifest and its syntax. 101 | 102 | It *will*, however, be a problem for people who are just using `tsrc` 103 | without knowledge of how it is implemented, because those error messages 104 | will *definitely* confuse them. 105 | 106 | 107 | ## Using the apply-manifest command to avoid breaking developers workflow 108 | 109 | If you have a file on your machine containing the manifest changes, you 110 | can use `tsrc apply-manifest` to check those changes against your own 111 | workspace: 112 | 113 | ```bash 114 | $ cd /path/to/work 115 | $ tsrc apply-manifest /path/to/manifest-repo/manifest.yml 116 | # Check that the changes are OK 117 | # If so, commit and push manifest changes: 118 | $ cd path/to/manifest-repo 119 | $ git commit -a -m "..." 120 | $ git push 121 | # Now you know that everyone can safely run `tsrc sync` 122 | ``` 123 | 124 | ## Additional notes 125 | 126 | * It is **not** advised to edit the file in 127 | `.tsrc/manifest/manifest.yml` directly, because `tsrc sync` will 128 | silently undo any local changes made to this file. This is a known bug, 129 | see [#279](https://github.com/your-tools/tsrc/issues/279) for details. 130 | 131 | 132 | * It is common to place the manifest repo itself in the manifest - so it's easy to edit or read: 133 | 134 | ```yaml 135 | # In acme.com:your-team/manifest - manifest.yml 136 | repos: 137 | - url: git@acme.com:your-team/manifest 138 | dest: manifest 139 | 140 | - url: git@acme.com:your-team/foo 141 | dest: foo 142 | 143 | - url: git@acme.com:your-team/bar 144 | dest: bar 145 | ``` 146 | 147 | In that case, you would use: 148 | 149 | ``` 150 | $ tsrc apply-manifest /manifest/manifest.yml 151 | ``` 152 | 153 | to check changes before pushing them. 154 | 155 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | ## What does the name mean? 4 | 5 | The `t` stands for `tool` and `src` for `sources`. 6 | 7 | If you speak French, you can also remember the name as "tes sources". 8 | 9 | ## Why not repo? 10 | 11 | We used [repo](https://android.googlesource.com/tools/repo/) for a while, but 12 | found that tsrc had both a better command line API and a nicer output. 13 | 14 | On a less subjective level: 15 | 16 | * Good support for Windows (no need for Cygwin or anything like that) 17 | 18 | * Also, tsrc tries hard to never do any destructive operation or unexpected 19 | actions. 20 | 21 | For instance, `tsrc` never puts you in a "detached HEAD" state, 22 | nor does automatic rebase. It also never touches dirty repositories. 23 | 24 | This is achieved by using mostly 'porcelain' commands from git, instead of 25 | relying on plumbings internals. 26 | 27 | 28 | Also (and this matters a lot if you think about contribution): 29 | 30 | * Uses PEP8 coding style, enforced with `black` 31 | * Comprehensive test suite 32 | * Fully type-checked with `mypy` 33 | 34 | Note that there are a few features present in `repo` that are missing from `tsrc` 35 | (but may be implemented in the future). Feel free to open a feature request 36 | if needed! 37 | 38 | ## Why not git-subrepo, mu-repo, or gr? 39 | 40 | All this projects are fine but did not match our needs: 41 | 42 | * [git-subrepo](https://github.com/ingydotnet/git-subrepo) squashes commits, and 43 | we prefer having normal clones everywhere. 44 | * [mu-repo](https://fabioz.github.io/mu-repo/) is nice and contains an 45 | interesting dependency management feature, but currently we do not need this complexity. 46 | 47 | In any case, now that the whole team is using `tsrc` all the time, it's likely 48 | we'll keep using `tsrc` in the future. 49 | 50 | ## Why not git submodule? 51 | 52 | It's all about workflow. 53 | 54 | With `git-submodule`, you have a 'parent' repository and you freeze the state of 55 | the 'children' repositories to a specific commit. 56 | 57 | It's useful when you want to re-build a library you've forked when you build 58 | your main project, or when you have a library or build tools you want to 59 | factorize across repositories: this means that each 'parent' repository can 60 | have its children on any commit they want. 61 | 62 | With `tsrc`, all repositories are equal, and what you do instead is to make sure 63 | all the branches (or tags) are consistent across repositories. 64 | 65 | For instance, if you have `foo` and `bar`, you are going to make sure the 66 | 'master' branch of `foo` is always compatible to the 'master' branch of `bar`. 67 | 68 | Or if you want to go back to the state of the '0.42' release, you will run: 69 | `tsrc foreach -- git reset --hard v0.42`. 70 | 71 | Note that since `tsrc 0.2` you can also freeze the commits of some of the 72 | repositories. 73 | 74 | Last but not least, if you really need to use fixed references, you may 75 | do so by adding a `sha1` or `tag` line to the manifest. See the 76 | [relevant guide](guide/fixed-refs.md) for more details. 77 | 78 | 79 | ## Why not using pygit2 or similar instead of running git commands? 80 | 81 | First off, we do use `pygit2`, but only for tests. 82 | 83 | Second, the `pygit2` package depends on a 3rd party C library (`libgit2`) - 84 | and that can cause problems in certain cases. If we can, we prefer 85 | using pure-Python libraries for the production code. 86 | 87 | Finally, we prefer calling git "porcelain" commands, both for readability 88 | of the source code and ease of debugging (see below). 89 | 90 | ## Why do you hide which git commands are run? 91 | 92 | It's mainly a matter of not cluttering the output. 93 | We take care of keeping the output of `tsrc` both concise, readable and 94 | informative. 95 | 96 | That being said: 97 | 98 | * In case a git command fails, we'll display the full command that was run. 99 | * If you still need to see *all* the git commands that are run, we provide a 100 | `--verbose` flag, like so: `tsrc --verbose sync` 101 | 102 | ## Why YAML? 103 | 104 | It's nice to read and write, and we use the excellent [ruamel.yaml]( 105 | https://yaml.readthedocs.io/en/latest/) which even has round-trip support. 106 | 107 | Also, being Python fans, we don't mind that white space is part of the syntax. 108 | 109 | ## Why do I have to create a separate git repo with just one file in it? 110 | 111 | See [#235](https://github.com/your-tools/tsrc/issues/235) for why you can't 112 | have multiple manifest files in the same repository. 113 | 114 | Also, note that you can put other files in the repo - for instance, 115 | add a CI script that verifies the yaml syntax and checks that all the repos 116 | in the manifest can be cloned. 117 | -------------------------------------------------------------------------------- /tsrc/pcs_repo.py: -------------------------------------------------------------------------------- 1 | """Pseudo Current Static Repo. 2 | 3 | 'pseudo' as it is not full-fledged Repository 4 | 'current' as it only get current data at the moment, 5 | 'static' as once object is initialized, it cannot be changed 6 | 'repo' as it has repo-like features 7 | 8 | This class is only for data transfer. 9 | The reason behind is not to pass many variables 10 | into the function, but one. 11 | """ 12 | 13 | from dataclasses import dataclass 14 | from typing import Dict, List, Optional, Tuple, Union 15 | 16 | from tsrc.git_remote import remote_urls_are_same 17 | from tsrc.groups_to_find import GroupsToFind 18 | from tsrc.manifest_common import ManifestGetRepos 19 | from tsrc.repo import Remote, Repo 20 | from tsrc.status_endpoint import Status 21 | from tsrc.workspace import Workspace 22 | 23 | 24 | @dataclass(frozen=True) 25 | class PCSRepo: 26 | dest: str 27 | branch: Union[str, None] 28 | url: Optional[str] = None 29 | _origin = "origin" 30 | 31 | def get_origin(self) -> str: 32 | return type(self)._origin 33 | 34 | origin = property(get_origin) 35 | 36 | 37 | def repo_from_pcsrepo( 38 | st_m: PCSRepo, 39 | ) -> Union[Repo, None]: 40 | if st_m.url and st_m.branch: 41 | origin = Remote(st_m.origin, st_m.url) 42 | remotes = [] 43 | remotes.append(origin) 44 | return Repo( 45 | dest=st_m.dest, 46 | remotes=remotes, 47 | branch=st_m.branch, 48 | ) 49 | return None 50 | 51 | 52 | StatusOrError = Union[Status, Exception] 53 | 54 | 55 | def get_deep_manifest_from_local_manifest_pcsrepo( 56 | # manifest: Manifest, 57 | workspace: Workspace, 58 | # groups: Union[List[str], None], 59 | gtf: GroupsToFind, 60 | ) -> Tuple[Union[PCSRepo, None], GroupsToFind]: 61 | """ 62 | Returns: 63 | * 1st: PCSRepo of Deep Manifest (if found) 64 | * 2nd: GroupsToFind: updated for future use 65 | """ 66 | manifest = workspace.local_manifest.get_manifest() 67 | mgr = ManifestGetRepos(workspace, manifest, workspace.config.clone_all_repos) 68 | all_repos, _, new_gtf = mgr.by_groups(gtf) 69 | _, pcs_repo = get_deep_manifest_pcsrepo(all_repos, workspace.config.manifest_url) 70 | 71 | return pcs_repo, new_gtf 72 | 73 | 74 | def get_deep_manifest_pcsrepo( 75 | all_repos: List[Repo], 76 | m_url: str, 77 | ) -> Tuple[List[Repo], Union[PCSRepo, None]]: 78 | """Gets Deep Manifest properly. 79 | If you call this function from 'status', than 80 | you can ignore 1st returned value and just use the 2nd one""" 81 | repos = [] 82 | for repo in all_repos: 83 | repo_remotes = repo.remotes 84 | is_found = False 85 | for remote in repo_remotes: 86 | if remote.url and remote_urls_are_same(remote.url, m_url) is True: 87 | is_found = True 88 | break 89 | if is_found is True: 90 | repos += [repo] 91 | break 92 | dm = None 93 | if repos: 94 | dm = PCSRepo(repos[0].dest, repos[0].branch, url=m_url) 95 | return repos, dm 96 | 97 | 98 | """here 'url' is provided directly 99 | search through statuses: 100 | (ManifestStatus (as '.manifest)) -> 101 | (Repo (as '.repo))""" 102 | 103 | 104 | def get_workspace_manifest_pcsrepo( 105 | statuses: Dict[str, StatusOrError], 106 | m_url: str, 107 | ) -> Union[PCSRepo, None]: 108 | for dest, status in statuses.items(): 109 | if isinstance(status, Status): 110 | for remote in status.manifest.repo.remotes: 111 | if remote_urls_are_same(remote.url, m_url) is True: 112 | branch = None 113 | if isinstance(status.git.branch, str): 114 | branch = status.git.branch 115 | return PCSRepo(dest=dest, branch=branch, url=m_url) 116 | return None 117 | 118 | 119 | """here the 'url' is taken from 'workspace' 120 | search through Repos to find Manifest Repo""" 121 | 122 | 123 | # TODO: move this to ManifestTools in some time 124 | def is_manifest_in_workspace( 125 | workspace: Workspace, 126 | repos: List[Repo], 127 | ) -> Union[PCSRepo, None]: 128 | for x in repos: 129 | this_dest = x.dest 130 | this_branch = x.branch 131 | for y in x.remotes: 132 | if ( 133 | y.url 134 | and remote_urls_are_same( # noqa: W503 135 | y.url, workspace.config.manifest_url 136 | ) 137 | is True 138 | ): 139 | # go with 1st one found 140 | return PCSRepo( 141 | this_dest, this_branch, url=workspace.config.manifest_url 142 | ) 143 | return None 144 | -------------------------------------------------------------------------------- /docs/ref/manifest-config.md: -------------------------------------------------------------------------------- 1 | # Manifest configuration 2 | 3 | The manifest configuration must be stored in a file named `manifest.yml`, using 4 | [YAML](https://yaml.org) syntax. 5 | 6 | It is always parsed as a *mapping*. Here's an example: 7 | 8 | ```yaml 9 | repos: 10 | - url: git@gitlab.local:proj1/foo 11 | dest: foo 12 | branch: next 13 | 14 | remotes: 15 | - name: origin 16 | url: git@gitlab.local:proj1/bar 17 | - name: upstream 18 | url: git@github.com:user/bar 19 | dest: bar 20 | branch: master 21 | sha1: ad2b68539c78e749a372414165acdf2a1bb68203 22 | 23 | - url: git@gitlab.local:proj1/app 24 | dest: app 25 | tag: v0.1 26 | copy: 27 | - file: top.cmake 28 | dest: CMakeLists.txt 29 | - file: .clangformat 30 | symlink: 31 | - source: app/some_file 32 | target: ../foo/some_file 33 | ``` 34 | 35 | In this example: 36 | 37 | * First, `proj1/foo` will be cloned into `/foo` using the `next` branch. 38 | * Then, `proj1/bar` will be cloned into `/bar` using the `master` branch, and reset to `ad2b68539c78e749a372414165acdf2a1bb68203`. 39 | * Finally: 40 | * `proj1/app` will be cloned into `/app` using the `v0.1` tag, 41 | * `top.cmake` will be copied from `proj1/app/top.cmake` to `/CMakeLists.txt`, 42 | * `.clang-format` will be copied from `proj1/app/` to `/`, and 43 | * a symlink will be created from `/app/some_file` to `/foo/some_file`. 44 | 45 | 46 | 47 | ## Top fields 48 | 49 | 50 | * `repos` (required): list of repositories to clone 51 | * `groups` (optional): list of groups 52 | 53 | ## repos 54 | 55 | Each repository is also a *mapping*, containing: 56 | 57 | * Either: 58 | * `url` if you just need one remote named `origin` 59 | * A list of remotes with a `name` and `url`. In that case, the first remote 60 | will be used for cloning the repository. 61 | * `dest` (required): relative path of the repository in the workspace 62 | * `branch` (optional): The branch to use when cloning the repository (defaults 63 | to `master`) 64 | * `tag` (optional): 65 | * When running `tsrc init`: Project will be cloned at the provided tag. 66 | * When running `tsrc sync`: If the project is clean, project will be reset 67 | to the given tag, else a warning message will be printed. 68 | * `sha1` (optional): 69 | * When running `tsrc init`: Project will be cloned, and then reset to the given sha1. 70 | * When running `tsrc sync`: If the project is clean, project will be reset 71 | to the given sha1, else a warning message will be printed. 72 | * `ignore_submodules` (optional, default=`false`): 73 | * When running `tsrc init`: if `ignore_submodules` is `true`, do not recursively clone submodules. 74 | * When running `tsrc sync`: if `ignore_submodules` is `true`, do not initialize or update submodules. 75 | to the given sha1, else a warning message will be printed. 76 | * `copy` (optional): A list of mappings with `file` and `dest` keys. 77 | * `symlink` (optional): A list of mappings with `source` and `target` keys. 78 | 79 | 80 | See the [Using fixed references](../guide/fixed-refs.md) and the [Performing file system operations](../guide/fs.md) guides for details about how and why you would use the `tag`, `sha1`, `copy` or `symlink` fields. 81 | 82 | ## groups 83 | 84 | The `groups` section lists the groups by name. Each group should have a `repos` field 85 | containing a list of repositories (only repositories defined in the `repos` section are allowed). 86 | 87 | The groups can optionally include other groups, with a `includes` field which should be 88 | a list of existing group names. 89 | 90 | The group named `default`, if it exists, will be used to know which repositories to clone 91 | when using `tsrc init` and the `--group` command line argument is not used. 92 | 93 | Example: 94 | 95 | ```yaml 96 | repos: 97 | - dest: a 98 | url: .. 99 | - dest: b 100 | url: .. 101 | - dest: bar 102 | url: .. 103 | - dest: baz 104 | url: .. 105 | 106 | groups: 107 | default: 108 | repos: [a, b] 109 | foo: 110 | repos: [bar, baz] 111 | includes: [default] 112 | ``` 113 | 114 | ```console 115 | $ tsrc init 116 | # Clones a, b 117 | $ tsrc init --group foo 118 | # Clones a, b, bar and baz 119 | ``` 120 | 121 | Note that `tsrc init` records the names of the groups it was invoked 122 | with, so that `tsrc sync` re-uses them later on. This means that if you 123 | want to change the groups used, you must re-run `tsrc init` with the new 124 | group list. 125 | 126 | !!! note 127 | More information about how to use groups is available in the [relevant guide](../guide/groups.md). 128 | 129 | -------------------------------------------------------------------------------- /tsrc/local_tmp_bare_repos.py: -------------------------------------------------------------------------------- 1 | """ 2 | # local_tmp_bare_repos 3 | 4 | ## conditions for use 5 | 6 | This is only usefull when these 3 conditions are met: 7 | 8 | 1st: Reasonable use is only for: 9 | * Deep Manifest 10 | * Future Manifest 11 | 12 | 2nd: SHA1 must be set for such Repo 13 | 14 | 3rd: remote must be set and reachable 15 | 16 | ## What it does: 17 | 18 | Bare Git repository should be created 19 | to some temporary location (under '.tsrc') 20 | or updated to required commit SHA1. 21 | 22 | current SHA1 is then checked with remote to 23 | count possition ahead/behind. 24 | """ 25 | 26 | import hashlib 27 | from pathlib import Path 28 | from typing import List, Optional 29 | 30 | from tsrc.cloner import BareCloner 31 | 32 | # import cli_ui as ui 33 | from tsrc.errors import LoadManifestSchemaError 34 | from tsrc.executor import process_items 35 | from tsrc.groups_to_find import GroupsToFind 36 | from tsrc.local_manifest import LocalManifest 37 | from tsrc.manifest_common import ManifestGetRepos 38 | from tsrc.manifest_common_data import ManifestsTypeOfData 39 | from tsrc.pcs_repo import PCSRepo 40 | from tsrc.repo import Repo 41 | from tsrc.utils import erase_last_line 42 | from tsrc.workspace import Workspace 43 | 44 | 45 | def prepare_tmp_bare_dm_repos( 46 | workspace: Workspace, 47 | dm: PCSRepo, 48 | gtf: GroupsToFind, 49 | num_jobs: int, 50 | ) -> List[Repo]: 51 | """ 52 | Take care of Deep Manifest's Repos to repsect Groups 53 | """ 54 | 55 | # get Repos from Deep Manifest (considering Groups) 56 | dm_path = workspace.root_path / dm.dest 57 | ldm = LocalManifest(dm_path) 58 | try: 59 | ldmm = ldm.get_manifest_safe_mode(ManifestsTypeOfData.DEEP) 60 | # except LoadManifestSchemaError as lmse: 61 | except LoadManifestSchemaError: 62 | # ui.warning(lmse) 63 | return [] 64 | 65 | # get repos that match Groups provided 66 | mgr = ManifestGetRepos(workspace, ldmm, True, workspace.config.clone_all_repos) 67 | repos, _, gtf = mgr.by_groups(gtf, must_find_all_groups=False) 68 | 69 | c_repos: List[Repo] = ready_tmp_bare_repos( 70 | workspace, ManifestsTypeOfData.DEEP, repos 71 | ) 72 | 73 | return c_repos 74 | 75 | 76 | def process_bare_repos( 77 | workspace: Workspace, c_repos: List[Repo], num_jobs: int 78 | ) -> List[Repo]: 79 | 80 | # TODO: possibly add 'config' -> 'remote_name=self.config.singular_remote' 81 | bare_cloner = BareCloner(workspace.root_path) 82 | 83 | process_items(c_repos, bare_cloner, num_jobs=num_jobs) 84 | erase_last_line() 85 | 86 | return c_repos 87 | 88 | 89 | def ready_tmp_bare_repos( 90 | workspace: Workspace, mtod: ManifestsTypeOfData, repos: List[Repo] 91 | ) -> List[Repo]: 92 | 93 | # create parent directory if it does not exists 94 | if repos: 95 | mtod_path: str 96 | if mtod == ManifestsTypeOfData.DEEP: 97 | mtod_path = ".tmp_dm_repos" 98 | elif mtod == ManifestsTypeOfData.FUTURE: 99 | mtod_path = ".tmp_fm_repos" 100 | else: 101 | return [] # do not continue 102 | 103 | tmp_dir = workspace.root_path / ".tsrc" / mtod_path 104 | if not tmp_dir.is_dir(): 105 | tmp_dir.mkdir(parents=True, exist_ok=True) 106 | 107 | # create Repo's directory if it does not exists when there is SHA1 108 | c_repos: List[Repo] = [] # consider repos (to get the possition) 109 | for repo in repos: 110 | if repo.sha1 and repo.remotes: 111 | h = hashlib.sha1(repo.clone_url.encode()) 112 | # create dirname with hash of URL 113 | this_dir = h.hexdigest()[:13] + "_" + repo.dest 114 | repo_dir = tmp_dir / this_dir 115 | 116 | # check if there is Repo in Workspace 117 | possible_w_path = workspace.root_path / repo.dest 118 | possible_w_path_git = possible_w_path / ".git" 119 | bare_clone_path: Optional[Path] = None 120 | if possible_w_path_git.is_dir(): 121 | bare_clone_path = possible_w_path 122 | 123 | # add Repo that can be processed by 'process_items' 124 | c_repos.append( 125 | Repo( 126 | dest=str(repo_dir), 127 | remotes=repo.remotes, 128 | branch=repo.branch, 129 | keep_branch=repo.keep_branch, 130 | is_default_branch=repo.is_default_branch, 131 | orig_branch=repo.orig_branch, 132 | sha1=repo.sha1, 133 | tag=repo.tag, 134 | shallow=False, 135 | is_bare=True, 136 | _bare_clone_path=bare_clone_path, 137 | _bare_clone_mtod=mtod, 138 | _bare_clone_orig_dest=repo.dest, 139 | _bare_clone_is_ok=repo._bare_clone_is_ok, 140 | ) 141 | ) 142 | 143 | return c_repos 144 | -------------------------------------------------------------------------------- /tsrc/dump_manifest_args_data.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field, fields 2 | from enum import Enum, Flag, unique 3 | from pathlib import Path 4 | from typing import Any, List, Optional, Union 5 | 6 | from tsrc.workspace import Workspace 7 | 8 | 9 | @unique 10 | class SourceModeEnum(Enum): 11 | NONE = 0 12 | RAW_DUMP = 1 13 | WORKSPACE_DUMP = 2 14 | YAML_FILE = 3 # not implemented 15 | 16 | 17 | @unique 18 | class UpdateSourceEnum(Enum): 19 | NONE = 0 20 | FILE = 1 21 | DEEP_MANIFEST = 2 22 | 23 | 24 | @dataclass 25 | class ManifestDataOptions: 26 | sha1_on: bool = False 27 | sha1_off: bool = False 28 | skip_manifest: bool = False 29 | only_manifest: bool = False 30 | ignore_groups: bool = False # not implemented 31 | 32 | def clean(self) -> None: 33 | for i in dir(self): 34 | if isinstance(getattr(self, i), bool) is True: 35 | setattr(self, i, False) 36 | 37 | 38 | class FinalOutputModeFlag(Flag): 39 | NONE = 0 40 | PREVIEW = 1 41 | NEW = 2 # use: 'destination_path' 42 | # if there is no NEW, that means we are using same file for 43 | # output as we are using for update. thus there cannot be a OVERWRITE 44 | UPDATE = 4 # use: 'update_path' 45 | OVERWRITE = 8 # must use force 46 | 47 | 48 | @dataclass 49 | class FinalOutputAllPaths: 50 | """ 51 | There may not be clear what output path will be used in the end, 52 | therefore we should keep them all and take one at the very end 53 | """ 54 | 55 | default_path: Path = Path("manifest.yml") # use when there is no other 56 | update_on_path: Optional[Path] = None # when using update|update_on 57 | save_to_path: Optional[Path] = None # whenever there should be a new file 58 | common_path: Optional[Path] = None # when on RAW mode, this gets calculated 59 | 60 | def __init__(self, **kwargs: Any) -> None: 61 | # only set those that are present 62 | names = {f.name for f in fields(self)} 63 | for key, value in kwargs.items(): 64 | if key in names: 65 | setattr(self, key, value) 66 | 67 | def clean_all_paths(self) -> None: 68 | for i in dir(self): 69 | if isinstance(getattr(self, i), Path) and i != "default_path": 70 | setattr(self, i, None) 71 | 72 | 73 | @dataclass 74 | class DumpManifestOperationDetails: 75 | """ 76 | Contains all data that should be used 77 | initially for 'dump-manifest' command 78 | 79 | There are cases however that will require 80 | to further checks during execution time 81 | (for example when COMMON PATH will be in place) 82 | """ 83 | 84 | # SOURCE of data (must be set) 85 | source_mode = SourceModeEnum.NONE 86 | source_path: Optional[Path] = None 87 | 88 | # UPDATE source (optional) 89 | update_source = UpdateSourceEnum.NONE 90 | update_source_path: Optional[Path] = None 91 | 92 | # OPTIONS applied when processing Manifest (optional to change, using defaults) 93 | manifest_data_options = ManifestDataOptions() 94 | 95 | # FINAL OUTPUT MODE (must be determined) 96 | final_output_mode: List[FinalOutputModeFlag] = field(default_factory=list) 97 | final_output_path_list = FinalOutputAllPaths() 98 | 99 | # helpers to be used later 100 | workspace: Optional[Workspace] = None 101 | 102 | def __init__(self, **kwargs: Any) -> None: 103 | # fix missing value 104 | if "final_output_mode" not in fields(self): 105 | self.final_output_mode: List[FinalOutputModeFlag] = [] 106 | 107 | # only set those that are present 108 | names = {f.name for f in fields(self)} 109 | for key, value in kwargs.items(): 110 | if key in names: 111 | setattr(self, key, value) 112 | 113 | def clean(self) -> None: 114 | self.final_output_path_list.clean_all_paths() 115 | self.final_output_mode = [] 116 | self.manifest_data_options.clean() 117 | 118 | # ---------- 119 | # 'get_path' - section 120 | # ---------- 121 | # it is used as for 'final_output_path' only 122 | 123 | def get_path_for_new(self) -> Union[Path, None]: 124 | if self.final_output_path_list.save_to_path: 125 | return self.final_output_path_list.save_to_path 126 | if self.final_output_path_list.common_path: 127 | return self.final_output_path_list.common_path 128 | if self.final_output_path_list.default_path: 129 | return self.final_output_path_list.default_path 130 | 131 | return None 132 | 133 | def get_path_for_update(self) -> Union[Path, None]: 134 | if self.final_output_path_list.save_to_path: 135 | return self.final_output_path_list.save_to_path 136 | if self.final_output_path_list.update_on_path: 137 | return self.final_output_path_list.update_on_path 138 | if self.final_output_path_list.common_path: 139 | return self.final_output_path_list.common_path 140 | 141 | return None 142 | -------------------------------------------------------------------------------- /docs/guide/foreach.md: -------------------------------------------------------------------------------- 1 | # Running a command for each repo in the workspace 2 | 3 | `tsrc` comes with a `foreach` command that allows you to run 4 | the same command for each repo in the workspace. 5 | 6 | This can be used for several things. For instance, if you are building 7 | an artifact from a group of repositories, you may want to put a tag on 8 | each repo that was used to produce it: 9 | 10 | ```bash 11 | $ tsrc foreach git tag v1.2 12 | ``` 13 | 14 | ```text 15 | :: Using workspace in /path/to/work 16 | :: Running `git tag v1.1` on 2 repos 17 | /path/to/work/foo $ git tag v1.2 18 | /path/to/work/bar $ git tag v1.2 19 | /path/to/work/baz $ git tag v1.2 20 | OK ✓ 21 | ``` 22 | 23 | ## Caveats 24 | 25 | * If the command you want to run contains arguments starting with `-`: you need 26 | to call `foreach` like this: 27 | 28 | ``` 29 | $ tsrc foreach -- some-command --with-option 30 | ``` 31 | 32 | * By default, the command is passed "as is", without starting a shell. If you want 33 | to use a shell, use the `-c` option: 34 | 35 | ``` 36 | $ tsrc foreach -c 'echo $PWD' 37 | ``` 38 | 39 | Note that we need single quotes here to prevent the shell from expanding 40 | the `PWD` environment variable when `tsrc` is run. 41 | 42 | ## Using repo and manifest data 43 | 44 | The current `tsrc` implementation may not contain all the features your organization needs. 45 | 46 | The good news is that you can extend `tsrc`'s feature set by using `tsrc 47 | foreach`. 48 | 49 | Let's take an example, where you have a manifest containing `foo` and `bar` and both 50 | repos are configured to use a `master` branch. 51 | 52 | Here's what happens if you run `tsrc sync` with `bar` on the correct branch (`master`), and `foo` on an incorrect branch (`devel`): 53 | 54 | ```bash 55 | $ tsrc sync 56 | ``` 57 | 58 | ```text 59 | :: Using workspace in /path/to/work 60 | => Updating manifest 61 | ... 62 | => Cloning missing repos 63 | => Configuring remotes 64 | => Synchronizing repos 65 | * (1/2) Synchronizing foo 66 | * Fetching origin 67 | * Updating branch: devel 68 | Updating 702f428..2e4fb45 69 | Fast-forward 70 | ... 71 | * (2/2) Synchronizing bar 72 | * Fetching origin 73 | * Updating branch: master 74 | Already up to date. 75 | Error: Failed to synchronize the following repos: 76 | * foo : Current branch: 'devel' does not match expected branch: 'master' 77 | ``` 78 | 79 | If this happens with multiple repos, you may want a command to checkout the correct branch automatically. 80 | 81 | Here's one way to do it: 82 | 83 | ```bash 84 | $ tsrc foreach -c 'git checkout $TSRC_PROJECT_MANIFEST_BRANCH' 85 | ``` 86 | 87 | Here we take advantage of the fact that `tsrc` sets the `TSRC_PROJECT_MANIFEST_BRANCH` 88 | environment variable correctly for each repository before running the command. 89 | 90 | Here's the whole list: 91 | 92 | 93 | | Variable | Description | 94 | |----------------------------------|--------------------------------------------------------| 95 | | `TSRC_WORKSPACE_PATH` | Full path of the workspace root | 96 | | `TSRC_MANIFEST_BRANCH` | Branch of the manifest | 97 | | `TSRC_MANIFEST_URL` | URL of the manifest | 98 | | `TSRC_PROJECT_CLONE_URL` | URL used to clone the repo | 99 | | `TSRC_PROJECT_DEST` | Relative path of the repo in the workspace | 100 | | `TSRC_PROJECT_MANIFEST_BRANCH` | Branch configured in the manifest for this repo | 101 | | `TSRC_PROJECT_REMOTE_` | URL of the remote named 'NAME' | 102 | | `TSRC_PROJECT_STATUS_DIRTY` | Set to `true` if the project is dirty, otherwise unset | 103 | | `TSRC_PROJECT_STATUS_AHEAD` | Number of commits ahead of the remote ref | 104 | | `TSRC_PROJECT_STATUS_BEHIND` | Number of commits behind the remote ref | 105 | | `TSRC_PROJECT_STATUS_BRANCH` | Current branch of the repo | 106 | | `TSRC_PROJECT_STATUS_SHA1` | SHA1 of the current branch | 107 | | `TSRC_PROJECT_STATUS_STAGED` | Number of files that are staged but not committed | 108 | | `TSRC_PROJECT_STATUS_NOT_STAGED` | Number of files that are changed but not staged | 109 | | `TSRC_PROJECT_STATUS_UNTRACKED` | Number of files that are untracked | 110 | 111 | You can implement more complex behavior using the environment variables above, for instance: 112 | 113 | ```sh 114 | #!/bin/bash 115 | # in switch-and-pull 116 | if [[ "${TSRC_PROJECT_STATUS_DIRTY}" = "true" ]]; then 117 | echo Error: project is dirty 118 | exit 1 119 | fi 120 | 121 | git switch $TSRC_PROJECT_MANIFEST_BRANCH 122 | git pull 123 | ``` 124 | 125 | ```text 126 | $ tsrc foreach switch-and-pull 127 | :: Running `switch-and-pull` on 2 repos 128 | * (1/2) foo 129 | /path/to/foo $ switch-and-pull 130 | Switched to branch 'master' 131 | Your branch is behind 'origin/master' by 1 commit, and can be fast-forwarded. 132 | (use "git pull" to update your local branch) 133 | Updating 9e7a8e4..5f9bbd4 134 | Fast-forward 135 | * (2/2) bar 136 | /path/to/bar $ switch-and-pull 137 | Error: project is dirty 138 | Error: Command failed for 1 repo(s) 139 | * bar 140 | ``` 141 | 142 | Of course, feel free to use your favorite programming language here :) 143 | -------------------------------------------------------------------------------- /tsrc/groups.py: -------------------------------------------------------------------------------- 1 | """ Support for groups of elements """ 2 | 3 | # Note that groups are allowed to include other groups. 4 | 5 | from typing import Any, Dict, Generic, List, Optional, TypeVar 6 | 7 | import cli_ui as ui 8 | 9 | from tsrc.errors import Error 10 | from tsrc.manifest_common_data import ManifestsTypeOfData, get_mtod_str 11 | 12 | T = TypeVar("T") 13 | 14 | 15 | class GroupError(Error): 16 | pass 17 | 18 | 19 | class Group(Generic[T]): 20 | def __init__( 21 | self, name: str, elements: List[T], includes: Optional[List[str]] = None 22 | ) -> None: 23 | self.name = name 24 | self.elements = elements 25 | self.includes = includes or [] 26 | 27 | 28 | class GroupNotFound(GroupError): 29 | def __init__( 30 | self, group_name: str, parent_group: Optional[Group[Any]] = None 31 | ) -> None: 32 | self.group_name = group_name 33 | self.parent_group = parent_group 34 | if self.parent_group: 35 | message = f"Invalid include detected for '{self.parent_group.name}':\n" 36 | else: 37 | message = "" 38 | message += f"No such group: '{self.group_name}'" 39 | super().__init__(message) 40 | 41 | 42 | class UnknownGroupElement(GroupError): 43 | def __init__(self, group_name: str, element: T) -> None: 44 | self.group_name = group_name 45 | self.element = element 46 | message = f"group '{group_name}': unknown element: '{element}'" 47 | super().__init__(message) 48 | 49 | 50 | class GroupList(Generic[T]): 51 | """Usage: 52 | 53 | >>> group_list = GroupList() 54 | >>> group_list.add("group1", ["foo", "bar"]) 55 | >>> group_list.add("group2", ["spam"], includes=["group"]) 56 | >>> elements = group_list.get_elements(groups=["group2"]) 57 | ["spam", "foo", "bar"] 58 | 59 | """ 60 | 61 | def __init__(self, *, elements: List[T]) -> None: 62 | self.groups: Dict[str, Group[T]] = {} 63 | self.all_elements = elements 64 | self._groups_seen: List[str] = [] 65 | self.missing_elements: List[Dict[str, T]] = [] 66 | 67 | def get_groups_seen(self) -> List[str]: 68 | return self._groups_seen 69 | 70 | def add( 71 | self, 72 | name: str, 73 | elements: List[T], 74 | includes: Optional[List[str]] = None, 75 | ignore_on_mtod: Optional[ManifestsTypeOfData] = None, 76 | ) -> None: 77 | can_add: bool = True 78 | ignored_elements: List[T] = [] 79 | for element in elements: 80 | if element not in self.all_elements: 81 | if ignore_on_mtod: 82 | can_add = False 83 | if ignore_on_mtod != ManifestsTypeOfData.DEEP_ON_UPDATE: 84 | ui.warning( 85 | f"{get_mtod_str(ignore_on_mtod)}: Groups: cannot add '{element}' to '{name}'." # noqa: E501 86 | ) 87 | # store missing element, also keep its asignment to the Group name 88 | self.missing_elements.append({name: element}) 89 | ignored_elements.append(element) 90 | else: 91 | raise UnknownGroupElement(name, element) 92 | if can_add is False: 93 | elements = list(set(elements).difference(ignored_elements)) 94 | self.groups[name] = Group(name, elements, includes=includes) 95 | 96 | def get_group(self, name: str) -> Optional[Group[T]]: 97 | return self.groups.get(name) 98 | 99 | def get_elements( 100 | self, 101 | groups: List[str], 102 | ignore_if_group_not_found: bool = False, 103 | ) -> List[T]: 104 | # Note: to get all elements in a group, recursively parse 105 | # the groups and their includes, while making sure no 106 | # group is processed twice. 107 | # 108 | # This algorithms allows to have groups that include each other 109 | # without creating infinite loops. 110 | self._groups_seen = [] 111 | # Note: we need to keep the result free of duplicates *and* 112 | # in the correct order. 113 | # There's no OrderedSet in the stdlib, so we use a dict instead 114 | # where keys don't matter 115 | res: Dict[T, bool] = {} 116 | self._rec_get_elements( 117 | res, 118 | groups, 119 | parent_group=None, 120 | ignore_if_group_not_found=ignore_if_group_not_found, 121 | ) 122 | return list(res.keys()) 123 | 124 | def _rec_get_elements( 125 | self, 126 | res: Dict[T, bool], 127 | group_names: List[str], 128 | *, 129 | parent_group: Optional[Group[T]], 130 | ignore_if_group_not_found: bool = False, 131 | ) -> None: 132 | for group_name in group_names: 133 | if group_name in self._groups_seen: 134 | return 135 | if group_name not in self.groups: 136 | if ignore_if_group_not_found is True: 137 | continue 138 | raise GroupNotFound(group_name, parent_group=parent_group) 139 | group = self.groups[group_name] 140 | self._groups_seen.append(group.name) 141 | self._rec_get_elements(res, group.includes, parent_group=group) 142 | for element in group.elements: 143 | res[element] = True 144 | -------------------------------------------------------------------------------- /tsrc/test/helpers/test_git_server.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from tsrc.file_system import Copy 4 | from tsrc.git import get_current_branch, run_git, run_git_captured 5 | from tsrc.manifest import Manifest, load_manifest 6 | from tsrc.test.helpers.git_server import GitServer 7 | 8 | 9 | def read_remote_manifest(workspace_path: Path, git_server: GitServer) -> Manifest: 10 | run_git(workspace_path, "clone", git_server.manifest_url) 11 | manifest_yml = workspace_path / "manifest/manifest.yml" 12 | manifest = load_manifest(manifest_yml) 13 | return manifest 14 | 15 | 16 | def test_add_repo_can_clone(workspace_path: Path, git_server: GitServer) -> None: 17 | """Check that repo added to the GitServer can be cloned, 18 | typically, they should be bare but not empty! 19 | 20 | """ 21 | foobar_url = git_server.add_repo("foo/bar") 22 | run_git(workspace_path, "clone", foobar_url) 23 | assert (workspace_path / "bar").exists() 24 | 25 | 26 | def test_can_set_remote_head(tmp_path: Path, git_server: GitServer) -> None: 27 | git_server.add_repo("foo") 28 | git_server.manifest.change_branch("main") 29 | git_server.manifest.set_head("main") 30 | 31 | run_git(tmp_path, "clone", git_server.manifest_url, "manifest") 32 | 33 | assert get_current_branch(tmp_path / "manifest") == "main" 34 | 35 | 36 | def test_can_add_copies(workspace_path: Path, git_server: GitServer) -> None: 37 | git_server.add_repo("foo") 38 | git_server.manifest.set_file_copy("foo", "foo.txt", "top.txt") 39 | manifest = read_remote_manifest(workspace_path, git_server) 40 | assert manifest.file_system_operations == [Copy("foo", "foo.txt", "top.txt")] 41 | 42 | 43 | def test_add_repo_updates_manifest(workspace_path: Path, git_server: GitServer) -> None: 44 | git_server.add_repo("foo/bar") 45 | git_server.add_repo("spam/eggs") 46 | manifest = read_remote_manifest(workspace_path, git_server) 47 | repos = manifest.get_repos() 48 | assert len(repos) == 2 49 | for repo in repos: 50 | clone_url = repo.clone_url 51 | _, out = run_git_captured(workspace_path, "ls-remote", clone_url) 52 | assert "refs/heads/master" in out 53 | 54 | 55 | def test_multiple_manifest_branches( 56 | workspace_path: Path, git_server: GitServer 57 | ) -> None: 58 | git_server.add_repo("foo") 59 | git_server.manifest.change_branch("devel") 60 | git_server.add_repo("bar") 61 | 62 | run_git(workspace_path, "clone", git_server.manifest_url) 63 | manifest_yml = workspace_path / "manifest/manifest.yml" 64 | manifest = load_manifest(manifest_yml) 65 | assert len(manifest.get_repos()) == 1 66 | 67 | run_git(workspace_path / "manifest", "reset", "--hard", "origin/devel") 68 | manifest = load_manifest(manifest_yml) 69 | assert len(manifest.get_repos()) == 2 70 | 71 | 72 | def test_push_to_other_branch(workspace_path: Path, git_server: GitServer) -> None: 73 | foo_url = git_server.add_repo("foo") 74 | git_server.push_file("foo", "devel.txt", contents="this is devel\n", branch="devel") 75 | run_git(workspace_path, "clone", foo_url, "--branch", "devel") 76 | foo_path = workspace_path / "foo" 77 | assert (foo_path / "devel.txt").read_text() == "this is devel\n" 78 | 79 | 80 | def test_tag(workspace_path: Path, git_server: GitServer) -> None: 81 | foo_url = git_server.add_repo("foo") 82 | git_server.tag("foo", "v0.1", branch="master") 83 | _, out = run_git_captured(workspace_path, "ls-remote", foo_url) 84 | assert "refs/tags/v0.1" in out 85 | 86 | 87 | def test_get_sha1(workspace_path: Path, git_server: GitServer) -> None: 88 | git_server.add_repo("foo") 89 | actual = git_server.get_sha1("foo") 90 | assert isinstance(actual, str) 91 | 92 | 93 | def test_default_branch_devel(workspace_path: Path, git_server: GitServer) -> None: 94 | foo_url = git_server.add_repo("foo", default_branch="devel") 95 | run_git(workspace_path, "clone", foo_url) 96 | foo_path = workspace_path / "foo" 97 | cloned_branch = get_current_branch(foo_path) 98 | assert cloned_branch == "devel" 99 | 100 | manifest = read_remote_manifest(workspace_path, git_server) 101 | foo_config = manifest.get_repo("foo") 102 | assert foo_config.branch == "devel" 103 | 104 | 105 | def test_create_submodule(workspace_path: Path, git_server: GitServer) -> None: 106 | top_url = git_server.add_repo("top") 107 | sub_url = git_server.add_repo("sub", add_to_manifest=False) 108 | git_server.add_submodule("top", url=sub_url, path=Path("sub")) 109 | 110 | run_git(workspace_path, "clone", top_url, "--recurse-submodules") 111 | 112 | top_path = workspace_path / "top" 113 | sub_readme = top_path / "sub" / "README" 114 | assert sub_readme.exists() 115 | 116 | 117 | def test_update_submodule(workspace_path: Path, git_server: GitServer) -> None: 118 | top_url = git_server.add_repo("top") 119 | sub_url = git_server.add_repo("sub", add_to_manifest=False) 120 | git_server.add_submodule("top", url=sub_url, path=Path("sub")) 121 | 122 | run_git(workspace_path, "clone", top_url, "--recurse-submodules") 123 | 124 | git_server.push_file("sub", "new.txt") 125 | git_server.update_submodule("top", "sub") 126 | 127 | top_path = workspace_path / "top" 128 | run_git(top_path, "fetch") 129 | run_git(top_path, "reset", "--hard", "origin/master") 130 | run_git(top_path, "submodule", "update", "--init", "--recursive") 131 | 132 | new_sub = top_path / "sub" / "new.txt" 133 | assert new_sub.exists() 134 | -------------------------------------------------------------------------------- /tsrc/test/test_git_status.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | 4 | import cli_ui as ui 5 | import pytest 6 | 7 | from tsrc.git import DOWN, UP, GitStatus 8 | from tsrc.test.helpers.git_server import BareRepo 9 | 10 | 11 | class GitProject: 12 | def __init__(self, path: Path, remote_repo: BareRepo): 13 | self.path = path 14 | # Make sure the initial branch is the same regardless of the user 15 | # git configuration 16 | self.run_git("init", "--initial-branch", "master") 17 | self.remote_repo = remote_repo 18 | self.run_git("remote", "add", "origin", str(remote_repo.path)) 19 | 20 | def get_status(self) -> GitStatus: 21 | status = GitStatus(self.path) 22 | status.update() 23 | return status 24 | 25 | def make_initial_commit(self) -> None: 26 | self.write_file("README", "This is the README") 27 | self.commit_changes("initial commit") 28 | 29 | def commit_changes(self, message: str) -> None: 30 | self.run_git("add", ".") 31 | self.run_git("commit", "-m", message) 32 | 33 | def run_git(self, *cmd: str) -> None: 34 | subprocess.run( 35 | ["git", *cmd], 36 | check=True, 37 | cwd=self.path, 38 | stdout=subprocess.PIPE, 39 | stderr=subprocess.PIPE, 40 | ) 41 | 42 | def write_file(self, name: str, contents: str) -> None: 43 | (self.path / name).write_text(contents) 44 | 45 | 46 | @pytest.fixture 47 | def git_project(tmp_path: Path, remote_repo: BareRepo) -> GitProject: 48 | src_path = tmp_path / "src" 49 | src_path.mkdir(parents=True) 50 | res = GitProject(src_path, remote_repo) 51 | return res 52 | 53 | 54 | @pytest.fixture 55 | def remote_repo(tmp_path: Path) -> BareRepo: 56 | srv_path = tmp_path / "srv" 57 | srv_path.mkdir(parents=True) 58 | return BareRepo.create(srv_path, "master", empty=True) 59 | 60 | 61 | def test_empty(git_project: GitProject) -> None: 62 | status = git_project.get_status() 63 | assert status.empty 64 | 65 | 66 | def test_clean_on_master(git_project: GitProject) -> None: 67 | git_project.make_initial_commit() 68 | actual = git_project.get_status() 69 | assert actual.branch == "master" 70 | 71 | 72 | def test_dirty_on_master(git_project: GitProject) -> None: 73 | git_project.make_initial_commit() 74 | git_project.write_file("new.txt", "new file") 75 | actual = git_project.get_status() 76 | assert actual.dirty 77 | 78 | 79 | def test_behind_1_commit(git_project: GitProject) -> None: 80 | git_project.make_initial_commit() 81 | git_project.run_git("push", "-u", "origin", "master") 82 | 83 | git_project.remote_repo.commit_file( 84 | "new.txt", branch="master", contents="new", message="add new file" 85 | ) 86 | 87 | git_project.run_git("fetch") 88 | 89 | actual = git_project.get_status() 90 | assert actual.ahead == 0 91 | assert actual.behind == 1 92 | 93 | 94 | def test_ahead_2_commits(git_project: GitProject) -> None: 95 | git_project.make_initial_commit() 96 | git_project.run_git("push", "-u", "origin", "master") 97 | 98 | git_project.write_file("first", "This is the first new file") 99 | git_project.commit_changes("first new file") 100 | 101 | git_project.write_file("second", "This is the second new file") 102 | git_project.commit_changes("second new file") 103 | 104 | actual = git_project.get_status() 105 | assert actual.ahead == 2 106 | assert actual.behind == 0 107 | 108 | 109 | def test_on_sha1(git_project: GitProject) -> None: 110 | git_project.make_initial_commit() 111 | 112 | git_project.write_file("first", "This is the first new file") 113 | git_project.commit_changes("first new file") 114 | 115 | git_project.run_git("checkout", "HEAD~1") 116 | 117 | actual = git_project.get_status() 118 | assert actual.branch is None 119 | assert actual.sha1 is not None 120 | 121 | 122 | def test_on_tag(git_project: GitProject) -> None: 123 | git_project.make_initial_commit() 124 | 125 | git_project.run_git("tag", "v0.1") 126 | 127 | actual = git_project.get_status() 128 | assert actual.tag == "v0.1" 129 | 130 | 131 | class TestDescribe: 132 | dummy_path = Path("src") 133 | 134 | def test_up_to_date(self) -> None: 135 | status = GitStatus(self.dummy_path) 136 | status.branch = "master" 137 | assert status.describe() == [ui.green, "master", ui.reset] 138 | 139 | def test_ahead_1_commit(self) -> None: 140 | status = GitStatus(self.dummy_path) 141 | status.branch = "master" 142 | status.ahead = 1 143 | # fmt: off 144 | assert status.describe() == [ 145 | ui.green, "master", ui.reset, 146 | ui.blue, f"{UP}1 commit", ui.reset 147 | ] 148 | # fmt: on 149 | 150 | def test_diverged(self) -> None: 151 | status = GitStatus(self.dummy_path) 152 | status.branch = "master" 153 | status.ahead = 1 154 | status.behind = 2 155 | # fmt: off 156 | assert status.describe() == [ 157 | ui.green, "master", ui.reset, 158 | ui.blue, f"{UP}1 commit", ui.reset, 159 | ui.blue, f"{DOWN}2 commits", ui.reset, 160 | ] 161 | # fmt: on 162 | 163 | def test_on_sha1(self) -> None: 164 | status = GitStatus(self.dummy_path) 165 | status.sha1 = "b6cfd80" 166 | assert status.describe() == [ui.red, "b6cfd80", ui.reset] 167 | 168 | def test_on_tag(self) -> None: 169 | status = GitStatus(self.dummy_path) 170 | status.tag = "v0.1" 171 | assert status.describe() == [ui.yellow, "on", "v0.1", ui.reset] 172 | -------------------------------------------------------------------------------- /tsrc/git_remote.py: -------------------------------------------------------------------------------- 1 | """ 2 | Git Remote 3 | 4 | Collection of simple GIT remote tools 5 | that can be called when additional checks 6 | needs to be performed on special occasions. 7 | 8 | This is pariculary useful for Manifest-related 9 | checks. 10 | """ 11 | 12 | from pathlib import Path 13 | from sys import platform 14 | from typing import List, Tuple, Union 15 | from urllib.parse import quote, urlparse 16 | 17 | from tsrc.git import run_git_captured 18 | from tsrc.repo import Remote 19 | 20 | 21 | class GitRemote: 22 | def __init__(self, working_path: Path, cur_branch: Union[str, None]) -> None: 23 | self.working_path = working_path 24 | self.branch = cur_branch 25 | self.remotes: List[Remote] = [] 26 | self.upstreamed = False # only related to current branch 27 | 28 | def update(self) -> None: 29 | self.update_remotes() 30 | if self.remotes and self.branch: 31 | self.update_upstreamed() 32 | 33 | def update_remotes(self) -> None: 34 | # obtain information about configured 'remotes' 35 | # in 'GitStatus' obtaining such information 36 | # is not useful as remotes are stored in Manifest 37 | _, out = run_git_captured(self.working_path, "remote") 38 | for line in out.splitlines(): 39 | _, url = run_git_captured(self.working_path, "remote", "get-url", line) 40 | if line and url: 41 | tmp_r = Remote(name=line, url=url) 42 | self.remotes.append(tmp_r) 43 | 44 | def update_upstreamed(self) -> None: 45 | use_branch = self.branch 46 | # if there is tag with same name as branch, it gets refered by 'heads/' 47 | if use_branch: 48 | if use_branch.startswith("heads/") is True: 49 | use_branch = use_branch[6:] 50 | else: 51 | # skip check if upstreamed when there is no branch 52 | return 53 | 54 | rc, _ = run_git_captured( 55 | self.working_path, 56 | "config", 57 | "--get", 58 | f"branch.{use_branch}.remote", 59 | check=False, 60 | ) 61 | if rc == 0: 62 | self.upstreamed = True 63 | 64 | 65 | def remote_urls_are_same(url_1: str, url_2: str) -> bool: 66 | """ 67 | return True if provided URLs are the same 68 | """ 69 | up_1 = urlparse(url_1) 70 | up_2 = urlparse(url_2) 71 | if up_1.scheme != "file" and up_2.scheme != "file": 72 | return ( 73 | up_1.scheme == up_2.scheme 74 | and up_1.hostname == up_2.hostname # noqa: W503 75 | and up_1.port == up_2.port # noqa: W503 76 | and _norm_path(quote(up_1.path)) # noqa: W503 77 | == _norm_path(quote(up_2.path)) # noqa: W503 78 | ) 79 | else: 80 | if platform.startswith("win"): 81 | return ( 82 | up_1.scheme == up_2.scheme and up_1.netloc == up_2.netloc # noqa: W503 83 | ) 84 | else: 85 | return ( 86 | up_1.scheme == up_2.scheme 87 | and up_1.netloc == up_2.netloc # noqa: W503 88 | and up_1.hostname == up_2.hostname # noqa: W503 89 | and _norm_path(quote(up_1.path)) # noqa: W503 90 | == _norm_path(quote(up_2.path)) # noqa: W503 91 | ) 92 | 93 | 94 | def _norm_path(path: str) -> str: 95 | ret: str = "" 96 | if path[0] == "/": 97 | ret += "/" 98 | u_seg: List[str] = [] 99 | path_split = path.split("/") 100 | for seg in path_split: 101 | if seg != "": 102 | u_seg.append(seg) 103 | ret += "/".join(u_seg) 104 | return ret 105 | 106 | 107 | def remote_branch_exist(url: str, branch: str) -> int: 108 | """ 109 | check if remote 'branch' exists 110 | 'url' of repository needs to be provided 111 | any other return code but '0' 112 | should be considered an error 113 | """ 114 | p = Path(".") 115 | rc, _ = run_git_captured( 116 | p, 117 | "ls-remote", 118 | "--exit-code", 119 | "--heads", 120 | url, 121 | f"refs/heads/{branch}", 122 | check=False, 123 | ) 124 | return rc 125 | 126 | 127 | def get_git_remotes(working_path: Path, cur_branch: str) -> GitRemote: 128 | remotes = GitRemote(working_path, cur_branch) 129 | remotes.update() 130 | return remotes 131 | 132 | 133 | def get_l_and_r_sha1_of_branch( 134 | w_r_path: Path, 135 | dest: str, 136 | branch: str, 137 | ) -> Tuple[Union[str, None], Union[str, None]]: 138 | """obtain local and remote SHA1 of given branch. 139 | This is useful when we need to check if we are exactly 140 | updated with remote down to the commit""" 141 | rc, l_b_sha = run_git_captured( 142 | w_r_path / dest, 143 | "rev-parse", 144 | "--verify", 145 | "HEAD", 146 | check=False, 147 | ) 148 | if rc != 0: 149 | return None, None 150 | 151 | _, l_ref = run_git_captured(w_r_path / dest, "symbolic-ref", "-q", "HEAD") 152 | _, r_ref = run_git_captured( 153 | w_r_path / dest, "for-each-ref", "--format='%(upstream)'", l_ref 154 | ) 155 | r_b_sha = None 156 | if rc == 0: 157 | tmp_r_ref = r_ref.split("/") 158 | this_remote = tmp_r_ref[2] 159 | _, r_b_sha = run_git_captured( 160 | w_r_path / dest, 161 | "ls-remote", 162 | "--exit-code", 163 | "--head", 164 | this_remote, 165 | l_ref, 166 | check=True, 167 | ) 168 | if r_b_sha: 169 | return l_b_sha, r_b_sha.split()[0] 170 | else: 171 | return l_b_sha, None 172 | -------------------------------------------------------------------------------- /tsrc/config_status.py: -------------------------------------------------------------------------------- 1 | """ 2 | Config Status 3 | 4 | Contains all operations and displaying 5 | in regard of config manipulation. 6 | 7 | Some atomic config operation can also be found in: 8 | 'ConfigTools' class 9 | """ 10 | 11 | from typing import List, Optional, Tuple, Union 12 | 13 | import cli_ui as ui 14 | 15 | from tsrc.config_data import ConfigUpdateData, ConfigUpdateType 16 | from tsrc.config_status_rc import ConfigStatusReturnCode 17 | from tsrc.config_tools import ConfigTools 18 | from tsrc.errors import Error 19 | from tsrc.status_header_dm import StatusHeaderDisplayMode 20 | from tsrc.workspace import Workspace 21 | 22 | 23 | class ConfigStatus: 24 | def __init__( 25 | self, 26 | workspace: Workspace, 27 | shdms: List[StatusHeaderDisplayMode], 28 | ) -> None: 29 | self.workspace = workspace 30 | self.shdms = shdms 31 | """self.shdms: if related markers will not be here, 32 | nothing will be displayed even when there is some issue""" 33 | 34 | def pre_check_change( 35 | self, 36 | cfgud: ConfigUpdateData, 37 | cfguts: List[ConfigUpdateType], 38 | ) -> Tuple[List[ConfigStatusReturnCode], List[ConfigUpdateType], bool]: 39 | cfgrcs: List[ConfigStatusReturnCode] = [] 40 | config_tools = ConfigTools(self.workspace) 41 | found_some: bool = False # set to True when there is any chage at all 42 | for i, this_type in enumerate(cfguts): 43 | if this_type == ConfigUpdateType.MANIFEST_BRANCH and cfgud.manifest_branch: 44 | rc = config_tools.update_manifest_branch(cfgud.manifest_branch) 45 | if rc == ConfigStatusReturnCode.SUCCESS: 46 | found_some = True # marker 47 | else: 48 | if StatusHeaderDisplayMode.BRANCH in self.shdms: 49 | # display issue via 'ui' 50 | self._manifest_branch_report_issue(rc, cfgud.manifest_branch) 51 | 52 | no_further_display: bool = False 53 | 54 | if rc == ConfigStatusReturnCode.REVERT: 55 | # in this case, reverting is valid response, thus: SUCCESS 56 | rc = ConfigStatusReturnCode.SUCCESS 57 | found_some = True # marker 58 | no_further_display = True 59 | 60 | if rc == ConfigStatusReturnCode.CANCEL: 61 | # basicaly mark as not to update 62 | cfguts[i] = ConfigUpdateType.NONE 63 | no_further_display = True 64 | 65 | if rc == ConfigStatusReturnCode.NOT_FOUND: 66 | # basicaly mark as not to update 67 | cfguts[i] = ConfigUpdateType.NONE 68 | 69 | # skip further display on some cases 70 | if no_further_display is True: 71 | self._no_further_display(StatusHeaderDisplayMode.BRANCH) 72 | 73 | # finaly add 'rc' 74 | cfgrcs.append(rc) 75 | return cfgrcs, cfguts, found_some 76 | 77 | def _no_further_display( 78 | self, 79 | mode: StatusHeaderDisplayMode, 80 | ) -> None: 81 | """replace given Status Header Display Mode 82 | by 'NONE', so it will not gets displayed""" 83 | if mode in self.shdms: 84 | for index, shdm in enumerate(self.shdms): 85 | if shdm == mode: 86 | self.shdms[index] = StatusHeaderDisplayMode.NONE 87 | 88 | def proceed_to_change( 89 | self, 90 | cfgud: ConfigUpdateData, 91 | cfguts: List[ConfigUpdateType], 92 | ) -> None: 93 | config_tools = ConfigTools(self.workspace) 94 | config_tools.commit_config_update(cfgud, cfguts) 95 | 96 | """display only part""" 97 | 98 | def _manifest_branch_report_issue( 99 | self, rc: ConfigStatusReturnCode, branch: Optional[str] = None 100 | ) -> None: 101 | """report only issue, success will be reported elsewhere, 102 | everything else will be taken care of elsewhere""" 103 | if rc == ConfigStatusReturnCode.NOT_FOUND: 104 | ui.info_2( 105 | "Such Manifest's branch:", 106 | ui.green, 107 | branch, 108 | ui.reset, 109 | "was not found on remote,", 110 | ui.red, 111 | "ignoring", 112 | ui.reset, 113 | ) 114 | raise Error("aborting Manifest branch change") 115 | if rc == ConfigStatusReturnCode.CANCEL: 116 | branch_0 = self.workspace.config.manifest_branch_0 117 | if branch == branch_0: 118 | ui.info_2( 119 | "No change to Manifest's branch, it will still stays on:", 120 | ui.green, 121 | branch, 122 | ui.reset, 123 | ) 124 | else: 125 | ui.info_2( 126 | "No update, Manifest's branch will still change from:", 127 | ui.green, 128 | branch_0, 129 | ui.reset, 130 | "~~>", 131 | ui.green, 132 | branch, 133 | ui.reset, 134 | ) 135 | if rc == ConfigStatusReturnCode.REVERT: 136 | ui.info_2( 137 | "Reverting previous update, Manifest's branch will stays on:", 138 | ui.green, 139 | branch, 140 | ui.reset, 141 | ) 142 | 143 | def manifest_branch_change(self, branch: str, branch_0: Union[str, None]) -> None: 144 | """report successful change""" 145 | if branch_0: 146 | ui.info_2( 147 | "Accepting Manifest's branch change from:", 148 | ui.green, 149 | branch_0, 150 | ui.reset, 151 | "~~>", 152 | ui.green, 153 | branch, 154 | ui.reset, 155 | ) 156 | -------------------------------------------------------------------------------- /tsrc/dump_manifest_args_update_source.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import cli_ui as ui 5 | 6 | from tsrc.dump_manifest_args_data import ( 7 | DumpManifestOperationDetails, 8 | FinalOutputModeFlag, 9 | UpdateSourceEnum, 10 | ) 11 | from tsrc.dump_manifest_args_source_mode import SourceModeEnum 12 | from tsrc.git import GitStatus, run_git_captured 13 | from tsrc.groups_to_find import GroupsToFind 14 | from tsrc.pcs_repo import get_deep_manifest_from_local_manifest_pcsrepo 15 | 16 | 17 | class UpdateSource: 18 | 19 | def __init__( 20 | self, args: argparse.Namespace, dmod: DumpManifestOperationDetails 21 | ) -> None: 22 | self.args = args 23 | self.dmod = dmod 24 | 25 | def get_update_source_and_path(self) -> DumpManifestOperationDetails: 26 | 27 | self._possible_mismatch_on_dump_path() 28 | 29 | self._allow_only_1_update_on_a_time() 30 | 31 | # decide on mode 32 | if self.args.do_update is True: 33 | # obtaining data - part 34 | if self.dmod.source_mode == SourceModeEnum.WORKSPACE_DUMP or ( 35 | self.dmod.source_mode == SourceModeEnum.RAW_DUMP 36 | ): 37 | self._get_dm_load_path() 38 | elif self.args.update_on: 39 | # test if such file exists 40 | self._update_on_file_must_exist() 41 | 42 | # we have data ready - no operation after this block in here 43 | self.dmod.update_source = UpdateSourceEnum.FILE 44 | self.dmod.update_source_path = self.args.update_on 45 | self.dmod.final_output_path_list.update_on_path = self.args.update_on 46 | self.dmod.final_output_mode.append(FinalOutputModeFlag.UPDATE) 47 | 48 | return self.dmod 49 | 50 | def _possible_mismatch_on_dump_path(self) -> None: 51 | # if we want to update Workspace Manifest with data from RAW dump 52 | if ( 53 | self.args.raw_dump_path 54 | and self.args.do_update is True # noqa: W503 55 | and self.args.just_preview is False # noqa: W503 56 | ): 57 | dump_path = self.args.raw_dump_path 58 | if self.args.raw_dump_path.is_absolute() is False: 59 | dump_path = os.getcwd() / self.args.raw_dump_path 60 | if self.args.workspace_path: 61 | if self.args.workspace_path.is_absolute() is False: 62 | root_path = os.getcwd() / self.args.workspace_path 63 | else: 64 | root_path = self.args.workspace_path 65 | else: 66 | root_path = os.getcwd() 67 | 68 | if os.path.normpath(dump_path) != os.path.normpath(root_path): 69 | if self.args.use_force is False: 70 | raise Exception( 71 | "Please consider again what you are trying to do.\nYou want to update Manifest in the Workspace by RAW dump, yet you want to start dump not from Workspace root.\nThis may lead to strange Manifest.\nIf you are still sure that this is what you want, use '--force'." # noqa: E501 72 | ) 73 | 74 | def _allow_only_1_update_on_a_time(self) -> None: 75 | # just 1 update type at a time 76 | if self.args.do_update is True and self.args.update_on: # noqa: W503 77 | raise Exception("Use only one out of '--update' or '--update-on' at a time") 78 | 79 | def _update_on_file_must_exist(self) -> None: 80 | if self.args.update_on: 81 | # check if provided file actually exists 82 | if self.args.update_on.is_file() is False: 83 | raise Exception("'UPDATE_AT' file does not exists") 84 | 85 | """ 86 | ===================== 87 | obtaining data - part 88 | ===================== 89 | """ 90 | 91 | def _get_dm_load_path(self) -> None: 92 | # obtains load_path as path of Deep Manifest repository 93 | dm_is_dirty: bool = False 94 | 95 | gtf = GroupsToFind(self.args.groups) 96 | dm = None 97 | if self.dmod.workspace: 98 | dm, _ = get_deep_manifest_from_local_manifest_pcsrepo( 99 | self.dmod.workspace, 100 | gtf, 101 | ) 102 | if dm: 103 | self.dmod.update_source_path = ( 104 | self.dmod.workspace.root_path / dm.dest / "manifest.yml" 105 | ) 106 | # look for git status if it is not dirty 107 | gits = GitStatus(self.dmod.workspace.root_path / dm.dest) 108 | gits.update() 109 | dm_is_dirty = gits.dirty 110 | if dm_is_dirty is True: 111 | # verify if 'manifest.yml' alone is dirty 112 | _, out_stat = run_git_captured( 113 | self.dmod.workspace.root_path / dm.dest, 114 | "status", 115 | "--porcelain=1", 116 | "manifest.yml", 117 | check=False, 118 | ) 119 | if out_stat == "": 120 | # cancel dirty flag if 'manifest.yml' is clean 121 | dm_is_dirty = False 122 | 123 | if ( 124 | dm_is_dirty is True 125 | and self.args.use_force is False # noqa: W503 126 | and not self.args.save_to # noqa: W503 127 | and self.args.just_preview is False # noqa: W503 128 | ): 129 | raise Exception( 130 | "not updating Deep Manifest as it is dirty, use '--force' to overide or '--save-to' somewhere else" # noqa: E501 131 | ) 132 | 133 | if self.dmod.update_source_path: 134 | ui.info_2("Loading Deep Manifest from", self.dmod.update_source_path) 135 | 136 | # save such path as possible output path 137 | self.dmod.final_output_path_list.update_on_path = ( 138 | self.dmod.update_source_path 139 | ) 140 | self.dmod.update_source = UpdateSourceEnum.DEEP_MANIFEST 141 | self.dmod.final_output_mode.append(FinalOutputModeFlag.UPDATE) 142 | 143 | else: 144 | raise Exception("Cannot obtain Deep Manifest from Workspace to update") 145 | -------------------------------------------------------------------------------- /tsrc/dump_manifest_args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from copy import deepcopy 4 | from pathlib import Path 5 | from typing import List 6 | 7 | import cli_ui as ui 8 | 9 | from tsrc.dump_manifest import ManifestDumpersOptions 10 | from tsrc.dump_manifest_args_data import ( 11 | DumpManifestOperationDetails, 12 | FinalOutputModeFlag, 13 | ManifestDataOptions, 14 | ) 15 | from tsrc.dump_manifest_args_final_output import FinalOutput 16 | from tsrc.dump_manifest_args_source_mode import SourceMode, SourceModeEnum 17 | from tsrc.dump_manifest_args_update_source import UpdateSource 18 | from tsrc.groups_and_constraints_data import ( 19 | GroupsAndConstraints, 20 | get_group_and_constraints_data, 21 | ) 22 | 23 | 24 | class DumpManifestArgs: 25 | """ 26 | atempt to separate logic around 'args' and its handling outside of 27 | DumpManifestsCMDLogic 28 | """ 29 | 30 | def __init__(self, args: argparse.Namespace) -> None: 31 | self.args = args 32 | 33 | # from args, get Group and constraints data 34 | self.gac: GroupsAndConstraints = get_group_and_constraints_data(args) 35 | 36 | # some (local) helpers 37 | self.mdo: ManifestDumpersOptions 38 | self.any_update = args.do_update or bool(args.update_on) 39 | if self.any_update is True: 40 | self.mdo = ManifestDumpersOptions(delete_repo=not args.no_repo_delete) 41 | else: 42 | self.mdo = ManifestDumpersOptions() 43 | 44 | # this will be returned when no Exception is hit 45 | self.dmod = DumpManifestOperationDetails() 46 | 47 | # ready source MODE 48 | self.s_m = SourceMode(args, self.dmod) 49 | self.dmod, self.args = self.s_m.get_source_mode_and_path() 50 | 51 | # take care of UPDATE source 52 | self.u_s = UpdateSource(args, self.dmod) 53 | self.dmod = self.u_s.get_update_source_and_path() 54 | 55 | # take care of OPTIONS of Manifest's handling 56 | self.dmod.manifest_data_options = self._get_manifest_data_options() 57 | 58 | # take care of Final Output Mode Flag and All Paths 59 | self.f_o = FinalOutput(args, self.dmod) 60 | self.dmod = self.f_o.get_final_output_modes_and_paths() 61 | 62 | # take care of default situation: get mode and path 63 | self.dmod = self._check_default_mode_and_path() 64 | 65 | # take care of Warning of common purpose 66 | self._take_care_of_common_warnings() 67 | 68 | def _get_manifest_data_options(self) -> ManifestDataOptions: 69 | mdo = ManifestDataOptions() 70 | if self.args.sha1_on is True and self.args.sha1_off is True: 71 | raise Exception("'--sha1-on' and '--sha1-off' are mutually exclusive") 72 | elif self.args.sha1_on is True: 73 | mdo.sha1_on = True 74 | elif self.args.sha1_off is True: 75 | mdo.sha1_off = True 76 | if self.args.skip_manifest is True: 77 | mdo.skip_manifest = True 78 | if self.args.only_manifest is True: 79 | mdo.only_manifest = True 80 | if self.args.skip_manifest is True and self.args.only_manifest is True: 81 | raise Exception( 82 | "'--skip-manifest-repo' and '--only-manifest-repo' are mutually exclusive" 83 | ) 84 | return mdo 85 | 86 | def _check_default_mode_and_path(self) -> DumpManifestOperationDetails: 87 | # use default only if COMMON PATH will not be calculated 88 | if ( 89 | not self.dmod.final_output_mode 90 | and self.dmod.source_mode != SourceModeEnum.RAW_DUMP # noqa: W503 91 | ): 92 | if self.dmod.final_output_path_list.default_path.is_file() is True: 93 | if self.args.use_force is False: 94 | raise Exception( 95 | f"such file '{self.dmod.final_output_path_list.default_path}' already exists, use '--force' to overwrite it" # noqa: E501 96 | ) 97 | else: 98 | 99 | # when there is '--force' allow overwrite of default file 100 | self.dmod.final_output_mode.append(FinalOutputModeFlag.OVERWRITE) 101 | 102 | else: 103 | 104 | # by default, new file will be created 105 | self.dmod.final_output_mode.append(FinalOutputModeFlag.NEW) 106 | 107 | return self.dmod 108 | 109 | def _take_care_of_common_warnings(self) -> None: 110 | if self.args.save_to and self.args.just_preview is True: 111 | ui.warning("'SAVE_TO' path will be ignored when using '--preview'") 112 | 113 | if self.args.save_to and self.args.update_on: 114 | ui.warning("'SAVE_TO' path will be ignored when using '--update-on'") 115 | 116 | if self.any_update is True and self.args.just_preview is True: 117 | ui.warning("When in preview mode, no actual update will be made") 118 | 119 | def consider_common_path( 120 | self, common_path: List[str] 121 | ) -> DumpManifestOperationDetails: 122 | 123 | # verify if it is ok to continue 124 | tmp_save_file = deepcopy(common_path) 125 | tmp_default_file = str(self.dmod.final_output_path_list.default_path).split( 126 | os.sep 127 | ) 128 | tmp_save_file += tmp_default_file 129 | tmp_save_file_path = Path(os.sep.join(tmp_save_file)) 130 | if self.args.raw_dump_path and not self.args.save_to: # noqa: W503 131 | grab_save_path = tmp_save_file_path 132 | if grab_save_path.is_file(): 133 | if FinalOutputModeFlag.PREVIEW in self.dmod.final_output_mode: 134 | return self.dmod 135 | if self.args.use_force is True: 136 | self.dmod.final_output_mode.append(FinalOutputModeFlag.OVERWRITE) 137 | else: 138 | raise Exception( 139 | f"Such file '{grab_save_path}' already exists, use '--force' if you want to overwrite it" # noqa: E501 140 | ) 141 | else: 142 | # create new file only if we are not updating 143 | if FinalOutputModeFlag.UPDATE not in self.dmod.final_output_mode: 144 | self.dmod.final_output_mode.append(FinalOutputModeFlag.NEW) 145 | 146 | # save data 147 | self.dmod.final_output_path_list.common_path = grab_save_path 148 | 149 | return self.dmod 150 | -------------------------------------------------------------------------------- /tsrc/cli/manifest.py: -------------------------------------------------------------------------------- 1 | """ Entry point for `tsrc manifest`. """ 2 | 3 | import argparse 4 | from copy import deepcopy 5 | 6 | from tsrc.cli import ( 7 | add_repos_selection_args, 8 | add_workspace_arg, 9 | get_workspace_with_repos, 10 | simulate_get_workspace_with_repos, 11 | ) 12 | from tsrc.config_data import ConfigUpdateData, ConfigUpdateType 13 | from tsrc.executor import process_items 14 | from tsrc.groups import GroupNotFound 15 | from tsrc.groups_to_find import GroupsToFind 16 | from tsrc.pcs_repo import ( 17 | get_deep_manifest_from_local_manifest_pcsrepo, 18 | get_deep_manifest_pcsrepo, 19 | ) 20 | from tsrc.status_endpoint import StatusCollector 21 | 22 | # from tsrc.status_footer import StatusFooter 23 | from tsrc.status_header import StatusHeader, StatusHeaderDisplayMode 24 | from tsrc.utils import erase_last_line 25 | from tsrc.workspace_repos_summary import WorkspaceReposSummary 26 | 27 | 28 | def configure_parser(subparser: argparse._SubParsersAction) -> None: 29 | parser = subparser.add_parser( 30 | "manifest", description="View and manage top-level Manifest's configuration" 31 | ) 32 | parser.add_argument( 33 | "-b", 34 | "--branch", 35 | help="change Manifest's branch for future sync", 36 | dest="manifest_branch", 37 | ) 38 | add_workspace_arg(parser) 39 | add_repos_selection_args(parser) 40 | # same option as in 'status' 41 | parser.add_argument( 42 | "--no-mm", 43 | action="store_false", 44 | help="do not display Manifest marker", 45 | dest="use_manifest_marker", 46 | ) 47 | parser.add_argument( 48 | "--no-dm", 49 | action="store_false", 50 | help="do not display Deep Manifest", 51 | dest="use_deep_manifest", 52 | ) 53 | parser.add_argument( 54 | "--no-fm", 55 | action="store_false", 56 | help="do not display Future Manifest", 57 | dest="use_future_manifest", 58 | ) 59 | parser.add_argument( 60 | "--same-fm", 61 | action="store_true", 62 | help="use buffered Future Manifest to speed-up execution", 63 | dest="use_same_future_manifest", 64 | ) 65 | parser.add_argument( 66 | "--strict", 67 | action="store_true", 68 | help="do not check for leftover's GIT descriptions", 69 | dest="strict_on_git_desc", 70 | ) 71 | parser.add_argument( 72 | "--ignore-missing-groups", 73 | action="store_true", 74 | dest="ignore_if_group_not_found", 75 | help="ignore configured group(s) if it is not found in groups defined in manifest. This may be particulary useful when switching Manifest version back when some Groups defined later, was not there yet. In which case we can avoid unecessary Error caused by missing group", # noqa: E501 76 | ) 77 | parser.add_argument( 78 | "--ignore-missing-group-items", 79 | action="store_true", 80 | dest="ignore_group_item", 81 | help="ignore group element if it is not found among Manifest's Repos. WARNING: If you end up in need of this option, you have to understand that you end up with useles Manifest. Warnings will be printed for each Group element that is missing, so it may be easier to fix that. Using this option is NOT RECOMMENDED for normal use", # noqa: E501 82 | ) 83 | parser.set_defaults(run=run) 84 | 85 | 86 | def run(args: argparse.Namespace) -> None: 87 | gtf = GroupsToFind(args.groups, args.ignore_if_group_not_found) 88 | groups_seen = simulate_get_workspace_with_repos(args) 89 | gtf.found_these(groups_seen) 90 | 91 | try: 92 | workspace = get_workspace_with_repos( 93 | args, ignore_group_item=args.ignore_group_item 94 | ) 95 | except GroupNotFound: 96 | # try to obtain workspace ignoring group error 97 | # if group is found in Deep Manifest or Future Manifest, 98 | # do not report GroupNotFound. 99 | # if not, than raise exception at the very end 100 | workspace = get_workspace_with_repos( 101 | args, 102 | ignore_if_group_not_found=True, 103 | ignore_group_item=args.ignore_group_item, 104 | ) 105 | 106 | dm = None 107 | if args.use_deep_manifest is True: 108 | dm, gtf = get_deep_manifest_from_local_manifest_pcsrepo( 109 | workspace, 110 | gtf, 111 | ) 112 | 113 | wrs = WorkspaceReposSummary( 114 | workspace, 115 | gtf, 116 | dm, 117 | only_manifest=True, 118 | manifest_marker=args.use_manifest_marker, 119 | future_manifest=args.use_future_manifest, 120 | use_same_future_manifest=args.use_same_future_manifest, 121 | ) 122 | 123 | workspace_config = workspace.config 124 | 125 | status_header = StatusHeader( 126 | workspace, 127 | # display both: 'url' and 'branch' 128 | [StatusHeaderDisplayMode.URL, StatusHeaderDisplayMode.BRANCH], 129 | ) 130 | if args.manifest_branch: 131 | cfg_update_data = ConfigUpdateData(manifest_branch=args.manifest_branch) 132 | if ( 133 | status_header.register_change( 134 | cfg_update_data, [ConfigUpdateType.MANIFEST_BRANCH] 135 | ) 136 | is False 137 | ): 138 | return 139 | status_header.display() 140 | status_collector = StatusCollector( 141 | workspace, ignore_group_item=args.ignore_group_item 142 | ) 143 | 144 | repos = deepcopy(workspace.repos) 145 | 146 | wrs.prepare_repos() 147 | 148 | if args.strict_on_git_desc is False: 149 | repos += wrs.obtain_leftovers_repos(repos) 150 | 151 | if repos: 152 | # get repos to process, in this case it will be just 1 153 | these_repos, _ = get_deep_manifest_pcsrepo(repos, workspace_config.manifest_url) 154 | 155 | # num_jobs=1 as we will have only (max) 1 repo to process 156 | process_items(these_repos, status_collector, num_jobs=1) 157 | erase_last_line() 158 | 159 | statuses = status_collector.statuses 160 | 161 | wrs.ready_data( 162 | statuses, 163 | ) 164 | wrs.separate_statuses(workspace.repos) 165 | wrs.calculate_fields_len() 166 | 167 | # only calculate summary when there are some Workspace repos 168 | if workspace.repos: 169 | wrs.summary() 170 | 171 | # if the normal Repo(s) were not found, 172 | # there still may be some Deep Manifest or Future manifest leftovers 173 | wrs.check_for_leftovers() 174 | 175 | # check if we have found all Groups (if any provided) 176 | # and if not, throw exception ManifestGroupNotFound 177 | wrs.must_match_all_groups(ignore_if_group_not_found=args.ignore_if_group_not_found) 178 | --------------------------------------------------------------------------------