├── .gitignore ├── LICENSE ├── README.md ├── git-checkout-all ├── git-clean-branches-all ├── git-fetch-all ├── git-replace-remote-all ├── git-search-all ├── git-set-author-all ├── gitlab-clone-all ├── gitlab_clone_all_utils ├── __init__.py ├── repo_processor.py └── tip_handler.py ├── mypy.ini ├── mypy.sh ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | ### Linux template 3 | *~ 4 | 5 | # temporary files which can be created if a process still has a handle open of a deleted file 6 | .fuse_hidden* 7 | 8 | # KDE directory preferences 9 | .directory 10 | 11 | # Linux trash folder which might appear on any partition or disk 12 | .Trash-* 13 | 14 | # .nfs files are created when an open file is removed but is still being accessed 15 | .nfs* 16 | ### Python template 17 | # Byte-compiled / optimized / DLL files 18 | __pycache__/ 19 | *.py[cod] 20 | *$py.class 21 | 22 | # C extensions 23 | *.so 24 | 25 | # Distribution / packaging 26 | .Python 27 | build/ 28 | develop-eggs/ 29 | dist/ 30 | downloads/ 31 | eggs/ 32 | .eggs/ 33 | lib/ 34 | lib64/ 35 | parts/ 36 | sdist/ 37 | var/ 38 | wheels/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | 64 | # Translations 65 | *.mo 66 | *.pot 67 | 68 | # Django stuff: 69 | *.log 70 | local_settings.py 71 | 72 | # Flask stuff: 73 | instance/ 74 | .webassets-cache 75 | 76 | # Scrapy stuff: 77 | .scrapy 78 | 79 | # Sphinx documentation 80 | docs/_build/ 81 | 82 | # PyBuilder 83 | target/ 84 | 85 | # Jupyter Notebook 86 | .ipynb_checkpoints 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # celery beat schedule file 92 | celerybeat-schedule 93 | 94 | # SageMath parsed files 95 | *.sage.py 96 | 97 | # Environments 98 | .env 99 | .venv 100 | env/ 101 | venv/ 102 | ENV/ 103 | 104 | # Spyder project settings 105 | .spyderproject 106 | .spyproject 107 | 108 | # Rope project settings 109 | .ropeproject 110 | 111 | # mkdocs documentation 112 | /site 113 | 114 | # mypy 115 | .mypy_cache/ 116 | 117 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright 2017 Allis Tauri 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software 6 | and associated documentation files (the "Software"), to deal in the Software without restriction, 7 | including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, 9 | subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or substantial 12 | portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT 15 | LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 16 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 17 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 18 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitLab Clone All 2 | 3 | Provides several scripts to: 4 | 5 | * clone all repositories from a gitlab server available to user 6 | * fetch/pull all repositories under the current directory 7 | * change author/email for all reposituries under the current directory 8 | -------------------------------------------------------------------------------- /git-checkout-all: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | 5 | from git import Repo 6 | 7 | from gitlab_clone_all_utils import RepoProcessor 8 | 9 | 10 | class Worker(RepoProcessor): 11 | def _process(self, repo: Repo, path: str) -> None: 12 | check_dirty = False 13 | local_head = None 14 | for branch in self._args.branch: 15 | try: 16 | local_head = repo.heads[branch] 17 | check_dirty = local_head.commit != repo.head.commit 18 | break 19 | except IndexError: 20 | continue 21 | if not local_head: 22 | remote = repo.remote(self._args.remote) 23 | if remote.exists(): 24 | remote_head = None 25 | remote_heads = {ref.remote_head: ref for ref in remote.refs} 26 | for branch in self._args.branch: 27 | remote_head = remote_heads.get(branch) 28 | if remote_head: 29 | break 30 | if remote_head: 31 | local_head = repo.create_head(remote_head.remote_head, 32 | commit=remote_head.commit) 33 | local_head.set_tracking_branch(remote_head) 34 | check_dirty = local_head.commit != repo.head.commit 35 | else: 36 | if not self._args.quite and not self._args.create: 37 | print('Remote "{}" does not have "{}" branch(es) in {}' 38 | .format(self._args.remote, self._args.branch, path)) 39 | else: 40 | if not self._args.quite and not self._args.create: 41 | print('Remote "{}" does not exist in {}'.format(self._args.remote, path)) 42 | if check_dirty and repo.is_dirty(): 43 | print('{} is dirty'.format(path)) 44 | return 45 | if not local_head and self._args.create: 46 | local_head = repo.create_head(self._args.branch[0]) 47 | print('Created {} branch in {}'.format(local_head.name, path)) 48 | if local_head: 49 | local_head.checkout() 50 | print('{} branch was checked out in {}'.format(local_head.name, path)) 51 | 52 | 53 | if __name__ == '__main__': 54 | parser = argparse.ArgumentParser(description='Checkout a particular branch ' 55 | 'in all the repos if it exists') 56 | parser.add_argument('-q', '--quite', action='store_true', 57 | help='Do not output warnings about missing remotes and branches') 58 | parser.add_argument('-c', '--create', action='store_true', 59 | help='Create new local branch at current commit even if there is no ' 60 | 'corresponding remote branch') 61 | parser.add_argument('branch', type=str, nargs='+', default=[], 62 | help='Local branch name. If local repo does not have this branch, ' 63 | 'but the remote does, it is checked out instead as a new local branch') 64 | parser.add_argument('remote', nargs='?', default='origin', 65 | help='Remote to check for the branch') 66 | Worker.execute(parser) 67 | -------------------------------------------------------------------------------- /git-clean-branches-all: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | import sys 5 | 6 | from git import Repo 7 | 8 | from gitlab_clone_all_utils import TipHandler, RepoProcessor 9 | 10 | 11 | class Worker(RepoProcessor, TipHandler): 12 | def _remove_head(self, repo, head): 13 | do_remove = True 14 | if not self._args.auto: 15 | answer = input('Remove? [yes]/no/quit: ') 16 | if answer.lower() in ['q', 'quit']: 17 | sys.exit(0) 18 | do_remove = not answer or answer.lower() in ['y', 'yes'] 19 | if do_remove: 20 | repo.delete_head(head) 21 | print('Removed.') 22 | 23 | def _can_remove(self, repo, head): 24 | if head.name in self._args.exclude: 25 | return False 26 | if head == repo.active_branch: 27 | return False 28 | if not self._args.include_tracking and head.tracking_branch(): 29 | return False 30 | return True 31 | 32 | def _process(self, repo: Repo, path: str) -> None: 33 | print('Processing {}:'.format(path)) 34 | commits = self._commits_per_head(repo.heads) 35 | tips_per_commit = {} 36 | to_remove = [] 37 | for head in commits: 38 | if not self._is_tip(head, commits): 39 | if self._can_remove(repo, head): 40 | to_remove.append((head, '{} is not a tip.'.format(head.name))) 41 | else: 42 | tips = tips_per_commit.setdefault(head.commit, set()) 43 | tips.add(head) 44 | for tips in tips_per_commit.values(): 45 | if len(tips) > 1: 46 | removable = [h for h in tips if self._can_remove(repo, h)] 47 | if len(removable) == 1: 48 | head = removable[0] 49 | tips.remove(head) 50 | to_remove.append((head, 51 | '{} points to the same commit as: {}' 52 | .format(head.name, tips.pop().name))) 53 | elif removable: 54 | print('These tips point to the same commit:\n{}' 55 | .format('\n'.join('[{}] {}'.format(i, h.name) 56 | for i, h in enumerate(removable)))) 57 | answer = input('Choose one to leave: ') 58 | for i, head in enumerate(removable): 59 | if str(i) != answer: 60 | to_remove.append((head, 61 | 'Removing duplicate branch: {}' 62 | .format(head.name))) 63 | for head, message in to_remove: 64 | print(message) 65 | self._remove_head(repo, head) 66 | print('Processed {}\n'.format(path)) 67 | 68 | 69 | if __name__ == '__main__': 70 | parser = argparse.ArgumentParser(description='Delete local branches that are not tips') 71 | parser.add_argument('-x', '--exclude', type=str, nargs='+', default=[], 72 | help='Exclude these branches') 73 | parser.add_argument('-t', '--include-tracking', action='store_true', 74 | help='Include tracking branches') 75 | parser.add_argument('-a', '--auto', action='store_true', 76 | help='Automatically remove branches instead of asking each time') 77 | Worker.execute(parser) 78 | -------------------------------------------------------------------------------- /git-fetch-all: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | 5 | from git import Repo 6 | 7 | from gitlab_clone_all_utils import RepoProcessor 8 | 9 | 10 | def _print_fetch_info(action, info_iter, heads, repo): 11 | for info in info_iter: 12 | head, commit = heads.get(info.ref, (None, None)) 13 | if head and commit != info.commit: 14 | if head.commit == info.commit: 15 | print('{} {} -> {}'.format(action, head, info.ref)) 16 | elif head.commit in info.commit.iter_parents(): 17 | if head != repo.head.ref: 18 | head.set_commit(info.commit) 19 | print('{} {} -> {}'.format(action, head, info.ref)) 20 | elif not repo.is_dirty(): 21 | repo.head.reset(info.commit, index=True, working_tree=True) 22 | print('{} {} -> {}'.format('Reset', head, info.ref)) 23 | else: 24 | print('Working directory of {} branch is dirty!\n' 25 | '{} {} -> {}'.format(head.name, action, info.ref, info.commit)) 26 | else: 27 | print('{} {} -> {}'.format(action, info.ref, info.commit)) 28 | if info.commit in head.commit.iter_parents(): 29 | print('Local head {} is ahead of the remote.'.format(head)) 30 | else: 31 | print('Local head {} is on a different branch.'.format(head)) 32 | if info.note: 33 | print('NOTE: {}'.format(info.note)) 34 | 35 | 36 | class Worker(RepoProcessor): 37 | def _process(self, repo: Repo, path: str) -> None: 38 | remote = repo.remote(self._args.remote) 39 | if not remote.exists(): 40 | return 41 | heads = {} 42 | r_heads = {ref.remote_head: ref for ref in remote.refs} 43 | for head in repo.heads: 44 | tb = head.tracking_branch() 45 | if tb: 46 | heads[tb] = (head, head.commit) 47 | elif self._args.set_remote: 48 | r_head = r_heads.get(head.name) 49 | if r_head: 50 | print('{} is now tracking {}'.format(head.name, r_head.name)) 51 | head.set_tracking_branch(r_head) 52 | heads[r_head] = (head, head.commit) 53 | if self._args.pull and not repo.is_dirty() and repo.active_branch.tracking_branch(): 54 | _print_fetch_info('Pulled', remote.pull(), heads, repo) 55 | else: 56 | _print_fetch_info('Fetched', remote.fetch(), heads, repo) 57 | print('{} updated\n'.format(path)) 58 | 59 | 60 | if __name__ == '__main__': 61 | parser = argparse.ArgumentParser(description='Fetch all branches of all the repositories ' 62 | 'found under the current directory') 63 | parser.add_argument('-p', '--pull', action='store_true', 64 | help='Pull active branch if the working directory is clean') 65 | parser.add_argument('-r', '--remote', type=str, default='origin', 66 | help='Use this remote. Default: "origin"') 67 | parser.add_argument('-s', '--set-remote', action='store_true', 68 | help="Set remote of local heads that don't have one but have a remote " 69 | "head with the same name") 70 | Worker.execute(parser) 71 | -------------------------------------------------------------------------------- /git-replace-remote-all: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | import os 5 | import traceback 6 | 7 | from git import Repo, InvalidGitRepositoryError 8 | 9 | if __name__ == '__main__': 10 | parser = argparse.ArgumentParser(description='Replace parts of the urls of the remotes ' 11 | 'of all repos under the current directory.') 12 | parser.add_argument('pattern', help='Search for this substring/pattern') 13 | parser.add_argument('replace', help='Replace with this string') 14 | parser.add_argument('-r', '--regex', action='store_true', 15 | help='Search string is a regex pattern') 16 | parser.add_argument('-a', '--all', action='store_true', 17 | help='Replace all occurrences of the pattern') 18 | args = parser.parse_args() 19 | root = os.getcwd() 20 | # define replace method 21 | if args.regex: 22 | import re 23 | 24 | pattern = re.compile(args.pattern) 25 | if args.all: 26 | def replace(s): 27 | return pattern.sub(args.replace, s) 28 | else: 29 | def replace(s): 30 | return pattern.subn(args.replace, s)[0] 31 | else: 32 | num = -1 if args.all else 1 33 | 34 | def replace(s): 35 | return s.replace(args.pattern, args.replace, num) 36 | # replace remote urls 37 | for entry in os.listdir('.'): 38 | path = os.path.join(root, entry) 39 | if os.path.isdir(path): 40 | os.chdir(path) 41 | try: 42 | try: 43 | repo = Repo(path) 44 | except InvalidGitRepositoryError: 45 | continue 46 | if repo.bare: 47 | continue 48 | for remote in repo.remotes: 49 | for url in remote.urls: 50 | new_url = replace(url) 51 | if new_url != url: 52 | remote.set_url(new_url, url) 53 | print('{}:\n\t{} -> {}'.format(path, url, new_url)) 54 | except Exception as ex: 55 | print('Error while changing remote url in {}:\n{!s}\n{}' 56 | .format(path, ex, traceback.format_exc())) 57 | -------------------------------------------------------------------------------- /git-search-all: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | import os 5 | import re 6 | import sys 7 | from types import MemberDescriptorType 8 | from typing import Any, List, Generator, Set, Tuple, Pattern 9 | 10 | from git import Repo, Head, Commit 11 | 12 | from gitlab_clone_all_utils import TipHandler, RepoProcessor 13 | from gitlab_clone_all_utils.tip_handler import HeadToCommitMap, CommitSet 14 | 15 | 16 | def _format_commit(commit): 17 | return '{} {} by {}: {}\n' \ 18 | '-------------------------------------------------------------------------------------' \ 19 | .format(commit.authored_datetime, 20 | commit.hexsha, 21 | commit.author, 22 | commit.message.strip()) 23 | 24 | 25 | def _get_attr(obj: Any, attrs: List[str]) -> Any: 26 | if not attrs: 27 | return obj 28 | res = getattr(obj, attrs[0], None) 29 | if res: 30 | return _get_attr(res, attrs[1:]) 31 | raise AttributeError('{!s} has no "{}" attribute'.format(type(obj), attrs[0])) 32 | 33 | 34 | class Worker(RepoProcessor, TipHandler): 35 | 36 | def __init__(self, args: argparse.Namespace, path: str) -> None: 37 | super().__init__(args, path) 38 | self._terms: List[Pattern[str]] = [] 39 | self._commit_terms: List[Tuple[List[str], Pattern[str]]] = [] 40 | if args.what == 'commit': 41 | for term_spec in args.term: 42 | attr_term = term_spec.split('::', 1) 43 | if len(attr_term) > 1: 44 | self._commit_terms.append((attr_term[0].strip().split('.'), re.compile(attr_term[1]))) 45 | else: 46 | self._commit_terms.append(([], re.compile(term_spec))) 47 | else: 48 | if isinstance(args.term, list): 49 | self._terms = [re.compile(term) for term in args.term] 50 | elif isinstance(args.term, str): 51 | self._terms = [re.compile(args.term)] 52 | else: 53 | print(f'ERROR: unsupported type of term: {type(args.term)}') 54 | self._terms = [] 55 | 56 | def _print_not_matched(self, term: Pattern[str], subject: str) -> None: 57 | if self._args.debug: 58 | print(f'Term "{term.pattern}" did not match against {type(subject)}: {subject!s}') 59 | 60 | def _check_subject(self, subject: Any) -> bool: 61 | if isinstance(subject, Commit): 62 | match = bool(self._commit_terms) 63 | for attr, term in self._commit_terms: 64 | s = _get_attr(subject, attr) if attr else subject.message 65 | if not term.search(str(s)): 66 | self._print_not_matched(term, s) 67 | match = False 68 | break 69 | else: 70 | s = str(subject) 71 | match = bool(self._terms) 72 | for term in self._terms: 73 | if not term.search(s): 74 | self._print_not_matched(term, s) 75 | match = False 76 | break 77 | return match 78 | 79 | def _all_heads(self, repo: Repo) -> Generator[Head, Any, None]: 80 | if self._args.local_only or not self._args.remote_only: 81 | for head in repo.heads: 82 | yield head 83 | if not self._args.local_only: 84 | remote = repo.remote(self._args.remote) 85 | if remote.exists(): 86 | for ref in remote.refs: 87 | yield ref 88 | 89 | def _all_commits(self, repo: Repo) -> HeadToCommitMap: 90 | return self._commits_per_head(self._all_heads(repo)) 91 | 92 | def _search_commit(self, repo: Repo) -> str: 93 | commits = self._all_commits(repo) 94 | found: CommitSet = set() 95 | for head in commits: 96 | for commit in commits[head]: 97 | if commit not in found: 98 | try: 99 | if self._check_subject(commit): 100 | found.add(commit) 101 | except AttributeError as exc: 102 | print(str(exc)) 103 | return '\n'.join(_format_commit(c) 104 | for c in sorted(found, 105 | key=lambda c: c.authored_datetime, 106 | reverse=True)) 107 | 108 | def _search_branch(self, repo: Repo) -> str: 109 | found: Set[Head] = set() 110 | for head in self._all_heads(repo): 111 | if head not in found: 112 | if self._check_subject(head.name): 113 | found.add(head) 114 | return '\n'.join(head.name for head in sorted(found, key=lambda h: h.name)) 115 | 116 | def _find_all_tips(self, repo: Repo) -> str: 117 | tips: List[Tuple[str, Commit]] = [] 118 | commits = self._commits_per_head(repo.heads) 119 | for head in commits: 120 | if self._is_tip(head, commits): 121 | name = head.name 122 | if not self._args.local_only: 123 | tracking = head.tracking_branch() 124 | if tracking: 125 | head = tracking 126 | name = tracking.name 127 | tips.append((name, head.commit)) 128 | if not self._args.local_only: 129 | remote = repo.remote(self._args.remote) 130 | if remote.exists(): 131 | head_commits = set(h.commit for h in repo.heads) 132 | remote_heads = [ref for ref in remote.refs 133 | if ref.remote_head != 'HEAD' and ref.commit not in head_commits] 134 | commits = self._commits_per_head(remote.refs) 135 | for ref in remote_heads: 136 | if self._is_tip(ref, commits): 137 | tips.append((ref.name, ref.commit)) 138 | return '\n'.join('{}\n{}'.format(tip[0], _format_commit(tip[1])) 139 | for tip in 140 | sorted(tips, reverse=True, key=lambda x: x[1].authored_datetime)) 141 | 142 | def _find_all_ahead_branches(self, repo: Repo) -> str: 143 | self._args.local_only = True 144 | found: Set[Head] = set() 145 | for head in self._all_heads(repo): 146 | if head not in found: 147 | tracking = head.tracking_branch() 148 | if not tracking: 149 | found.add(head) 150 | elif tracking.commit in head.commit.iter_parents(): 151 | found.add(head) 152 | return '\n'.join('{}\n{}'.format(head, _format_commit(head.commit)) 153 | for head in sorted(found, key=lambda h: h.name)) 154 | 155 | @staticmethod 156 | def _print_res(path: str, res: Any) -> None: 157 | if res: 158 | print('{}:\n{}\n'.format(os.path.basename(path), res)) 159 | 160 | def _process(self, repo: Repo, path: str) -> None: 161 | if self._args.what == 'commit': 162 | self._print_res(path, self._search_commit(repo)) 163 | elif self._args.what == 'branch': 164 | self._print_res(path, self._search_branch(repo)) 165 | elif self._args.what == 'repo': 166 | if self._args.term == 'tips': 167 | self._print_res(path, self._find_all_tips(repo)) 168 | elif self._args.term == 'ahead': 169 | self._print_res(path, self._find_all_ahead_branches(repo)) 170 | else: 171 | if self._check_subject(path): 172 | print(path) 173 | 174 | 175 | def _print_commit_attrs(): 176 | attrs: List[str] = [] 177 | for name, obj in vars(Commit).items(): 178 | if name.startswith('_'): 179 | continue 180 | if isinstance(obj, property) or isinstance(obj, MemberDescriptorType): 181 | attrs.append(f'{name} - {obj.__doc__.strip()}' 182 | if obj.__doc__ and not obj.__doc__.isspace() 183 | else name) 184 | print('\n'.join(sorted(attrs))) 185 | 186 | 187 | if __name__ == '__main__': 188 | parser = argparse.ArgumentParser(description='Search within all repositories') 189 | parser.add_argument('-d', '--debug', action='store_true', 190 | help='Enable additional debugging output') 191 | parser.add_argument('-l', '--local-only', action='store_true', 192 | help='Only search within local branches') 193 | parser.add_argument('-r', '--remote-only', action='store_true', 194 | help='Only search within local branches') 195 | parser.add_argument('-R', '--remote', type=str, default='origin', 196 | help='Remote to check if required') 197 | subparsers = parser.add_subparsers(dest='what') 198 | repo_parser = subparsers.add_parser('repo', 199 | help='Search repo for tip heads, or branches that are ahead of their remotes') 200 | repo_parser.add_argument('term', type=str, 201 | help='If term is "tips", the search will return commit info of all the tips, ' 202 | 'while "ahead" will return only those that are ' 203 | 'ahead of their remotes. Ano other term will be matched as regex against repo path.') 204 | branch_parser = subparsers.add_parser('branch', 205 | help='Search branch names') 206 | branch_parser.add_argument('term', type=str, nargs="+", 207 | help='Regex to match branch names') 208 | commit_parser = subparsers.add_parser('commit', help='Search for particular commits') 209 | commit_parser.add_argument('--list-attrs', action='store_true', 210 | help='List all attributes of a commit') 211 | commit_parser.add_argument('term', type=str, nargs='*', 212 | help='Search term in the form [attribute[.attr[.attr...]]::]regex. ' 213 | 'The optional part tells which attribute of a commit to check, ' 214 | 'and the regex is the search term. If your regex contains double colon, ' 215 | 'prepend it with empty attribute specification like this "::reg::ex.+to\\search".' 216 | 'If several terms are given, they all should match for a commit to match.') 217 | args = parser.parse_args() 218 | if args.what is None: 219 | parser.print_help() 220 | sys.exit(1) 221 | if args.what == 'commit': 222 | if args.list_attrs: 223 | _print_commit_attrs() 224 | sys.exit() 225 | if args.term is None: 226 | commit_parser.print_help() 227 | sys.exit(1) 228 | Worker.execute(args) 229 | -------------------------------------------------------------------------------- /git-set-author-all: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | 5 | from git import Repo 6 | 7 | from gitlab_clone_all_utils import RepoProcessor 8 | 9 | 10 | class Worker(RepoProcessor): 11 | def _process(self, repo: Repo, path: str) -> None: 12 | cfg = repo.config_writer() 13 | if not cfg.has_section('user'): 14 | cfg.add_section('user') 15 | cfg.set('user', 'name', self._args.author) 16 | if self._args.email: 17 | cfg.set('user', 'email', self._args.email) 18 | cfg.write() 19 | print('{}: {} <{}>'.format(path, 20 | cfg.get('user', 'name'), 21 | cfg.get('user', 'email') or 'no email')) 22 | 23 | 24 | if __name__ == '__main__': 25 | parser = argparse.ArgumentParser(description='Set author and email for all repos ' 26 | 'found under the current directory') 27 | parser.add_argument('author', help='Author name') 28 | parser.add_argument('email', nargs='?', default='', help='Author email') 29 | Worker.execute(parser) 30 | -------------------------------------------------------------------------------- /gitlab-clone-all: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | import json 5 | import sys 6 | import pathlib 7 | import traceback 8 | import urllib.parse as parse 9 | import urllib.request as request 10 | from http import HTTPStatus 11 | 12 | from git import Repo 13 | 14 | api_path = '/api/v{:d}/projects/' 15 | 16 | 17 | def _exit(_response): 18 | print(str(_response.read())) 19 | sys.exit(1) 20 | 21 | 22 | def _read_json(_response): 23 | return json.loads(_response.read().decode()) 24 | 25 | 26 | if __name__ == '__main__': 27 | parser = argparse.ArgumentParser(description='Clone ALL repositories from GitLab server ' 28 | 'available to user') 29 | parser.add_argument('token', type=str, 30 | help='GitLab secret token of the user') 31 | parser.add_argument('server', type=str, 32 | help='URL of the GitLab server') 33 | parser.add_argument('-b', '--branch', type=str, 34 | help='Checkout this branch after cloning') 35 | parser.add_argument('-n', '--namespace', type=str, 36 | help='Only clone projects from this namespace') 37 | parser.add_argument('-g', '--group', type=str, 38 | help='Only clone projects from this group with path, for example "group" or "group/subgroup"') 39 | parser.add_argument('-in', '--into-namespace', action='store_true', 40 | help='Clone into namespace subdirectory') 41 | parser.add_argument('-f', '--fresh', action='store_true', 42 | help='Get the freshest branch') 43 | parser.add_argument('-v', '--api-version', type=int, default=4, 44 | help='GitLab api version. Default: 4') 45 | # parse and process args 46 | args = parser.parse_args() 47 | server_url = parse.urlsplit(args.server) 48 | if not server_url.scheme: 49 | server_url = 'http://'+args.server 50 | else: 51 | server_url = server_url.geturl() 52 | server_url = parse.urljoin(server_url, api_path.format(args.api_version)) 53 | header = {'PRIVATE-TOKEN': args.token} 54 | proxy_support = request.ProxyHandler({}) 55 | opener = request.build_opener(proxy_support) 56 | request.install_opener(opener) 57 | # fetch pand process project data 58 | req = request.Request(server_url) 59 | req.headers.update(header) 60 | projects = [] 61 | try: 62 | num_pages = 1 63 | print('Fetching 1 page of project infos.') 64 | with request.urlopen(req) as response: 65 | if response.getcode() == HTTPStatus.OK: 66 | projects += _read_json(response) 67 | num_pages = int(response.info()['X-Total-Pages']) 68 | else: 69 | exit(response) 70 | for page_num in range(num_pages-1): 71 | pn = page_num + 2 72 | print('Fetching {} of {} page of project infos.'.format(pn, num_pages)) 73 | req.full_url = parse.urljoin(server_url, '?page={}'.format(pn)) 74 | with request.urlopen(req) as response: 75 | if response.getcode() == HTTPStatus.OK: 76 | projects += _read_json(response) 77 | else: 78 | exit(response) 79 | num_projects = len(projects) 80 | for i, p in enumerate(projects): 81 | pdir = pathdir = p['name'] 82 | if args.into_namespace: 83 | pdir = pathdir = p['path_with_namespace'] 84 | if args.namespace: 85 | ns = p['namespace']['name'] 86 | if args.namespace != p['namespace']['name']: 87 | print('Skipping {} from {} namespace...'.format(pdir, ns)) 88 | continue 89 | if args.group: 90 | nwn = p['name_with_namespace'].replace(" ", "") 91 | if not nwn.startswith(args.group): 92 | print('Skipping {} from {} group...'.format(pdir, nwn)) 93 | continue 94 | pathdir = p['path_with_namespace'] 95 | if pathlib.Path(pathdir).exists(): 96 | print('Project {} already exists. Skipping...'.format(pdir)) 97 | continue 98 | try: 99 | pathlib.Path(pathdir).mkdir(parents=True, exist_ok=True) 100 | repo = Repo.clone_from(p['ssh_url_to_repo'], pathdir) 101 | origin = repo.remote('origin') 102 | for f in origin.fetch(): 103 | pass 104 | if not args.fresh and args.branch and repo.active_branch.name != args.branch: 105 | for r in origin.refs: 106 | if r.remote_head == args.branch: 107 | repo.create_head(args.branch, r).set_tracking_branch(r).checkout() 108 | if args.fresh: 109 | last_commit = repo.head.commit 110 | repo.head.reference = repo.commit(last_commit) 111 | repo.head.reset(index=True, working_tree=True) 112 | print('Cloned {} of {}: {} path: {}'.format(i+1, num_projects, pdir, pathdir)) 113 | except Exception as error: 114 | print('Error while cloning {}:\n{!s}\n{}' 115 | .format(pdir, error, traceback.format_exc())) 116 | continue 117 | except Exception as error: 118 | print(str(error)) 119 | sys.exit(2) 120 | -------------------------------------------------------------------------------- /gitlab_clone_all_utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .repo_processor import RepoProcessor 2 | from .tip_handler import TipHandler 3 | 4 | __all__ = ['RepoProcessor', 5 | 'TipHandler'] 6 | -------------------------------------------------------------------------------- /gitlab_clone_all_utils/repo_processor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import traceback 4 | from argparse import Namespace, ArgumentParser 5 | from typing import Union 6 | 7 | from git import Repo, InvalidGitRepositoryError 8 | 9 | 10 | class RepoProcessor: 11 | def __init__(self, args: Namespace, path: str = '.') -> None: 12 | self._args = args 13 | self._path = path 14 | 15 | def _process_path(self, path: str) -> None: 16 | try: 17 | os.chdir(path) 18 | try: 19 | repo = Repo(path) 20 | except InvalidGitRepositoryError: 21 | return 22 | if repo.bare: 23 | return 24 | self._process(repo, path) 25 | except Exception as ex: 26 | print('Error while processing {}:\n{!s}\n{}' 27 | .format(path, ex, traceback.format_exc())) 28 | 29 | def run(self) -> int: 30 | start_path = os.path.abspath(self._path) 31 | self._process_path(start_path) 32 | for root, dirs, _files in os.walk(start_path): 33 | for dirname in dirs: 34 | if dirname == '.git': 35 | continue 36 | self._process_path(os.path.join(root, dirname)) 37 | return 0 38 | 39 | def _process(self, repo: Repo, path: str) -> None: 40 | raise NotImplementedError() 41 | 42 | @classmethod 43 | def execute(cls, parser_or_args: Union[ArgumentParser, Namespace], path: str = '.') -> None: 44 | if isinstance(parser_or_args, ArgumentParser): 45 | args = parser_or_args.parse_args() 46 | elif isinstance(parser_or_args, Namespace): 47 | args = parser_or_args 48 | else: 49 | raise ValueError(f'Unsupported type of parser_or_args: {type(parser_or_args)}') 50 | sys.exit(cls(args, path).run()) 51 | -------------------------------------------------------------------------------- /gitlab_clone_all_utils/tip_handler.py: -------------------------------------------------------------------------------- 1 | from typing import Iterator, Set, Dict, Union 2 | 3 | from git import Commit, Head, Blob, Tree, TagObject 4 | 5 | CommitSet = Set[Union[TagObject, Tree, Commit, Blob]] 6 | HeadToCommitMap = Dict[Head, CommitSet] 7 | 8 | 9 | class TipHandler(object): 10 | @staticmethod 11 | def _commits_per_head(heads: Iterator[Head]) -> HeadToCommitMap: 12 | return {head: {head.commit} | set(head.commit.iter_parents()) 13 | for head in heads} 14 | 15 | @staticmethod 16 | def _is_tip(head: Head, commits: HeadToCommitMap) -> bool: 17 | tip = True 18 | for other in commits: 19 | if head.commit != other.commit: 20 | tip = head.commit not in commits[other] 21 | if not tip: 22 | break 23 | return tip 24 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | follow_imports = silent 3 | disallow_untyped_calls = False 4 | disallow_untyped_defs = False 5 | disallow_incomplete_defs = True 6 | check_untyped_defs = True 7 | disallow_subclassing_any = False 8 | disallow_untyped_decorators = False 9 | warn_redundant_casts = True 10 | warn_return_any = True 11 | warn_unused_ignores = True 12 | warn_unused_configs = False 13 | no_implicit_optional = False 14 | ignore_missing_imports = True 15 | strict_optional = False 16 | 17 | mypy_path = gitlab-clone-all 18 | -------------------------------------------------------------------------------- /mypy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cwd=$(pwd) 4 | 5 | abs_path_regex="s|^(?!/)|$cwd/|g" 6 | 7 | export MYPYPATH="$1" 8 | shift 9 | mypy $@ | perl -pe $abs_path_regex 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | GitPython 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='gitlab-clone-all', 5 | version='1.2', 6 | packages=['gitlab_clone_all_utils'], 7 | url='', 8 | license='MIT', 9 | author='Evgeny Taranov', 10 | author_email='evgeny.taranov@gmail.com', 11 | description='Tool to work with multi-repo projects ' 12 | 'based on git and gitlab', 13 | scripts=['gitlab-clone-all', 14 | 'git-fetch-all', 15 | 'git-set-author-all', 16 | 'git-checkout-all', 17 | 'git-replace-remote-all', 18 | 'git-clean-branches-all', 19 | 'git-search-all'] 20 | ) 21 | --------------------------------------------------------------------------------