├── LICENSE ├── readme.md ├── .gitignore └── backup.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Alexander Kapitanov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## GitHub backup repositories 2 | Save your repos and list of stargazers & list of forks for them. Pure python3 and git with no dependencies to install. 3 | 4 | [GitHub API Documentation](https://docs.github.com/en/rest) 5 | 6 | | **Title** | GitHub Backup | 7 | |:------------|:--------------------| 8 | | **Author** | [Alexander Kapitanov](https://habr.com/ru/users/hukenovs/) | 9 | | **Sources** | Python3 | 10 | | **Contact** | `` | 11 | | **Release** | 16 Apr 2022 | 12 | | **License** | MIT | 13 | 14 | ### Command line arguments 15 | ```bash 16 | usage: backup.py [-h] -u USER_LOGIN [-t USER_TOKEN] [--user_forks] [-v] [-f] \ 17 | [--forks] [--stars] [--save | --clone] [--bare] [--recursive] [--starred] [-p SAVE_PATH] \ 18 | [-l REPO_LIST [REPO_LIST ...]] 19 | 20 | GitHub saver for stargazers, forks, repos. 21 | 22 | optional arguments: 23 | -h, --help show this help message and exit 24 | -u USER_LOGIN, --user_login USER_LOGIN 25 | User login 26 | -t USER_TOKEN, --user_token USER_TOKEN 27 | User access token 28 | --user_forks Save forked repos by user 29 | -v, --verbose Logging level=debug 30 | -f, --force Force save 31 | --forks Save list of forks 32 | --stars Save list of stargazers 33 | --save Save repos to `save_path` 34 | --clone Clone repos to `save_path` 35 | --bare Clone a bare git repo 36 | --recursive Recursive submodules 37 | --starred Get repositories starred by user 38 | -p SAVE_PATH, --save_path SAVE_PATH 39 | Save path to your repos 40 | -l REPO_LIST [REPO_LIST ...], --repo_list REPO_LIST [REPO_LIST ...] 41 | List of repos to clone or to save 42 | ``` 43 | 44 | #### Save list of stargazers to `{user_login}_stargazers.json`: 45 | `python backup.py -u USER -t TOKEN --stars` 46 | 47 | #### Save list of forks to `{user_login}_forks.json` 48 | `python backup.py -u USER -t TOKEN --forks ` 49 | 50 | #### Download repos to `save_path` 51 | `python backup.py -u USER -t TOKEN --repos -p ~/backups` 52 | 53 | #### Clone repos to `save_path` 54 | `python backup.py -u USER -t TOKEN --clone -p ~/backups` 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # User added 132 | .idea 133 | .DS_Store -------------------------------------------------------------------------------- /backup.py: -------------------------------------------------------------------------------- 1 | r"""GitHub saver for stargazers, forks, repos. Can clone all user repos. 2 | """ 3 | 4 | import os 5 | import json 6 | import argparse 7 | import logging 8 | import requests 9 | from typing import Optional 10 | from functools import cached_property 11 | 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class GitHubSaver: 17 | """GitHub Saver (download or clone user repos or starred repos, save stargazers and list of forks) 18 | 19 | Parameters 20 | ---------- 21 | user_login: str 22 | GitHub user login 23 | 24 | user_token: Optional[str] 25 | GitHub user access token. May be left. 26 | 27 | user_forks: bool 28 | Save forked repos by user. Default: False. 29 | 30 | """ 31 | API_URL = "https://api.github.com/" 32 | HEADERS = { 33 | "Accept": "application/vnd.github.v3+json", 34 | "Authorization": "", 35 | } 36 | 37 | def __init__(self, user_login: str, user_token: Optional[str] = None, user_forks: bool = False): 38 | self._user_login = user_login 39 | self._user_forks = user_forks 40 | self.__user_token = user_token 41 | if user_token is not None: 42 | self.HEADERS.update({"Authorization": user_token}) 43 | 44 | @property 45 | def user_login(self): 46 | return self._user_login 47 | 48 | @user_login.setter 49 | def user_login(self, value: str): 50 | self._user_login = value 51 | 52 | @cached_property 53 | def _repositories(self) -> list: 54 | return self.__api_request(stage='repos') 55 | 56 | @cached_property 57 | def _starred_list(self) -> list: 58 | return self.__api_request(stage='starred') 59 | 60 | @cached_property 61 | def owner_repositories(self) -> list: 62 | """List of user repositories""" 63 | return self._remote_request(content=self._repositories) 64 | 65 | @cached_property 66 | def owner_clone_links(self) -> list: 67 | """List of user repositories in clone link format""" 68 | return self._remote_request(content=self._repositories, clone_url=True) 69 | 70 | @cached_property 71 | def user_starred_list(self): 72 | """List of repos starred by user""" 73 | return self._remote_request(content=self._starred_list) 74 | 75 | @cached_property 76 | def user_starred_links(self): 77 | """List of links to repos starred by user""" 78 | return self._remote_request(content=self._starred_list, clone_url=True) 79 | 80 | def __response(self, url: str, stage: str = "") -> list: 81 | response = requests.get(url=os.path.join(url, stage), headers=self.HEADERS) 82 | if response.status_code == 200: 83 | return response.json() 84 | else: 85 | logger.warning(f"Cannot get response from {url}") 86 | 87 | def __api_request(self, stage: str) -> list: 88 | response = self.__response(self.API_URL, stage=f"users/{self._user_login}/{stage}") 89 | if not response: 90 | raise Exception(f"Cannot get response for {self._user_login}, {stage}") 91 | return response 92 | 93 | def _remote_request(self, content: list, clone_url: bool = False) -> list: 94 | repos = [] 95 | for repo in content: 96 | if not self._user_forks and repo['fork']: 97 | continue 98 | 99 | url_path = repo['clone_url'] if clone_url else repo['url'] 100 | repos.append(url_path) 101 | return repos 102 | 103 | def __save_list(self, destination: str): 104 | """Save list to json""" 105 | if destination == "stargazers": 106 | method = self.get_stargazers 107 | elif destination == "forks": 108 | method = self.get_forks 109 | elif destination == "issues": 110 | method = self.get_issues 111 | else: 112 | raise NotImplemented(f"Implement method for {destination}") 113 | 114 | repo_dict = {} 115 | for repo_url in self.owner_repositories: 116 | repo_name = os.path.basename(repo_url) 117 | if result := method(repo_url): 118 | repo_dict[repo_name] = result 119 | 120 | save_path = f"{self._user_login}_{destination}.json" 121 | with open(save_path, "w") as f: 122 | json.dump(repo_dict, f, indent=2, ensure_ascii=True) 123 | 124 | def save_stargazers(self): 125 | """Save all stargazers""" 126 | self.__save_list(destination="stargazers") 127 | 128 | def save_forks(self): 129 | """Save all forks""" 130 | self.__save_list(destination="forks") 131 | 132 | def save_issues(self): 133 | """Save all issues""" 134 | self.__save_list(destination="issues") 135 | 136 | def save_repos(self, save_path: str = ".", force: bool = False, starred: bool = False): 137 | """Save all repos""" 138 | repositories = self.user_starred_list if starred else self.owner_repositories 139 | 140 | for repo_url in repositories: 141 | repo_name = os.path.basename(repo_url) 142 | repo_resp = requests.get(url=os.path.join(repo_url, "zipball")) 143 | if repo_resp.status_code == 200: 144 | repo_path = os.path.join(save_path, f"{repo_name}.zip") 145 | logger.info(f"Save {repo_url} to {repo_path}") 146 | 147 | if force or not os.path.isfile(repo_path): 148 | with open(repo_path, 'wb') as ff: 149 | ff.write(repo_resp.content) 150 | else: 151 | logger.info(f"Repo {repo_resp} is already saved!") 152 | 153 | else: 154 | logger.warning(f"Cannot download repo {repo_resp}") 155 | 156 | def clone_repos(self, clone_path: str = ".", bare: bool = False, recursive: bool = False, starred: bool = False): 157 | """Save all repos""" 158 | repositories = self.user_starred_links if starred else self.owner_clone_links 159 | 160 | for repo_url in repositories: 161 | repo_name, _ = os.path.splitext(os.path.basename(repo_url)) 162 | repo_path = os.path.join(clone_path, repo_name) 163 | logger.info(f"Clone {repo_url} to {repo_path}") 164 | 165 | if self.__user_token: 166 | repo_url = repo_url.replace("github.com", f"{self._user_login}:{self.__user_token}@github.com") 167 | 168 | command = f"git clone {repo_url} {repo_path}" + " --bare" * bare + " --recursive" * recursive 169 | try: 170 | os.system(command) 171 | except SystemError: 172 | logger.warning(f"Cannot clone {repo_url}") 173 | 174 | def get_stargazers(self, url: str) -> list: 175 | """Get all stargazers""" 176 | star_list = [] 177 | if response := self.__response(url, "stargazers"): 178 | for gazer in response: 179 | star_list.append( 180 | { 181 | 'login': gazer['login'], 182 | 'id': gazer['id'], 183 | 'node_id': gazer['node_id'], 184 | } 185 | ) 186 | logger.info(f"Get stargazers for {url}") 187 | return star_list 188 | 189 | def get_forks(self, url: str) -> list: 190 | """Get all forks""" 191 | fork_list = [] 192 | if response := self.__response(url, "forks"): 193 | for fork in response: 194 | fork_list.append( 195 | { 196 | 'login': fork['owner']['login'], 197 | 'id': fork['id'], 198 | 'node_id': fork['node_id'], 199 | } 200 | ) 201 | logger.info(f"Get forks for {url}") 202 | return fork_list 203 | 204 | def get_issues(self, url: str) -> list: 205 | """Get all issues""" 206 | logger.info(f"Get issues for {url}") 207 | return self.__response(url, "issues") 208 | 209 | 210 | def __parser_github(): 211 | parser = argparse.ArgumentParser(description="GitHub saver for stargazers, forks, repos.") 212 | 213 | parser.add_argument("-u", "--user_login", type=str, required=True, help="User login") 214 | parser.add_argument("-t", "--user_token", type=str, default=None, help="User access token") 215 | parser.add_argument("--user_forks", action="store_true", help="Save forked repos by user") 216 | parser.add_argument("-v", "--verbose", action="store_true", help="Logging level=debug") 217 | parser.add_argument("-f", "--force", action="store_true", help="Force save") 218 | 219 | parser.add_argument("--forks", action="store_true", help="Save list of forks") 220 | parser.add_argument("--stars", action="store_true", help="Save list of stargazers") 221 | parser.add_argument("--issues", action="store_true", help="Save list of issues") 222 | groups = parser.add_mutually_exclusive_group() 223 | groups.add_argument("--save", action="store_true", help="Save repos to `save_path`") 224 | groups.add_argument("--clone", action="store_true", help=f"Clone repos to `save_path`") 225 | parser.add_argument("--bare", action="store_true", help="Clone a bare git repo") 226 | parser.add_argument("--recursive", action="store_true", help="Recursive submodules") 227 | parser.add_argument("--starred", action="store_true", help="Get repositories starred by user") 228 | parser.add_argument("-p", "--save_path", type=str, default=".", help="Save path to your repos") 229 | parser.add_argument("-l", "--repo_list", nargs="+", help="List of repos to clone or to save") 230 | # TODO: Implement repo list for saving custom repositories from input arguments. 231 | 232 | args, _ = parser.parse_known_args() 233 | 234 | for kk, vv in vars(args).items(): 235 | if vv: 236 | print(f"{kk :<20}: {vv}") 237 | 238 | logging.basicConfig( 239 | format=u'[LINE:%(lineno)d] %(levelname)-8s [%(asctime)s] %(message)s', 240 | level=logging.DEBUG if args.verbose else logging.INFO 241 | ) 242 | 243 | github_saver = GitHubSaver( 244 | user_login=args.user_login, 245 | user_token=args.user_token, 246 | user_forks=args.user_forks, 247 | ) 248 | 249 | if args.stars: 250 | github_saver.save_stargazers() 251 | 252 | if args.forks: 253 | github_saver.save_forks() 254 | 255 | if args.issues: 256 | github_saver.save_issues() 257 | 258 | if args.save: 259 | github_saver.save_repos( 260 | save_path=args.save_path, 261 | force=args.force, 262 | starred=args.starred, 263 | ) 264 | 265 | if args.clone: 266 | github_saver.clone_repos( 267 | clone_path=args.save_path, 268 | bare=args.bare, 269 | recursive=args.recursive, 270 | starred=args.starred, 271 | ) 272 | 273 | 274 | if __name__ == '__main__': 275 | __parser_github() 276 | --------------------------------------------------------------------------------