├── tests ├── __init__.py └── test_backup.py ├── requirements.txt ├── bitbucket-backup ├── .travis.yml ├── setup.cfg ├── LICENSE.txt ├── .gitignore ├── setup.py ├── README.md └── backup.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ipdb==0.12 2 | requests==2.22.0 3 | requests-oauthlib==1.2.0 4 | -------------------------------------------------------------------------------- /bitbucket-backup: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | from backup import main 3 | 4 | main() 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.7" 6 | 7 | install: 8 | - pip install -e .[tests] -q 9 | 10 | # command to run tests 11 | script: 12 | - nosetests 13 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | detailed-errors=1 3 | with-coverage=1 4 | verbosity=2 5 | cover-package=compass 6 | cover-html=1 7 | 8 | [flake8] 9 | max-line-length = 200 10 | exclude = ./tests/*.py,./compass/__init__.py 11 | -------------------------------------------------------------------------------- /tests/test_backup.py: -------------------------------------------------------------------------------- 1 | import unittest2 as unittest 2 | 3 | from backup import * # noqa 4 | 5 | 6 | class BackupTestCase(unittest.TestCase): 7 | def test_stub(self): 8 | self.assertEqual(1, 1) 9 | 10 | 11 | if __name__ == "__main__": 12 | unittest.main() 13 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | DO WHAT YOU WANT TO PUBLIC LICENSE 2 | Version 4, May 2015 3 | 4 | Copyright (C) 2015 Sam Kuehn 5 | 6 | Everyone is permitted to copy and distribute verbatim or modified 7 | copies of this license document, and changing it is allowed as long 8 | as the name is changed. 9 | 10 | DO WHAT YOU WANT TO PUBLIC LICENSE 11 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 12 | 13 | 0. You just DO WHAT YOU WANT TO. -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .cache 41 | nosetests.xml 42 | coverage.xml 43 | 44 | # Translations 45 | *.mo 46 | *.pot 47 | 48 | # Django stuff: 49 | *.log 50 | 51 | # Sphinx documentation 52 | docs/_build/ 53 | 54 | # PyBuilder 55 | target/ 56 | .idea 57 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import find_packages, setup 3 | 4 | TEST_REQUIRES = ["nose", "unittest2"] 5 | 6 | INSTALL_REQUIRES = ["requests", "requests-oauthlib"] 7 | try: 8 | import argparse # noqa 9 | except ImportError: 10 | INSTALL_REQUIRES.append("argparse") 11 | 12 | SCRIPTS = ["backup.py", "bitbucket-backup"] 13 | 14 | setup( 15 | name="bitbucket-backup", 16 | version="0.0.1", 17 | author="Sam Kuehn", 18 | author_email="samkuehn@gmail.com", 19 | url="https://github.com/samkuehn/bitbucket-backup", 20 | description="Python script to backup Bitbucket repos", 21 | long_description=__doc__, 22 | packages=find_packages(exclude=("tests", "tests.*")), 23 | scripts=SCRIPTS, 24 | zip_safe=False, 25 | extras_require={"tests": TEST_REQUIRES}, 26 | tests_require=TEST_REQUIRES, 27 | install_requires=INSTALL_REQUIRES, 28 | test_suite="tests", 29 | include_package_data=True, 30 | classifiers=[ 31 | "Intended Audience :: System Administrators", 32 | "Operating System :: OS Independent", 33 | "Topic :: System :: Systems Administrationt", 34 | "Programming Language :: Python", 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bitbucket backup 2 | 3 | [![Build Status](https://travis-ci.org/samkuehn/bitbucket-backup.svg?branch=master)](https://travis-ci.org/samkuehn/bitbucket-backup) 4 | 5 | ## Description 6 | This python script will backup all of your Bitbucket repos (both mercurial and git) locally. 7 | If the repository does not exist locally the repo will be cloned to the . 8 | If the repo does exist locally an `hg pull` will be run for mercurial repos, 9 | and `git remote update` will be run for git repos. 10 | 11 | ## Installation 12 | 13 | ```bash 14 | pip install -U https://github.com/samkuehn/bitbucket-backup/archive/master.zip 15 | ``` 16 | 17 | ## Quickstart 18 | ```bash 19 | bitbucket-backup [-u ] [-p ] [-k ] [-s ] 20 | [-l ] [-t ] [-a] [-v] [-q] [-c] [--http] [--skip-password] [--mirror] 21 | [--prune] [--fetchlfs] 22 | ``` 23 | Username/password are needed to access the Bitbucket API to get a repo listing. 24 | At this time it is not used to do the clone/update. 25 | Clone/update requires that your SSH keys have been uploaded to Bitbucket. 26 | 27 | You can backup a team's repositories instead of your own by supplying the optional `-t` parameter 28 | and entering the team slug (this is now called a "Workspace" by BitBucket). 29 | 30 | ## App passwords 31 | If would like, you can use app passwords instead of your password used to login to Bitbucket. 32 | The password must have read repositories permission. 33 | 34 | 35 | ## OAuth authentication 36 | 37 | You can use OAuth authentication if you do not want to use username/password. There are 4 steps to using OAuth. 38 | 39 | 1. Create OAuth consumer keys here: https://bitbucket.org/account/user/{username}/api 40 | 2. Make sure that your consumer has the following permissions: 41 | * Account: Read 42 | * Projects: Read 43 | * Repositories: Read 44 | 3. Specify the -k -s flags 45 | 4. Make sure requests-oauthlib is installed `pip install requests-oauthlib` 46 | 47 | 48 | ## Requirements 49 | 50 | You do need to have your SSH keys uploaded for the computer that you are running the backup on. 51 | 52 | ## Additional notes 53 | I am hosting this on GitHub because I believe it is superior for public repos (I understand the irony). 54 | -------------------------------------------------------------------------------- /backup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import datetime 4 | import os 5 | import shutil 6 | import subprocess 7 | import sys 8 | from getpass import getpass 9 | 10 | import requests 11 | from requests.auth import HTTPBasicAuth 12 | 13 | try: 14 | from urllib.parse import quote 15 | except ImportError: 16 | from urllib import quote 17 | 18 | try: 19 | input = raw_input 20 | except NameError: 21 | pass 22 | 23 | try: 24 | _range = xrange 25 | except NameError: 26 | _range = range 27 | 28 | _verbose = False 29 | _quiet = False 30 | 31 | 32 | class MaxBackupAttemptsReached(Exception): 33 | pass 34 | 35 | 36 | def debug(message, output_no_verbose=False): 37 | """ 38 | Outputs a message to stdout taking into account the options verbose/quiet. 39 | """ 40 | global _quiet, _verbose 41 | if not _quiet and (output_no_verbose or _verbose): 42 | print("%s - %s" % (datetime.datetime.now(), message)) 43 | 44 | 45 | def exit(message, code=1): 46 | """ 47 | Forces script termination using C based error codes. 48 | By default, it uses error 1 (EPERM - Operation not permitted) 49 | """ 50 | global _quiet 51 | if not _quiet and message and len(message) > 0: 52 | sys.stderr.write("%s (%s)\n" % (message, code)) 53 | sys.exit(code) 54 | 55 | 56 | def exec_cmd(command, stop_on_error=True): 57 | """ 58 | Executes an external command taking into account errors and logging. 59 | """ 60 | global _verbose 61 | debug("Executing command: %s" % command) 62 | if not _verbose: 63 | if "nt" == os.name: 64 | command = "%s > nul 2> nul" % command 65 | else: 66 | command = "%s > /dev/null 2>&1" % command 67 | resp = subprocess.call(command, shell=True) 68 | if resp != 0: 69 | if stop_on_error: 70 | exit("Command [%s] failed" % command, resp) 71 | else: 72 | debug("Command [%s] failed: %s" % (command, resp)) 73 | 74 | 75 | def compress(repo, location): 76 | """ 77 | Creates a TAR.GZ file with all contents cloned by this script. 78 | """ 79 | os.chdir(location) 80 | debug("Compressing repositories in [%s]..." % location, True) 81 | exec_cmd( 82 | "tar -zcvf bitbucket-backup-%s-%s.tar.gz `ls -d *`" 83 | % ( 84 | repo.get("owner").get("username") or repo.get("owner").get("nickname"), 85 | datetime.datetime.now().strftime("%Y%m%d%H%m%s"), 86 | ) 87 | ) 88 | debug("Cleaning up...", True) 89 | for d in os.listdir(location): 90 | path = os.path.join(location, d) 91 | if os.path.isdir(path): 92 | exec_cmd("rm -rfv %s" % path) 93 | 94 | 95 | def fetch_lfs_content(backup_dir): 96 | debug("Fetching LFS content...") 97 | os.chdir(backup_dir) 98 | command = "git lfs fetch --all" 99 | exec_cmd(command, stop_on_error=False) 100 | 101 | 102 | def get_repositories( 103 | username=None, password=None, oauth_key=None, oauth_secret=None, team=None 104 | ): 105 | auth = None 106 | repos = [] 107 | try: 108 | if all((oauth_key, oauth_secret)): 109 | from requests_oauthlib import OAuth1 110 | 111 | auth = OAuth1(oauth_key, oauth_secret) 112 | if all((username, password)): 113 | auth = HTTPBasicAuth(username, password) 114 | if auth is None: 115 | exit("Must provide username/password or oath credentials") 116 | if not team or username: 117 | response = requests.get("https://api.bitbucket.org/2.0/user/", auth=auth) 118 | username = response.json().get("username") 119 | url = "https://api.bitbucket.org/2.0/repositories/{}/".format(team or username) 120 | 121 | response = requests.get(url, auth=auth) 122 | response.raise_for_status() 123 | repos_data = response.json() 124 | for repo in repos_data.get("values"): 125 | repos.append(repo) 126 | while repos_data.get("next"): 127 | response = requests.get(repos_data.get("next"), auth=auth) 128 | repos_data = response.json() 129 | for repo in repos_data.get("values"): 130 | repos.append(repo) 131 | except requests.exceptions.RequestException as e: 132 | 133 | if e.response.status_code == 401: 134 | exit( 135 | "Unauthorized! Check your credentials and try again.", 22 136 | ) # EINVAL - Invalid argument 137 | else: 138 | exit( 139 | "Connection Error! Bitbucket returned HTTP error [%s]." 140 | % e.response.status_code 141 | ) 142 | return repos 143 | 144 | 145 | def clone_repo( 146 | repo, 147 | backup_dir, 148 | http, 149 | username, 150 | password, 151 | mirror=False, 152 | with_wiki=False, 153 | fetch_lfs=False, 154 | ): 155 | global _quiet, _verbose 156 | scm = repo.get("scm") 157 | slug = repo.get("slug") 158 | owner = repo.get("owner").get("username") or repo.get("owner").get("nickname") 159 | owner_url = quote(owner) 160 | if http and not all((username, password)): 161 | exit("Cannot backup via http without username and password" % scm) 162 | slug_url = quote(slug) 163 | command = None 164 | if scm == "hg": 165 | if http: 166 | command = "hg clone https://%s:%s@bitbucket.org/%s/%s" % ( 167 | quote(username), 168 | quote(password), 169 | owner_url, 170 | slug_url, 171 | ) 172 | else: 173 | command = "hg clone ssh://hg@bitbucket.org/%s/%s" % (owner_url, slug_url) 174 | if scm == "git": 175 | git_command = "git clone" 176 | if mirror: 177 | git_command = "git clone --mirror" 178 | if http: 179 | command = "%s https://%s:%s@bitbucket.org/%s/%s.git" % ( 180 | git_command, 181 | quote(username), 182 | quote(password), 183 | owner_url, 184 | slug_url, 185 | ) 186 | else: 187 | command = "%s git@bitbucket.org:%s/%s.git" % ( 188 | git_command, 189 | owner_url, 190 | slug_url, 191 | ) 192 | if not command: 193 | exit("could not build command (scm [%s] not recognized?)" % scm) 194 | debug("Cloning %s..." % repo.get("name")) 195 | exec_cmd('%s "%s"' % (command, backup_dir)) 196 | if scm == "git" and fetch_lfs: 197 | fetch_lfs_content(backup_dir) 198 | if with_wiki and repo.get("has_wiki"): 199 | debug("Cloning %s's Wiki..." % repo.get("name")) 200 | exec_cmd("%s/wiki %s_wiki" % (command, backup_dir)) 201 | 202 | 203 | def update_repo(repo, backup_dir, with_wiki=False, prune=False, fetch_lfs=False): 204 | scm = repo.get("scm") 205 | command = None 206 | os.chdir(backup_dir) 207 | if scm == "hg": 208 | command = "hg pull -u" 209 | if scm == "git": 210 | command = "git remote update" 211 | if prune: 212 | command = "%s %s" % (command, "--prune") 213 | if not command: 214 | exit("could not build command (scm [%s] not recognized?)" % scm) 215 | debug("Updating %s..." % repo.get("name")) 216 | exec_cmd(command) 217 | if scm == "git" and fetch_lfs: 218 | fetch_lfs_content(backup_dir) 219 | wiki_dir = "%s_wiki" % backup_dir 220 | if with_wiki and repo.get("has_wiki") and os.path.isdir(wiki_dir): 221 | os.chdir(wiki_dir) 222 | debug("Updating %s's Wiki..." % repo.get("name")) 223 | exec_cmd(command) 224 | 225 | 226 | def main(): 227 | parser = argparse.ArgumentParser(description="Usage: %prog [options] ") 228 | parser.add_argument("-u", "--username", dest="username", help="Bitbucket username") 229 | parser.add_argument("-p", "--password", dest="password", help="Bitbucket password") 230 | parser.add_argument( 231 | "-k", "--oauth-key", dest="oauth_key", help="Bitbucket oauth key" 232 | ) 233 | parser.add_argument( 234 | "-s", "--oauth-secret", dest="oauth_secret", help="Bitbucket oauth secret" 235 | ) 236 | parser.add_argument("-t", "--team", dest="team", help="Bitbucket team") 237 | parser.add_argument( 238 | "-l", "--location", dest="location", help="Local backup location" 239 | ) 240 | parser.add_argument( 241 | "-v", 242 | "--verbose", 243 | action="store_true", 244 | dest="verbose", 245 | help="Verbose output of all cloning commands", 246 | ) 247 | parser.add_argument( 248 | "-q", "--quiet", action="store_true", dest="quiet", help="No output to stdout" 249 | ) 250 | parser.add_argument( 251 | "-c", 252 | "--compress", 253 | action="store_true", 254 | dest="compress", 255 | help="Creates a compressed file with all cloned repositories (cleans up location directory)", 256 | ) 257 | parser.add_argument( 258 | "-a", 259 | "--attempts", 260 | dest="attempts", 261 | type=int, 262 | default=1, 263 | help="max. number of attempts to backup repository", 264 | ) 265 | parser.add_argument( 266 | "--mirror", 267 | action="store_true", 268 | help="Clone just bare repositories with git clone --mirror (git only)", 269 | ) 270 | parser.add_argument( 271 | "--fetchlfs", 272 | action="store_true", 273 | help="Fetch LFS content after clone/pull (git only)", 274 | ) 275 | parser.add_argument( 276 | "--with-wiki", dest="with_wiki", action="store_true", help="Includes wiki" 277 | ) 278 | parser.add_argument( 279 | "--http", action="store_true", help="Fetch via https instead of SSH" 280 | ) 281 | parser.add_argument( 282 | "--skip-password", 283 | dest="skip_password", 284 | action="store_true", 285 | help="Ignores password prompting if no password is provided (for public repositories)", 286 | ) 287 | parser.add_argument( 288 | "--prune", dest="prune", action="store_true", help="Prune repo on remote update" 289 | ) 290 | parser.add_argument( 291 | "--delete-extraneous", 292 | dest="delete_extraneous", 293 | action="store_true", 294 | help="Delete extraneous repositories from backup", 295 | ) 296 | parser.add_argument( 297 | "--ignore-repo-list", 298 | dest="ignore_repo_list", 299 | nargs="+", 300 | type=str, 301 | help="specify list of repo slug names to skip", 302 | ) 303 | parser.add_argument( 304 | "--only-repos", 305 | dest="repo_whitelist", 306 | nargs="+", 307 | type=str, 308 | help="specify list of repo slug names to download", 309 | ) 310 | args = parser.parse_args() 311 | location = args.location 312 | username = args.username 313 | password = args.password 314 | oauth_key = args.oauth_key 315 | oauth_secret = args.oauth_secret 316 | repo_whitelist = args.repo_whitelist 317 | http = args.http 318 | max_attempts = args.attempts 319 | global _quiet 320 | _quiet = args.quiet 321 | global _verbose 322 | _verbose = args.verbose 323 | _mirror = args.mirror 324 | _fetchlfs = args.fetchlfs 325 | _with_wiki = args.with_wiki 326 | _delete_extraneous = args.delete_extraneous 327 | if _quiet: 328 | _verbose = False # override in case both are selected 329 | team = args.team 330 | 331 | if not all((oauth_key, oauth_secret)): 332 | if not username: 333 | username = input("Enter bitbucket username: ") 334 | if not password: 335 | password = getpass(prompt="Enter your bitbucket password: ") 336 | if not location: 337 | location = input("Enter local location to backup to: ") 338 | location = os.path.abspath(location) 339 | 340 | # ok to proceed 341 | try: 342 | repos = get_repositories( 343 | username=username, 344 | password=password, 345 | oauth_key=oauth_key, 346 | oauth_secret=oauth_secret, 347 | team=team, 348 | ) 349 | repos = sorted(repos, key=lambda repo_: repo_.get("name")) 350 | dir_list = [] 351 | if not repos: 352 | print( 353 | "No repositories found. Are you sure you provided the correct password" 354 | ) 355 | for repo in repos: 356 | dir_list.append(repo.get("slug")) 357 | if repo.get("has_wiki"): 358 | dir_list.append(repo.get("slug") + "_wiki") 359 | 360 | if args.ignore_repo_list and repo.get("slug") in args.ignore_repo_list: 361 | debug( 362 | "ignoring repo %s with slug: %s" 363 | % (repo.get("name"), repo.get("slug")) 364 | ) 365 | continue 366 | 367 | if ( 368 | repo_whitelist 369 | and len(repo_whitelist) != 0 370 | and repo.get("slug") not in repo_whitelist 371 | ): 372 | debug( 373 | "ignoring repo %s with slug: %s" 374 | % (repo.get("name"), repo.get("slug")) 375 | ) 376 | continue 377 | 378 | debug("Backing up [%s]..." % repo.get("name"), True) 379 | backup_dir = os.path.join(location, repo.get("slug")) 380 | 381 | for attempt in range(1, max_attempts + 1): 382 | try: 383 | if not os.path.isdir(backup_dir): 384 | clone_repo( 385 | repo, 386 | backup_dir, 387 | http, 388 | username, 389 | password, 390 | mirror=_mirror, 391 | with_wiki=_with_wiki, 392 | fetch_lfs=_fetchlfs, 393 | ) 394 | else: 395 | debug( 396 | "Repository [%s] already in place, just updating..." 397 | % repo.get("name") 398 | ) 399 | update_repo( 400 | repo, 401 | backup_dir, 402 | with_wiki=_with_wiki, 403 | prune=args.prune, 404 | fetch_lfs=_fetchlfs, 405 | ) 406 | except: 407 | if attempt == max_attempts: 408 | raise MaxBackupAttemptsReached( 409 | "repo [%s] is reached maximum number [%d] of backup tries" 410 | % (repo.get("name"), attempt) 411 | ) 412 | debug( 413 | "Failed to backup repository [%s], keep trying, %d attempts remain" 414 | % (repo.get("name"), max_attempts - attempt) 415 | ) 416 | else: 417 | break 418 | 419 | for dir in os.listdir(location): 420 | if ( 421 | _delete_extraneous 422 | and os.path.isdir(os.path.join(location, dir)) 423 | and not dir in dir_list 424 | ): 425 | debug("Removing repository [%s]..." % dir, True) 426 | shutil.rmtree(os.path.join(location, dir)) 427 | 428 | if args.compress: 429 | compress(repo, location) 430 | debug("Finished!", True) 431 | except (KeyboardInterrupt, SystemExit): 432 | exit( 433 | "Operation cancelled. There might be inconsistent data in location directory.", 434 | 0, 435 | ) 436 | except MaxBackupAttemptsReached as e: 437 | exit("Unable to backup: %s" % e) 438 | except: 439 | if not _quiet: 440 | import traceback 441 | 442 | traceback.print_exc() 443 | exit("Unknown error.", 11) # EAGAIN - Try again 444 | 445 | 446 | if __name__ == "__main__": 447 | main() 448 | --------------------------------------------------------------------------------