├── .editorconfig ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── bin └── github-backup.js ├── lib ├── backup.js ├── github.js └── promisify.js └── package.json /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.js] 2 | indent_style = tab 3 | 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | 4 | ## 0.2.0 - 2016-11-27 5 | - Add command line arguments for dry runs, and including different types of 6 | repos. 7 | 8 | ## 0.1.0 - 2014-09-08 9 | ### Added 10 | - Now backs up all starred repositories. 11 | 12 | ### Fixed 13 | - Project name in README.md 14 | 15 | 16 | ## 0.0.2 - 2014-09-07 17 | ### Fixed 18 | - Project name in README.md 19 | 20 | ## 0.0.1 - 2014-09-07 21 | ### Added 22 | - Initial release. 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Eric Lathrop 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # github-backup 2 | 3 | Backup GitHub repositories locally. This tool mirrors all public repositories of a GitHub user and all of the organizations that user is publicly a member of. It also mirrors any repositories the user has starred. If the repositories already exist on disk, they are updated. This script is meant to be run as a cron job. 4 | 5 | The program uses the [GitHub API](https://developer.github.com/) to discover repositories, and by default it accesses it unauthenticated, [which subjects it to lower rate limits](https://developer.github.com/v3/#rate-limiting). For most people running this every few hours won't be a problem. If you start getting 403 Forbidden errors, you can create a [personal access token](https://github.com/settings/applications) and store it in the GITHUB_ACCESS_TOKEN environment variable to get a higher rate limit. 6 | 7 | # Installation 8 | 9 | 1. Install [Node.js](http://nodejs.org/) 10 | 2. Run `npm install -g github-backup` 11 | 12 | # Usage 13 | 14 | ``` 15 | github-backup [-h] [-v] [--include INCLUDE] [--dry-run] username path 16 | ``` 17 | 18 | # To-do 19 | 20 | 1. Add ability to specify single individual repository as in "user/repository" or "organization/repository". 21 | 2. Discover and backup private repositories. 22 | -------------------------------------------------------------------------------- /bin/github-backup.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | "use strict"; 3 | 4 | var backup = require("../lib/backup"); 5 | var ArgumentParser = require('argparse').ArgumentParser; 6 | var pkg = require('../package.json'); 7 | 8 | function main(argv) { 9 | 10 | var parser = new ArgumentParser({ 11 | version: pkg.version, 12 | addHelp: true, 13 | description: pkg.description 14 | }); 15 | 16 | parser.addArgument("username", { 17 | help: "The github username to backup." 18 | }); 19 | parser.addArgument("path", { 20 | help: "The path where to backup the repositories to." 21 | }); 22 | 23 | parser.addArgument(['--include', '-i'], { 24 | defaultValue: 'user,org,starred', 25 | type: function(i) { return i.split(","); }, 26 | help: "Choose which repositories to backup. You can " + 27 | "select \"user\", \"org\" and \"starred\", and any combination" + 28 | " separated by a comma" 29 | }); 30 | 31 | parser.addArgument(['--dry-run', '-n'], { 32 | action: "storeTrue", 33 | help: "If set, no action is actually performed." 34 | }); 35 | 36 | var args = parser.parseArgs(); 37 | 38 | backup.publicUserRepos(args).catch(function(err) { 39 | console.error("Unhandled error:", err); 40 | }).done(); 41 | } 42 | 43 | main(); 44 | -------------------------------------------------------------------------------- /lib/backup.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var Promise = require("bluebird"); // jshint ignore:line 4 | 5 | var promisify = require("../lib/promisify"); 6 | var path = require("path"); 7 | var fs = Promise.promisifyAll(require("fs")); 8 | function backupRepo(url, destinationDir, dry_run) { 9 | var re = new RegExp("https://github\\.com/([^/]+)/([^/]+)"); 10 | var matches = url.match(re); 11 | var user = matches[1]; 12 | var repoName = matches[2]; 13 | var userPath = path.join(destinationDir, user); 14 | var repoPath = path.join(userPath, repoName); 15 | 16 | return fs.statAsync(userPath).error(function() { 17 | return fs.mkdirAsync(userPath); 18 | }).then(function() { 19 | return fs.statAsync(repoPath); 20 | }).then(function() { 21 | console.log("Updating", url); 22 | if(dry_run) return Promise.resolve(); 23 | return promisify.exec("git", ["remote", "update"], { cwd: repoPath }); 24 | }, function() { 25 | console.log("Cloning", url); 26 | if(dry_run) return Promise.resolve(); 27 | return promisify.exec("git", ["clone", "--mirror", url, repoPath]); 28 | }); 29 | } 30 | 31 | function backupRepoSerialized(url, destinationDir, promise, dry_run) { 32 | if (promise) { 33 | return promise.then(function() { 34 | return backupRepo(url, destinationDir, dry_run); 35 | }); 36 | } else { 37 | return backupRepo(url, destinationDir, dry_run); 38 | } 39 | } 40 | 41 | var github = require("../lib/github"); 42 | function publicUserRepos(args) { 43 | var username = args.username; 44 | var destinationDir = args.path; 45 | return github.getPublicUserRepos(username, args.include).then(function(repos) { 46 | var promise; 47 | for (var i = 0; i < repos.length; i++) { 48 | var url = repos[i].clone_url; // jshint ignore:line 49 | promise = backupRepoSerialized(url, destinationDir, promise, args.dry_run); 50 | } 51 | return promise; 52 | }); 53 | } 54 | 55 | module.exports = { 56 | publicUserRepos: publicUserRepos 57 | }; 58 | -------------------------------------------------------------------------------- /lib/github.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var Promise = require("bluebird"); // jshint ignore:line 4 | 5 | /* 6 | * A GitHub Access Token for higher API rate limits. 7 | * You can generate on on https://github.com/settings/applications#personal-access-tokens 8 | */ 9 | var accessToken = process.env.GITHUB_ACCESS_TOKEN; 10 | 11 | var promisify = require("./promisify"); 12 | 13 | var url = require("url"); 14 | function get(path) { 15 | var options = url.parse("https://api.github.com" + path); 16 | options.headers = { 17 | "User-Agent": "github-backup", 18 | "Accept": "application/vnd.github.v3+json" 19 | }; 20 | if (accessToken) { 21 | options.headers["Authorization"] = "token " + accessToken; 22 | } 23 | 24 | return promisify.httpsGet(options).then(promisify.readableStream).then(JSON.parse); 25 | } 26 | 27 | function getPublicUserRepos(user, include) { 28 | var pr = []; 29 | 30 | if (include.indexOf("user") !== -1) { 31 | pr.push(get("/users/" + user + "/repos")); 32 | } 33 | 34 | if (include.indexOf("org") !== -1) { 35 | pr.push(get("/users/" + user + "/orgs").map(getOrgRepos).reduce(concatArrays)); 36 | } 37 | 38 | if (include.indexOf("starred") !== -1) { 39 | pr.push(get("/users/" + user + "/starred")); 40 | } 41 | 42 | return Promise.all(pr).then(function(res) { 43 | return [].concat.apply([], res); 44 | }); 45 | } 46 | 47 | function getOrgRepos(org) { 48 | return get("/orgs/" + org.login + "/repos"); 49 | } 50 | 51 | function concatArrays(accum, item) { 52 | return accum.concat.apply(accum, item); 53 | } 54 | 55 | module.exports = { 56 | getPublicUserRepos: getPublicUserRepos 57 | }; 58 | -------------------------------------------------------------------------------- /lib/promisify.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var Promise = require("bluebird"); // jshint ignore:line 4 | 5 | var http = require("http"); 6 | var https = require("https"); 7 | function httpsGet(options) { 8 | return new Promise(function(resolve, reject) { 9 | https.get(options, function(res) { 10 | if (res.statusCode >= 400) { 11 | readableStream(res).then(function(data) { 12 | reject(res.statusCode + " " + http.STATUS_CODES[res.statusCode] + "\n" + data); 13 | }, function(err) { 14 | reject(err); 15 | }); 16 | return; 17 | } 18 | resolve(res); 19 | }).on("error", function(err) { 20 | reject(err); 21 | }); 22 | }); 23 | } 24 | 25 | function readableStream(stream) { 26 | return new Promise(function(resolve, reject) { 27 | var data = ""; 28 | stream.on("data", function(d) { 29 | data += d; 30 | }).on("end", function() { 31 | resolve(data); 32 | }).on("error", function(err) { 33 | reject(err); 34 | }); 35 | }); 36 | } 37 | 38 | var childProcess = require("child_process"); 39 | function exec(command, args, options) { 40 | return new Promise(function(resolve, reject) { 41 | options = options || {}; 42 | options.stdio = ["ignore", process.stdout, process.stderr]; 43 | childProcess.spawn(command, args, options).on("close", function(code) { 44 | if (code === 0) { 45 | resolve(); 46 | } else { 47 | reject("Process exited with code " + code); 48 | } 49 | }); 50 | }); 51 | } 52 | 53 | module.exports = { 54 | httpsGet: httpsGet, 55 | readableStream: readableStream, 56 | exec: exec 57 | }; 58 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "github-backup", 3 | "version": "0.2.0", 4 | "description": "Backup GitHub repositories locally", 5 | "main": "bin/github-backup.js", 6 | "bin": "bin/github-backup.js", 7 | "preferGlobal": true, 8 | "scripts": { 9 | "lint-js": "bash -c \"jshint bin/*.js lib/*.js\"" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git://github.com/ericlathrop/github-backup.git" 14 | }, 15 | "keywords": [ 16 | "git", 17 | "backup", 18 | "github" 19 | ], 20 | "author": "Eric Lathrop (http://ericlathrop.com/)", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/ericlathrop/github-backup/issues" 24 | }, 25 | "homepage": "https://github.com/ericlathrop/github-backup", 26 | "dependencies": { 27 | "argparse": "^1.0.7", 28 | "bluebird": "^2.3.2" 29 | }, 30 | "devDependencies": { 31 | "jshint": "^2.5.5" 32 | }, 33 | "jshintConfig": { 34 | "camelcase": true, 35 | "curly": true, 36 | "eqeqeq": true, 37 | "forin": true, 38 | "freeze": true, 39 | "immed": true, 40 | "indent": 4, 41 | "latedef": "nofunc", 42 | "newcap": true, 43 | "noarg": true, 44 | "node": true, 45 | "noempty": true, 46 | "nonbsp": true, 47 | "nonew": true, 48 | "quotmark": "double", 49 | "unused": "strict", 50 | "trailing": true, 51 | "browser": true, 52 | "devel": true, 53 | "globalstrict": true, 54 | "globals": { 55 | "module": false, 56 | "require": false 57 | } 58 | } 59 | } 60 | --------------------------------------------------------------------------------