├── .gitignore ├── .goreleaser.yml ├── .travis.yml ├── ghbackup ├── backup.go ├── backup_test.go ├── fetch.go ├── ghbackup.go ├── run.go └── run_test.go ├── license ├── main.go └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | dist 3 | .DS_Store -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - env: 3 | - CGO_ENABLED=0 4 | goos: 5 | - windows 6 | - darwin 7 | - linux 8 | goarch: 9 | - amd64 10 | - 386 11 | - arm 12 | - arm64 13 | ignore: 14 | - goos: darwin 15 | goarch: 386 16 | archives: 17 | - format_overrides: 18 | - goos: windows 19 | format: zip 20 | name_template: "{{.Binary}}_{{.Version}}_{{.Os}}_{{.Arch}}" 21 | replacements: 22 | amd64: 64bit 23 | 386: 32bit 24 | arm: ARM 25 | arm64: ARM64 26 | darwin: mac 27 | linux: linux 28 | windows: windows 29 | brews: 30 | - tap: 31 | owner: qvl 32 | name: homebrew-tap 33 | folder: Formula 34 | homepage: https://qvl.io/ghbackup 35 | description: Embarrassingly simple GitHub backup tool 36 | dependencies: 37 | - git 38 | nfpms: 39 | - homepage: https://qvl.io/ghbackup 40 | description: Embarrassingly simple GitHub backup tool 41 | maintainer: Jorin Vogel 42 | license: MIT 43 | vendor: qvl 44 | formats: 45 | - deb 46 | - rpm 47 | dependencies: 48 | - git 49 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go_import_path: qvl.io/ghbackup 3 | go: 4 | - 1.14.2 5 | os: 6 | - linux 7 | - osx 8 | script: 9 | - go test ./... 10 | - curl -sfL https://git.io/goreleaser | sh -s -- check 11 | deploy: 12 | - provider: script 13 | skip_cleanup: true 14 | script: curl -sL https://git.io/goreleaser | bash 15 | on: 16 | tags: true 17 | condition: $TRAVIS_OS_NAME = linux 18 | -------------------------------------------------------------------------------- /ghbackup/backup.go: -------------------------------------------------------------------------------- 1 | package ghbackup 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "os/exec" 8 | "path" 9 | "path/filepath" 10 | "strings" 11 | ) 12 | 13 | type repoState int 14 | 15 | const ( 16 | stateNew = iota 17 | stateChanged 18 | stateUnchanged 19 | stateFailed 20 | ) 21 | 22 | // Clone new repo or pull in existing repo. 23 | // Returns state of repo. 24 | func (c Config) backup(r repo) (repoState, error) { 25 | repoDir := getRepoDir(c.Dir, r.Path, c.Account) 26 | 27 | repoExists, err := exists(repoDir) 28 | if err != nil { 29 | return stateFailed, fmt.Errorf("cannot check if repo exists: %v", err) 30 | } 31 | 32 | var cmd *exec.Cmd 33 | if repoExists { 34 | c.Log.Printf("Updating %s", r.Path) 35 | cmd = exec.Command("git", "remote", "update") 36 | cmd.Dir = repoDir 37 | } else { 38 | c.Log.Printf("Cloning %s", r.Path) 39 | cmd = exec.Command("git", "clone", "--mirror", "--no-checkout", "--progress", getCloneURL(r, c.Secret), repoDir) 40 | } 41 | out, err := cmd.CombinedOutput() 42 | if err != nil { 43 | if !repoExists { 44 | // clean up clone dir after a failed clone 45 | // if it was a clean clone only 46 | _ = os.RemoveAll(repoDir) 47 | } 48 | return stateFailed, fmt.Errorf("error running command %v (%v): %v (%v)", maskSecrets(cmd.Args, []string{c.Secret}), cmd.Path, string(out), err) 49 | } 50 | return gitState(repoExists, string(out)), nil 51 | } 52 | 53 | // maskSecrets hides sensitive data 54 | func maskSecrets(values, secrets []string) []string { 55 | out := make([]string, len(values)) 56 | for vIndex, value := range values { 57 | out[vIndex] = value 58 | } 59 | for _, secret := range secrets { 60 | for vIndex, value := range out { 61 | out[vIndex] = strings.Replace(value, secret, "###", -1) 62 | } 63 | } 64 | return out 65 | } 66 | 67 | func getRepoDir(backupDir, repoPath, account string) string { 68 | repoGit := repoPath + ".git" 69 | // For single account, skip sub-directories 70 | if account != "" { 71 | return filepath.Join(backupDir, path.Base(repoGit)) 72 | } 73 | return filepath.Join(backupDir, repoGit) 74 | } 75 | 76 | // Check if a file or directory exists 77 | func exists(f string) (bool, error) { 78 | _, err := os.Stat(f) 79 | if err != nil { 80 | if os.IsNotExist(err) { 81 | return false, nil 82 | } 83 | return false, fmt.Errorf("cannot get stats for path `%s`: %v", f, err) 84 | } 85 | return true, nil 86 | } 87 | 88 | // Add secret token to URL of private repos. 89 | // Allows cloning without manual authentication or SSH setup. 90 | // However, this saves the secret in the git config file. 91 | func getCloneURL(r repo, secret string) string { 92 | if !r.Private { 93 | return r.URL 94 | } 95 | u, err := url.Parse(r.URL) 96 | if err != nil { 97 | return "" 98 | } 99 | u.User = url.User(secret) 100 | return u.String() 101 | } 102 | 103 | // Get the state of a repo from command output. 104 | func gitState(repoExisted bool, out string) repoState { 105 | if !repoExisted { 106 | return stateNew 107 | } 108 | if lines := strings.Split(out, "\n"); len(lines) > 2 { 109 | return stateChanged 110 | } 111 | return stateUnchanged 112 | } 113 | -------------------------------------------------------------------------------- /ghbackup/backup_test.go: -------------------------------------------------------------------------------- 1 | package ghbackup 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func Test_maskSecrets(t *testing.T) { 9 | type args struct { 10 | values []string 11 | secrets []string 12 | } 13 | tests := []struct { 14 | name string 15 | args args 16 | want []string 17 | }{ 18 | { 19 | name: "identicall", 20 | args: args{ 21 | values: []string{"ok", "haha secrethaha", "sdjdsajsasecretsdsasecret,tmp"}, 22 | secrets: []string{"ok", "haha secrethaha", "sdjdsajsasecretsdsasecret,tmp"}, 23 | }, 24 | want: []string{"###", "###", "###"}, 25 | }, 26 | { 27 | name: "generic", 28 | args: args{ 29 | values: []string{"ok", "haha secrethaha", "sdjdsajsasecretsdsasecret,tmp"}, 30 | secrets: []string{"secret"}, 31 | }, 32 | want: []string{"ok", "haha ###haha", "sdjdsajsa###sdsa###,tmp"}, 33 | }, 34 | } 35 | for _, tt := range tests { 36 | t.Run(tt.name, func(t *testing.T) { 37 | if got := maskSecrets(tt.args.values, tt.args.secrets); !reflect.DeepEqual(got, tt.want) { 38 | t.Errorf("maskSecrets() = %v, want %v", got, tt.want) 39 | } 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /ghbackup/fetch.go: -------------------------------------------------------------------------------- 1 | package ghbackup 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "path" 8 | "strings" 9 | ) 10 | 11 | // Get repositories from Github. 12 | // Follow all "next" links. 13 | func fetch(account, secret, api string, doer Doer) ([]repo, error) { 14 | var allRepos []repo 15 | 16 | currentURL, err := getURL(account, secret, api, doer) 17 | if err != nil { 18 | return allRepos, err 19 | } 20 | 21 | // Go through all pages 22 | for { 23 | req, err := http.NewRequest("GET", currentURL, nil) 24 | if err != nil { 25 | return nil, fmt.Errorf("cannot create request: %v", err) 26 | } 27 | if secret != "" { 28 | // For token authentication `account` will be ignored 29 | req.SetBasicAuth(account, secret) 30 | } 31 | res, err := doer.Do(req) 32 | if err != nil { 33 | return nil, fmt.Errorf("cannot get repos: %v", err) 34 | } 35 | defer func() { 36 | _ = res.Body.Close() 37 | }() 38 | if res.StatusCode >= 300 { 39 | return nil, fmt.Errorf("bad response from %s: %v", res.Request.URL, res.Status) 40 | } 41 | 42 | var repos []repo 43 | err = json.NewDecoder(res.Body).Decode(&repos) 44 | if err != nil { 45 | return nil, fmt.Errorf("cannot decode JSON response: %v", err) 46 | } 47 | 48 | allRepos = append(allRepos, selectRepos(repos, account)...) 49 | 50 | // Set url for next iteration 51 | currentURL = getNextURL(res.Header) 52 | 53 | // Done if no next URL 54 | if currentURL == "" { 55 | return allRepos, nil 56 | } 57 | } 58 | } 59 | 60 | func getURL(account, secret, api string, doer Doer) (string, error) { 61 | user := "user" 62 | if secret == "" { 63 | category, err := getCategory(account, api, doer) 64 | if err != nil { 65 | return "", err 66 | } 67 | user = category + "/" + account 68 | } 69 | url := api + "/" + user + "/repos?per_page=100" 70 | return url, nil 71 | } 72 | 73 | // Returns "users" or "orgs" depending on type of account 74 | func getCategory(account, api string, doer Doer) (string, error) { 75 | req, err := http.NewRequest("GET", strings.Join([]string{api, "users", account}, "/"), nil) 76 | if err != nil { 77 | return "", fmt.Errorf("cannot create HTTP request: %v", err) 78 | } 79 | res, err := doer.Do(req) 80 | if err != nil { 81 | return "", fmt.Errorf("cannot get user info: %v", err) 82 | } 83 | defer func() { 84 | _ = res.Body.Close() 85 | }() 86 | if res.StatusCode >= 300 { 87 | return "", fmt.Errorf("bad response from %s: %v", res.Request.URL, res.Status) 88 | } 89 | 90 | var a struct { 91 | Type string 92 | } 93 | err = json.NewDecoder(res.Body).Decode(&a) 94 | if err != nil { 95 | return "", fmt.Errorf("cannot decode JSON response: %v", err) 96 | } 97 | 98 | if a.Type == "User" { 99 | return "users", nil 100 | } 101 | if a.Type == "Organization" { 102 | return "orgs", nil 103 | } 104 | return "", fmt.Errorf("unknown type of account %s for %s", a.Type, account) 105 | } 106 | 107 | func selectRepos(repos []repo, account string) []repo { 108 | if account == "" { 109 | return repos 110 | } 111 | var res []repo 112 | for _, r := range repos { 113 | if path.Dir(r.Path) == account { 114 | res = append(res, r) 115 | } 116 | } 117 | return res 118 | } 119 | 120 | func getNextURL(header http.Header) string { 121 | linkHeader := header["Link"] 122 | if len(linkHeader) == 0 { 123 | return "" 124 | } 125 | parts := strings.Split(linkHeader[0], ",") 126 | if len(parts) == 0 { 127 | return "" 128 | } 129 | firstLink := parts[0] 130 | if !strings.Contains(firstLink, "rel=\"next\"") { 131 | return "" 132 | } 133 | parts = strings.Split(firstLink, ";") 134 | if len(parts) == 0 { 135 | return "" 136 | } 137 | urlInBrackets := parts[0] 138 | if len(urlInBrackets) < 3 { 139 | return "" 140 | } 141 | return urlInBrackets[1 : len(urlInBrackets)-1] 142 | } 143 | -------------------------------------------------------------------------------- /ghbackup/ghbackup.go: -------------------------------------------------------------------------------- 1 | // Package ghbackup provides access to run all the functionality of ghbackup. 2 | // The binary is just a wrapper around the Run function of package ghbackup. 3 | // This way you can directly use it from any other Go program. 4 | package ghbackup 5 | 6 | import ( 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | // Config should be passed to Run. 12 | // Only Account, Dir, Updates are required. 13 | type Config struct { 14 | Account string 15 | Dir string 16 | // Optional: 17 | Err *log.Logger 18 | Log *log.Logger 19 | Secret string 20 | API string 21 | Workers int 22 | Doer 23 | } 24 | 25 | // Doer makes HTTP requests. 26 | // http.HTTPClient implements Doer but simpler implementations can be used too. 27 | type Doer interface { 28 | Do(*http.Request) (*http.Response, error) 29 | } 30 | 31 | type repo struct { 32 | Path string `json:"full_name"` 33 | URL string `json:"clone_url"` 34 | Private bool `json:"private"` 35 | } 36 | 37 | const defaultMaxWorkers = 10 38 | const defaultAPI = "https://api.github.com" 39 | -------------------------------------------------------------------------------- /ghbackup/run.go: -------------------------------------------------------------------------------- 1 | package ghbackup 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | // Run update for the given Config. 12 | func Run(config Config) error { 13 | // Defaults 14 | if config.Log == nil { 15 | config.Log = log.New(ioutil.Discard, "", 0) 16 | } 17 | if config.Err == nil { 18 | config.Err = log.New(ioutil.Discard, "", 0) 19 | } 20 | if config.Workers == 0 { 21 | config.Workers = defaultMaxWorkers 22 | } 23 | if config.API == "" { 24 | config.API = defaultAPI 25 | } 26 | if config.Doer == nil { 27 | config.Doer = http.DefaultClient 28 | } 29 | 30 | // Fetch list of repositories 31 | repos, err := fetch(config.Account, config.Secret, config.API, config.Doer) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | config.Log.Printf("%d repositories:", len(repos)) 37 | 38 | results := make(chan repoState) 39 | 40 | // Backup repositories in parallel with retries 41 | go each(repos, config.Workers, func(r repo) { 42 | state, err := config.backup(r) 43 | for _, sleepDuration := range []time.Duration{5, 15, 45, 90, 180, -1} { 44 | if err != nil { 45 | if sleepDuration == -1 { 46 | config.Log.Printf("repository %v failed to get cloned: %v", r, err) 47 | break 48 | } 49 | config.Err.Println(err) 50 | time.Sleep(sleepDuration * time.Second) 51 | state, err = config.backup(r) 52 | continue 53 | } 54 | break 55 | } 56 | results <- state 57 | }) 58 | 59 | var creations, updates, unchanged, failed int 60 | 61 | for i := 0; i < len(repos); i++ { 62 | state := <-results 63 | if state == stateNew { 64 | creations++ 65 | } else if state == stateChanged { 66 | updates++ 67 | } else if state == stateUnchanged { 68 | unchanged++ 69 | } else { 70 | failed++ 71 | } 72 | } 73 | close(results) 74 | 75 | config.Log.Printf( 76 | "done: %d new, %d updated, %d unchanged", 77 | creations, 78 | updates, 79 | unchanged, 80 | ) 81 | if failed > 0 { 82 | return fmt.Errorf("failed to get %d repositories", failed) 83 | } 84 | return nil 85 | } 86 | 87 | func each(repos []repo, workers int, worker func(repo)) { 88 | if len(repos) < workers { 89 | workers = len(repos) 90 | } 91 | 92 | jobs := make(chan repo) 93 | 94 | for w := 0; w < workers; w++ { 95 | go func() { 96 | for r := range jobs { 97 | worker(r) 98 | } 99 | }() 100 | } 101 | 102 | for _, r := range repos { 103 | jobs <- r 104 | } 105 | close(jobs) 106 | } 107 | -------------------------------------------------------------------------------- /ghbackup/run_test.go: -------------------------------------------------------------------------------- 1 | package ghbackup_test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | "strings" 12 | "testing" 13 | 14 | "qvl.io/ghbackup/ghbackup" 15 | ) 16 | 17 | const ( 18 | expectedRepos = " ghbackup homebrew-tap promplot qvl.io slangbrain.com sleepto " 19 | gitFiles = " HEAD branches config description hooks info objects packed-refs refs " 20 | ) 21 | 22 | func TestRun(t *testing.T) { 23 | dir, err := ioutil.TempDir("", "qvl-backup") 24 | if err != nil { 25 | t.Error(err) 26 | } 27 | defer func() { 28 | if err := os.RemoveAll(dir); err != nil { 29 | t.Error(err) 30 | } 31 | }() 32 | 33 | var logs, errs bytes.Buffer 34 | err = ghbackup.Run(ghbackup.Config{ 35 | Account: "qvl", 36 | Dir: dir, 37 | Secret: os.Getenv("SECRET"), 38 | Log: log.New(&logs, "", 0), 39 | Err: log.New(&errs, "", log.Lshortfile), 40 | }) 41 | 42 | if errs.Len() != 0 { 43 | t.Error("Unexpected error messages:", errs.String()) 44 | } 45 | if err != nil { 46 | t.Error("Unexpected error:", err) 47 | } 48 | 49 | // Check log output to be of the following form: 50 | // 6 repositories: 51 | // Cloning qvl/ghbackup 52 | // Cloning qvl/slangbrain.com 53 | // Cloning qvl/qvl.io 54 | // Cloning qvl/sleepto 55 | // Cloning qvl/promplot 56 | // Cloning qvl/homebrew-tap 57 | // done: 6 new, 0 updated, 0 unchanged 58 | lines := strings.Split(logs.String(), "\n") 59 | countFirstLine, err := strconv.Atoi(strings.Split(lines[0], " ")[0]) 60 | if err != nil { 61 | t.Errorf("Cannot parse repository count from first line of output: '%s'", lines[0]) 62 | } 63 | if lines[countFirstLine+1] != fmt.Sprintf("done: %d new, 0 updated, 0 unchanged", countFirstLine) { 64 | t.Errorf("Last line contains unexpected status information: '%s'", lines[countFirstLine+1]) 65 | } 66 | 67 | // Check contents of backup directory 68 | files, err := ioutil.ReadDir(dir) 69 | if err != nil { 70 | t.Error(err) 71 | } 72 | minRepos := len(strings.Split(strings.TrimSpace(expectedRepos), " ")) 73 | if len(files) < minRepos { 74 | t.Errorf("Expected to fetch at least %d repositories; got %d", minRepos, len(files)) 75 | } 76 | 77 | for _, f := range files { 78 | if !f.IsDir() { 79 | t.Errorf("Expected %s to be a directory", f.Name()) 80 | } 81 | strings.Contains(expectedRepos, " "+f.Name()+".git ") 82 | repoFiles, err := ioutil.ReadDir(filepath.Join(dir, f.Name())) 83 | if err != nil { 84 | t.Error(err) 85 | } 86 | 87 | if len(repoFiles) < 8 { 88 | t.Errorf("Expected repository %s to contain at least 8 files; found %d", f.Name(), len(repoFiles)) 89 | } 90 | for _, r := range repoFiles { 91 | if !strings.Contains(gitFiles, " "+r.Name()+" ") { 92 | t.Errorf("Expected repo %s to contain only files '%s'; found '%s'", f.Name(), gitFiles, r.Name()) 93 | } 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Jorin Vogel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Package main is the entry point for the ghbackup binary. 2 | // Here is where you can find argument parsing, usage information and the actual execution. 3 | package main 4 | 5 | import ( 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "runtime" 12 | 13 | "qvl.io/ghbackup/ghbackup" 14 | ) 15 | 16 | // Can be set in build step using -ldflags 17 | var version string 18 | 19 | const ( 20 | // Printed for -help, -h or with wrong number of arguments 21 | usage = `Embarrassing simple GitHub backup tool 22 | 23 | Usage: %s [flags] directory 24 | 25 | directory path to save the repositories to 26 | 27 | At least one of -account or -secret must be specified. 28 | 29 | Flags: 30 | ` 31 | more = "\nFor more visit https://qvl.io/ghbackup." 32 | accountUsage = `GitHub user or organization name to get repositories from. 33 | If not specified, all repositories the authenticated user has access to will be loaded.` 34 | secretUsage = `Authentication secret for GitHub API. 35 | Can use the users password or a personal access token (https://github.com/settings/tokens). 36 | Authentication increases rate limiting (https://developer.github.com/v3/#rate-limiting) and enables backup of private repositories.` 37 | ) 38 | 39 | // Get command line arguments and start updating repositories 40 | func main() { 41 | // Flags 42 | account := flag.String("account", "", accountUsage) 43 | secret := flag.String("secret", "", secretUsage) 44 | versionFlag := flag.Bool("version", false, "Print binary version") 45 | silent := flag.Bool("silent", false, "Suppress all output") 46 | 47 | // Parse args 48 | flag.Usage = func() { 49 | fmt.Fprintf(os.Stderr, usage, os.Args[0]) 50 | flag.PrintDefaults() 51 | fmt.Fprintln(os.Stderr, more) 52 | } 53 | flag.Parse() 54 | 55 | if *versionFlag { 56 | fmt.Printf("ghbackup %s %s %s\n", version, runtime.GOOS, runtime.GOARCH) 57 | os.Exit(0) 58 | } 59 | 60 | args := flag.Args() 61 | if len(args) != 1 || (*account == "" && *secret == "") { 62 | flag.Usage() 63 | os.Exit(1) 64 | } 65 | 66 | logger := log.New(os.Stdout, "", 0) 67 | if *silent { 68 | logger = log.New(ioutil.Discard, "", 0) 69 | } 70 | 71 | err := ghbackup.Run(ghbackup.Config{ 72 | Account: *account, 73 | Dir: args[0], 74 | Secret: *secret, 75 | Log: logger, 76 | Err: log.New(os.Stderr, "", 0), 77 | }) 78 | 79 | if err != nil { 80 | fmt.Fprintln(os.Stderr, err) 81 | os.Exit(1) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # :floppy_disk: ghbackup 2 | 3 | [![GoDoc](https://godoc.org/qvl.io/ghbackup?status.svg)](https://godoc.org/qvl.io/ghbackup) 4 | [![Build Status](https://travis-ci.org/qvl/ghbackup.svg?branch=master)](https://travis-ci.org/qvl/ghbackup) 5 | [![Go Report Card](https://goreportcard.com/badge/qvl.io/ghbackup)](https://goreportcard.com/report/qvl.io/ghbackup) 6 | 7 | 8 | Backup your GitHub repositories with a simple command-line application written in Go. 9 | 10 | The simplest way to keep your repositories save: 11 | 12 | 1. [Install](#install) `ghbackup` 13 | 1. Get a token from https://github.com/settings/tokens 14 | 2. `ghbackup -secret token /path/to/backup/dir` 15 | 16 | This will backup all repositories you have access to. 17 | 18 | ----------------------------------- 19 | 20 | 21 | 22 | Embarrassing simple GitHub backup tool 23 | 24 | Usage: ghbackup [flags] directory 25 | 26 | directory path to save the repositories to 27 | 28 | At least one of -account or -secret must be specified. 29 | 30 | Flags: 31 | -account string 32 | GitHub user or organization name to get repositories from. 33 | If not specified, all repositories the authenticated user has access to 34 | will be loaded. 35 | -secret string 36 | Authentication secret for GitHub API. 37 | Can use the users password or a personal access token (https://github.c 38 | om/settings/tokens). 39 | Authentication increases rate limiting (https://developer.github.com/v3 40 | /#rate-limiting) and enables backup of private repositories. 41 | -silent 42 | Suppress all output 43 | -version 44 | Print binary version 45 | 46 | For more visit https://qvl.io/ghbackup. 47 | 48 | 49 | ## Install 50 | 51 | - Note that `ghbackup` uses `git` under the hood. Please make sure it is installed on your system. 52 | 53 | - With [Go](https://golang.org/): 54 | ``` 55 | go get qvl.io/ghbackup 56 | ``` 57 | 58 | - With [Homebrew](http://brew.sh/): 59 | ``` 60 | brew install qvl/tap/ghbackup 61 | ``` 62 | 63 | - Download binary: https://github.com/qvl/ghbackup/releases 64 | 65 | 66 | ## Automation 67 | 68 | Mostly, we like to setup backups to run automatically in an interval. 69 | 70 | Let's setup `ghbackup` on a Linux server and make it run daily at 1am. This works similar on other platforms. 71 | There are different tools to do this: 72 | 73 | 74 | ### systemd and sleepto 75 | 76 | *Also see [this tutorial](https://jorin.me/automating-github-backup-with-ghbackup/).* 77 | 78 | [systemd](https://freedesktop.org/wiki/Software/systemd/) runs on most Linux systems and using [sleepto](https://qvl.io/sleepto) it's easy to create a service to schedule a backup. 79 | 80 | - Create a new unit file: 81 | ``` sh 82 | sudo touch /etc/systemd/system/ghbackup.service && sudo chmod 644 $_ 83 | ``` 84 | 85 | - Edit file: 86 | ``` 87 | [Unit] 88 | Description=GitHub backup 89 | After=network.target 90 | 91 | [Service] 92 | User=jorin 93 | ExecStart=/PATH/TO/sleepto -hour 1 /PATH/TO/ghbackup -account qvl /home/USER/github 94 | Restart=always 95 | 96 | [Install] 97 | WantedBy=multi-user.target 98 | ``` 99 | 100 | - Replace the paths with your options. 101 | 102 | - Start service and enable it on boot: 103 | ``` sh 104 | sudo systemctl daemon-reload 105 | sudo systemctl enable --now ghbackup 106 | ``` 107 | 108 | - Check if service is running: 109 | ``` sh 110 | systemctl status ghbackup 111 | ``` 112 | 113 | 114 | ### Cron 115 | 116 | Cron is a job scheduler that already runs on most Unix systems. 117 | 118 | - Run `crontab -e` 119 | - Add a new line and replace `NAME` and `DIR` with your options: 120 | 121 | ``` sh 122 | 0 1 * * * ghbackup -account NAME DIR 123 | ``` 124 | 125 | For example: 126 | 127 | ``` sh 128 | 0 1 * * * ghbackup -account qvl /home/qvl/backup-qvl 129 | ``` 130 | 131 | ### Sending statistics 132 | 133 | The last line of the output contains a summary. 134 | You can use this to collect statistics about your backups. 135 | An easy way would be to use a [Slack hook](https://api.slack.com/incoming-webhooks) and send it like this: 136 | 137 | ```sh 138 | ghbackup -secret $GITHUB_TOKEN $DIR \ 139 | | tail -n1 \ 140 | | xargs -I%% curl -s -X POST --data-urlencode 'payload={"text": "%%"}' $SLACK_HOOK 141 | ``` 142 | 143 | 144 | ## What happens? 145 | 146 | Get all repositories of a GitHub account. 147 | Save them to a folder. 148 | Update already cloned repositories. 149 | 150 | Best served as a scheduled job to keep your backups up to date! 151 | 152 | 153 | ## Limits 154 | 155 | `ghbackup` is about repositories. 156 | There are other solutions if you like to backup issues and wikis. 157 | 158 | 159 | ## Use as Go package 160 | 161 | From another Go program you can directly use the `ghbackup` sub-package. 162 | Have a look at the [GoDoc](https://godoc.org/qvl.io/ghbackup/ghbackup). 163 | 164 | 165 | ## Development 166 | 167 | Make sure to use `gofmt` and create a [Pull Request](https://github.com/qvl/ghbackup/pulls). 168 | 169 | ### Releasing 170 | 171 | Push a new Git tag and [GoReleaser](https://github.com/goreleaser/releaser) will automatically create a release. 172 | 173 | 174 | ## License 175 | 176 | [MIT](./license) 177 | --------------------------------------------------------------------------------