├── backup_github_repositories.sublime-project ├── .gitignore ├── .gitattributes ├── LICENSE ├── README.md └── backup_github_repositories.ps1 /backup_github_repositories.sublime-project: -------------------------------------------------------------------------------- 1 | { 2 | "folders": 3 | [ 4 | { 5 | "path": "." 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything inside this directory. 2 | * 3 | 4 | # But not this file. 5 | !.gitignore 6 | 7 | # And not the project files. 8 | !.gitattributes 9 | !backup_github_repositories.ps1 10 | !backup_github_repositories.sublime-project 11 | !README.md 12 | !LICENSE 13 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # This overrides the core.autocrlf setting - http://git-scm.com/docs/gitattributes 2 | # Set default behaviour, in case users don't have core.autocrlf set. 3 | * text=auto 4 | 5 | # Declare files that will always have LF line endings on checkout. 6 | .gitignore text eol=lf 7 | .gitattributes text eol=lf 8 | .gitmodules text eol=lf 9 | *.md text eol=lf 10 | *.ps1 text eol=lf 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Finn Kumkar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Backup all git repositories from GitHub 2 | A PowerShell script that automatically backups all GitHub repositories of a user or an organisation to a local directory as a bare Git repository. 3 | 4 | ## Installation 5 | Download and unpack the [latest release](https://github.com/countzero/backup_github_repositories/releases/latest) to your machine. 6 | 7 | ## Usage 8 | Open a PowerShell console at the location of the unpacked release and execute the [./backup_github_repositories.ps1](https://github.com/countzero/backup_github_repositories/blob/master/backup_github_repositories.ps1). 9 | 10 | ## Examples 11 | 12 | ### Backup all git repositories of a user 13 | Execute the following to backup all git repositories of a GitHub user into the subdirectory `./YYYY-MM-DD/`. 14 | ```PowerShell 15 | .\backup_github_repositories.ps1 -userName "user" -userSecret "token" 16 | ``` 17 | 18 | ### Backup all git repositores of a organisation 19 | Execute the following to backup all git repositories of a GitHub organisation into the subdirectory `./YYYY-MM-DD/`. 20 | ```PowerShell 21 | .\backup_github_repositories.ps1 -userName "user" -userSecret "token" -organisationName "organisation" 22 | ``` 23 | 24 | ### Backup all git repositories of a user into a specific directory 25 | Execute the following to backup all git repositories of a GitHub user into the directory `C:\myBackupDirectory` and let the script prompt for the user secret. 26 | ```PowerShell 27 | .\backup_github_repositories.ps1 -userName "user" -backupDirectory "C:\myBackupDirectory" 28 | ``` 29 | 30 | ### Backup all git repositories with a maximum concurrency of 2 31 | Execute the following to backup all git repositories of a GitHub user into the subdirectory `./YYYY-MM-DD/` with a maximum concurrency of 2 background jobs. 32 | ```PowerShell 33 | .\backup_github_repositories.ps1 -userName "user" -backupDirectory "C:\myBackupDirectory" -maxConcurrency 2 34 | ``` 35 | 36 | ### Get detailed help 37 | Execute the following command to get detailed help. 38 | ```PowerShell 39 | Get-Help .\backup_github_repositories.ps1 -detailed 40 | ``` 41 | 42 | ## Frequently Asked Questions 43 | 44 | ### How do I work with a bare repository? 45 | 46 | A bare repository only consists of the history and is not intended to be used directly. If you want to work on a specific repository you must first convert it into a non-bare Git repository. That will give you the working tree of a specific branch. 47 | 48 | Execute the following to clone the bare Git repository `.\my_project.git` into a non-bare Git repository `.\my_project`: 49 | ```Shell 50 | git clone .\my_project.git 51 | ``` 52 | -------------------------------------------------------------------------------- /backup_github_repositories.ps1: -------------------------------------------------------------------------------- 1 | #Requires -Version 5.0 2 | 3 | <# 4 | .SYNOPSIS 5 | Automatically backups all remote GitHub repositories. 6 | 7 | .DESCRIPTION 8 | This script automatically backups all remote GitHub repositories of a user or an organisation to a local directory. 9 | 10 | .PARAMETER userName 11 | Specifies the GitHub user name. 12 | 13 | .PARAMETER userSecret 14 | Specifies the personal access token of the GitHub user. 15 | 16 | .PARAMETER organisationName 17 | Specifies the optional GitHub organisation name. 18 | 19 | .PARAMETER backupDirectory 20 | Overrides the default backup directory. 21 | 22 | .PARAMETER maxConcurrency 23 | Overrides the default concurrency of 8. 24 | 25 | .EXAMPLE 26 | .\backup_github_repositories.ps1 -userName "user" -userSecret "token" 27 | 28 | .EXAMPLE 29 | .\backup_github_repositories.ps1 -userName "user" -userSecret "token" -organisationName "organisation" 30 | 31 | .EXAMPLE 32 | .\backup_github_repositories.ps1 -backupDirectory "C:\myBackupDirectory" -maxConcurrency 1 33 | #> 34 | 35 | [CmdletBinding( 36 | DefaultParameterSetName = 'SecureSecret' 37 | )] 38 | Param ( 39 | 40 | [Parameter( 41 | Mandatory=$True, 42 | HelpMessage="The name of a GitHub user that has access to the GitHub API." 43 | )] 44 | [String] 45 | $userName, 46 | 47 | [Parameter( 48 | Mandatory=$True, 49 | HelpMessage="The personal access token of the GitHub user.", 50 | ParameterSetName = 'SecureSecret' 51 | )] 52 | [Security.SecureString]${personal access token}, 53 | [Parameter( 54 | Mandatory = $True, 55 | ParameterSetName = 'PlainTextSecret' 56 | )] 57 | [String] 58 | $userSecret, 59 | 60 | [String] 61 | $organisationName, 62 | 63 | [String] 64 | $backupDirectory, 65 | 66 | [ValidateRange(1,256)] 67 | [Int] 68 | $maxConcurrency=8 69 | ) 70 | 71 | # Consolidate the user secret, either from the argument or the prompt, in a secure string format. 72 | if ($userSecret) { 73 | $secureStringUserSecret = $userSecret | ConvertTo-SecureString -AsPlainText -Force 74 | } else { 75 | $secureStringUserSecret = ${personal access token} 76 | } 77 | 78 | # Convert the secure user secret string into a plain text representation. 79 | $plainTextUserSecret = [Runtime.InteropServices.Marshal]::PtrToStringAuto( 80 | [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secureStringUserSecret) 81 | ) 82 | 83 | # Default the backup directory to './YYYY-MM-DD'. This can 84 | # not be done in the Param section because $PSScriptRoot 85 | # will not be resolved if this script gets invoked from cmd. 86 | if (!$backupDirectory) { 87 | $backupDirectory = $(Join-Path -Path "$PSScriptRoot" -ChildPath $(Get-Date -UFormat "%Y-%m-%d")) 88 | } 89 | 90 | # Calculates the total repositories size in megabytes based on GitHubs 'size' property. 91 | function Get-TotalRepositoriesSizeInMegabytes([Object] $repositories) { 92 | 93 | $totalSizeInKilobytes = 0 94 | ForEach ($repository in $repositories) { 95 | $totalSizeInKilobytes += $repository.size 96 | } 97 | 98 | $([math]::Round($totalSizeInKilobytes/1024)) 99 | } 100 | 101 | # Measure the execution time of the backup script. 102 | $stopwatch = [System.Diagnostics.Stopwatch]::startNew() 103 | 104 | # Use TLS v1.2 105 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 106 | 107 | # 108 | # Use different API endpoints for user and organisation repositories. 109 | # 110 | # @see https://developer.github.com/v3/repos/#list-organization-repositories 111 | # @see https://developer.github.com/v3/repos/#list-your-repositories 112 | # 113 | if ($organisationName) { 114 | 115 | $gitHubRepositoriesUrl = "https://api.github.com/orgs/${organisationName}/repos?type=all&per_page=50" 116 | 117 | } else { 118 | 119 | $gitHubRepositoriesUrl = "https://api.github.com/user/repos?affiliation=owner&per_page=50" 120 | } 121 | 122 | # 123 | # Compose a Basic Authentication request header. 124 | # 125 | # @see https://developer.github.com/v3/auth/#basic-authentication 126 | # 127 | $basicAuthenticationCredentials = "${userName}:${plainTextUserSecret}" 128 | $encodedBasicAuthenticationCredentials = [System.Convert]::ToBase64String( 129 | [System.Text.Encoding]::ASCII.GetBytes($basicAuthenticationCredentials) 130 | ) 131 | $requestHeaders = @{ 132 | Authorization = "Basic $encodedBasicAuthenticationCredentials" 133 | } 134 | 135 | # Request the paginated GitHub API to get all repositories of a user or an organisation. 136 | $repositories = @() 137 | $pageNumber = 0 138 | Do { 139 | 140 | $pageNumber++ 141 | $paginatedGitHubApiUri = "${gitHubRepositoriesUrl}&page=${pageNumber}" 142 | 143 | Write-Host "Requesting '${paginatedGitHubApiUri}'..." -ForegroundColor "Yellow" 144 | $paginatedRepositories = Invoke-WebRequest -Uri $paginatedGitHubApiUri -Headers $requestHeaders | ` 145 | Select-Object -ExpandProperty Content | ` 146 | ConvertFrom-Json 147 | 148 | $repositories += $paginatedRepositories 149 | 150 | } Until ($paginatedRepositories.Count -eq 0) 151 | 152 | # Print a userfriendly message what will happen next. 153 | $totalSizeInMegabytes = Get-TotalRepositoriesSizeInMegabytes -repositories $repositories 154 | Write-Host "Cloning $($repositories.Count) repositories (~${totalSizeInMegabytes} MB) " -NoNewLine 155 | Write-Host "into '${backupDirectory}' with a maximum concurrency of ${maxConcurrency}:" 156 | 157 | # Clone each repository into the backup directory. 158 | ForEach ($repository in $repositories) { 159 | 160 | while ($true) { 161 | 162 | # Handle completed jobs as soon as possible. 163 | $completedJobs = $(Get-Job -State Completed) 164 | ForEach ($job in $completedJobs) { 165 | $job | Receive-Job 166 | $job | Remove-Job 167 | } 168 | 169 | $concurrencyLimitIsReached = $($(Get-Job -State Running).Count -ge $maxConcurrency) 170 | if ($concurrencyLimitIsReached) { 171 | 172 | $pollingFrequencyInMilliseconds = 50 173 | Start-Sleep -Milliseconds $pollingFrequencyInMilliseconds 174 | continue 175 | } 176 | 177 | # Clone or fetch a remote GitHub repository into a local directory. 178 | $scriptBlock = { 179 | 180 | Param ( 181 | [Parameter(Mandatory=$true)] 182 | [String] 183 | $fullName, 184 | 185 | [Parameter(Mandatory=$true)] 186 | [String] 187 | $directory 188 | ) 189 | 190 | if (Test-Path "${directory}") { 191 | 192 | git --git-dir="${directory}" fetch --quiet --all 193 | git --git-dir="${directory}" fetch --quiet --tags 194 | Write-Host "[${fullName}] Backup completed with git fetch strategy." 195 | return 196 | } 197 | 198 | git clone --quiet --mirror "git@github.com:${fullName}.git" "${directory}" 199 | Write-Host "[${fullName}] Backup completed with git clone strategy." 200 | } 201 | 202 | # Suffix the repository directory with a ".git" to indicate a bare repository. 203 | $directory = $(Join-Path -Path $backupDirectory -ChildPath "$($repository.name).git") 204 | 205 | Write-Host "[$($repository.full_name)] Starting backup to ${directory}..." -ForegroundColor "DarkYellow" 206 | Start-Job $scriptBlock -ArgumentList $repository.full_name, $directory | Out-Null 207 | 208 | # Give the job some time to start. 209 | $warmUpTimeoutInMilliseconds = 50 210 | Start-Sleep -Milliseconds $warmUpTimeoutInMilliseconds 211 | 212 | break 213 | } 214 | } 215 | 216 | # Wait for the last jobs to complete and output their results. 217 | Get-Job | Receive-job -AutoRemoveJob -Wait 218 | 219 | $stopwatch.Stop() 220 | $durationInSeconds = [Math]::Floor([Decimal]($stopwatch.Elapsed.TotalSeconds)) 221 | Write-Host "Successfully finished the backup in ${durationInSeconds} seconds." -ForegroundColor "Yellow" 222 | --------------------------------------------------------------------------------