├── .github └── workflows │ └── push.yml ├── LICENSE ├── README.md └── gitlab-repo-dl.sh /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | on: push 2 | name: ShellCheck 3 | jobs: 4 | shellcheck: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@master 8 | - name: shellcheck 9 | uses: actions/bin/shellcheck@master 10 | with: 11 | args: "*.sh -e SC2162,SC2006" 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 angristan (Stanislas Lange) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitLab repositories downloader 2 | 3 | Bash script to easily mass-download repos from a GitLab instance. 4 | 5 | Get yourself an API token from you GitLab instance then export `GITLAB_URL` and `GITLAB_ENV`. 6 | 7 | The script assume you will clone/pull over SSH. 8 | 9 | Requirements: bash, curl, jq, sed, awk, grep 10 | 11 | ## Download all projects from a group 12 | 13 | ```sh 14 | ./gitlab-repo-dl.sh group 15 | ``` 16 | 17 | It will clone all repos in `group_name` to the `./group_name` folder. If a repo already exists, it will be pulled. 18 | 19 | It doesn't support subgroups. 20 | 21 | ## Download all project from a GitLab instance 22 | 23 | This is seperated in two scripts to prevent an API brute-force everytime you want to clone/pull repositories. 24 | 25 | First, get all the repositories names including their namespace (= the whole path): 26 | 27 | ```sh 28 | ./gitlab-repo-dl.sh all-repo-list > list.txt 29 | ``` 30 | 31 | This can take quite a long time because it will call the API for each page of the list of projects, with 20 projects per page. 32 | 33 | Clone all the repositories from the list into a directory: 34 | 35 | ```sh 36 | ./gitlab-repo-dl.sh from-list list.txt . 37 | ``` 38 | 39 | This can take all long time for obvious reasons. When running the script multiple times, it will pull existing repositories. 40 | -------------------------------------------------------------------------------- /gitlab-repo-dl.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -z "$GITLAB_URL" ]; then 4 | echo "Missing environment variable: GITLAB_URL (e.g. https://gitlab.com)" 5 | exit 1 6 | fi 7 | 8 | if [ -z "$GITLAB_TOKEN" ]; then 9 | echo "Missing environment variable: GITLAB_TOKEN" 10 | echo "See ${GITLAB_URL}profile/account." 11 | exit 1 12 | fi 13 | 14 | if [ -z "$1" ]; then 15 | echo "Action is required. Can be one of 'group', 'all-repo-list', 'from-list'" 16 | exit 1 17 | fi 18 | 19 | if [ "$1" == "group" ]; then 20 | if [ -z "$2" ]; then 21 | echo "Group name is required." 22 | exit 1 23 | fi 24 | 25 | GROUP_NAME="$2" 26 | 27 | echo "Cloning all git projects in group $GROUP_NAME" 28 | 29 | REPO_SSH_URLS=$(curl -s "$GITLAB_URL/api/v4/groups/$GROUP_NAME/projects?private_token=$GITLAB_TOKEN&per_page=999" | jq '.[] | .ssh_url_to_repo' | sed 's/"//g') 30 | 31 | for REPO_SSH_URL in $REPO_SSH_URLS; do 32 | REPO_PATH="$GROUP_NAME/$(echo "$REPO_SSH_URL" | awk -F'/' '{print $NF}' | awk -F'.' '{print $1}')" 33 | 34 | if [ ! -d "$REPO_PATH" ]; then 35 | echo "git clone $REPO_PATH" 36 | git clone "$REPO_SSH_URL" "$REPO_PATH" 37 | else 38 | echo "git pull $REPO_PATH" 39 | (cd "$REPO_PATH" && git pull) 40 | fi 41 | done 42 | elif [ "$1" == "all-repo-list" ]; then 43 | # Get total number of pages (with 20 projects per page) from HTTP header 44 | TOTAL_PAGES=`curl "$GITLAB_URL/api/v4/projects?private_token=$GITLAB_TOKEN" -sI | grep X-Total-Pages | awk '{print $2}' | sed 's/\\r//g'` 45 | 46 | for ((PAGE_NUMBER = 1; PAGE_NUMBER <= TOTAL_PAGES; PAGE_NUMBER++)); do 47 | # echo git@instance:namespace/repo.git 48 | curl "$GITLAB_URL/api/v4/projects?private_token=$GITLAB_TOKEN&per_page=20&page=$PAGE_NUMBER" | jq '.[] | .ssh_url_to_repo' | sed 's/"//g' 49 | done 50 | elif [ "$1" == "from-list" ]; then 51 | if [ -z "$2" ]; then 52 | echo "List file name required" 53 | exit 1 54 | fi 55 | 56 | if [ -z "$3" ]; then 57 | echo "Target directory required" 58 | exit 1 59 | fi 60 | 61 | LIST_FILE="$2" 62 | TARGET_DIR="$3" 63 | 64 | if [ ! -d "$TARGET_DIR" ]; then 65 | mkdir -p "$TARGET_DIR" 66 | fi 67 | 68 | while read REPO_SSH_URL; do 69 | REPO_PATH="$(echo "$REPO_SSH_URL" | awk -F':' '{print $NF}' | awk -F'.' '{print $1}')" 70 | 71 | if [ ! -d "$TARGET_DIR/$REPO_PATH" ]; then 72 | echo "git clone $REPO_PATH" 73 | git clone "$REPO_SSH_URL" "$TARGET_DIR/$REPO_PATH" 74 | else 75 | echo "git pull $REPO_PATH" 76 | (cd "$TARGET_DIR/$REPO_PATH" && git pull) 77 | fi 78 | done <"$LIST_FILE" 79 | fi 80 | --------------------------------------------------------------------------------