├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ ├── codeql-analysis.yml │ ├── lint.yml │ └── release.yml ├── .gitignore ├── .idea ├── .gitignore ├── encodings.xml ├── gh-org-repo-sync.iml ├── modules.xml └── vcs.xml ├── .tool-versions ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── internal ├── cli │ └── cmd_exec.go ├── github │ ├── gh_cli.go │ └── repo.go └── reposync │ └── repo_handler.go └── main.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: weekly 8 | day: "saturday" 9 | time: '04:00' 10 | open-pull-requests-limit: 10 11 | reviewers: 12 | - rm3l 13 | labels: 14 | - dependencies 15 | - package-ecosystem: gomod 16 | directory: "/" 17 | schedule: 18 | interval: weekly 19 | day: "saturday" 20 | time: '04:00' 21 | open-pull-requests-limit: 10 22 | reviewers: 23 | - rm3l 24 | labels: 25 | - dependencies 26 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Install asdf dependencies 17 | uses: asdf-vm/actions/install@v4 18 | 19 | - name: Build 20 | run: go build -v ./... 21 | 22 | test: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v4 26 | 27 | - name: Install asdf dependencies 28 | uses: asdf-vm/actions/install@v4 29 | 30 | - name: Test 31 | run: go test -v ./... 32 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '42 12 * * 6' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v4 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v3 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v3 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v3 71 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | on: 3 | push: 4 | tags: 5 | - "v*" 6 | branches: 7 | - main 8 | pull_request: 9 | branches: [ main ] 10 | permissions: 11 | contents: read 12 | # Optional: allow read access to pull request. Use with `only-new-issues` option. 13 | pull-requests: read 14 | jobs: 15 | golangci: 16 | name: lint 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - name: Install asdf dependencies 23 | uses: asdf-vm/actions/install@v4 24 | 25 | - name: Read the right version of golangci-lint 26 | id: golangci_lint_version 27 | run: | 28 | golangCiLintVersion=$(grep '^golangci-lint ' .tool-versions | awk '{print $2}') 29 | echo "::set-output name=golangCiLintVersion::${golangCiLintVersion}" 30 | 31 | - name: golangci-lint 32 | uses: golangci/golangci-lint-action@v8 33 | with: 34 | # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. 35 | version: v${{ steps.golangci_lint_version.outputs.golangCiLintVersion }} 36 | # Optional: working directory, useful for monorepos 37 | # working-directory: somedir 38 | 39 | # Optional: golangci-lint command line arguments. 40 | # args: --issues-exit-code=0 41 | args: --timeout 3m0s 42 | 43 | # Optional: show only new issues if it's a pull request. The default value is `false`. 44 | only-new-issues: true -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | on: 3 | push: 4 | tags: 5 | - "v*" 6 | permissions: 7 | contents: write 8 | 9 | jobs: 10 | release: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Read the right version of Go 16 | id: golang_version 17 | run: | 18 | golangVersion=$(grep '^golang ' .tool-versions | awk '{print $2}') 19 | echo "::set-output name=golangVersion::${golangVersion}" 20 | 21 | - uses: cli/gh-extension-precompile@v2 22 | with: 23 | go_version: ${{ steps.golang_version.outputs.golangVersion }} 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /gh-org-repo-sync 2 | /gh-org-repo-sync.exe 3 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/gh-org-repo-sync.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | golang 1.24.3 2 | golangci-lint 2.1.6 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Armel Soro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gh-org-repo-sync 2 | 3 | [![Build](https://github.com/rm3l/gh-org-repo-sync/actions/workflows/build.yml/badge.svg)](https://github.com/rm3l/gh-org-repo-sync/actions/workflows/build.yml) 4 | [![Lint](https://github.com/rm3l/gh-org-repo-sync/actions/workflows/lint.yml/badge.svg)](https://github.com/rm3l/gh-org-repo-sync/actions/workflows/lint.yml) 5 | [![CodeQL Analysis](https://github.com/rm3l/gh-org-repo-sync/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/rm3l/gh-org-repo-sync/actions/workflows/codeql-analysis.yml) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/rm3l/gh-org-repo-sync)](https://goreportcard.com/report/github.com/rm3l/gh-org-repo-sync) 7 | 8 | > GitHub CLI extension to clone all repositories in an Organization, with the ability to filter via search queries. 9 | > If a local clone already exists, it fetches all remotes and pulls changes from the default branch. 10 | 11 | ## Installation 12 | 13 | - Install the `gh` CLI. See [https://github.com/cli/cli#installation](https://github.com/cli/cli#installation) for further details. 14 | - If not done already, also install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). 15 | - To be able to clone private repos you have access to inside the Organization, authenticate with your GitHub account by running `gh auth login`. Alternatively, the CLI will respect the `GITHUB_TOKEN` [environment variable](https://cli.github.com/manual/gh_help_environment). 16 | - Install the extension: 17 | 18 | ```bash 19 | gh extension install rm3l/gh-org-repo-sync 20 | ``` 21 | 22 | ## Usage 23 | 24 | ```bash 25 | ❯ gh org-repo-sync -h 26 | 27 | Usage: gh org-repo-sync [options] 28 | Options: 29 | -batch-size int 30 | the number of elements to retrieve at once. Must not exceed 100 (default 50) 31 | -dry-run 32 | dry run mode. to display the repositories that will get cloned or updated, 33 | without actually performing those actions 34 | -force 35 | whether to force sync repositories. 36 | Caution: this will hard-reset the branch of the destination repository to match the source repository. 37 | -output string 38 | the output path (default ".") 39 | -protocol string 40 | the protocol to use for cloning. Possible values: system, ssh, https. (default "system") 41 | -query string 42 | GitHub search query, to filter the Organization repositories. 43 | Example: "language:Java stars:>10 pushed:>2010-11-12" 44 | See https://bit.ly/3HurHe3 for more details on the search syntax 45 | ``` 46 | 47 | ## Working with the source code 48 | 49 | 1. Clone the repository: 50 | 51 | ``` 52 | git clone https://github.com/rm3l/gh-org-repo-sync 53 | cd gh-org-repo-sync 54 | ``` 55 | 56 | 2. Download and install [Go](https://go.dev/doc/install) to build the project. 57 | Or if you are already using the [asdf](https://asdf-vm.com/) version manager, you can just run `asdf install` to install all the necessary tools (declared in the [.tool-versions](.tool-versions) file). 58 | 59 | 3. Install the local version of this extension; `gh` symlinks to your local source code directory. 60 | 61 | ```bash 62 | # Install the local version 63 | gh extension install . 64 | ``` 65 | 66 | 4. At this point, you can start using it: 67 | 68 | ```bash 69 | gh org-repo-sync 70 | ``` 71 | 72 | 5. To see changes in the code as you develop, simply build and use the extension 73 | 74 | ```bash 75 | go build && gh org-repo-sync 76 | ``` 77 | 78 | ## Contribution Guidelines 79 | 80 | Contributions and issue reporting are more than welcome. So to help out, do feel free to fork this repo and open up a pull request. 81 | I'll review and merge your changes as quickly as possible. 82 | 83 | You can use [GitHub issues](https://github.com/rm3l/gh-org-repo-sync/issues) to report bugs. 84 | However, please make sure your description is clear enough and has sufficient instructions to be able to reproduce the issue. 85 | 86 | ## Developed by 87 | 88 | * Armel Soro 89 | * [keybase.io/rm3l](https://keybase.io/rm3l) 90 | * [rm3l.org](https://rm3l.org) - <armel+gh-org-repo-sync@rm3l.org> - [@rm3l](https://twitter.com/rm3l) 91 | * [paypal.me/rm3l](https://paypal.me/rm3l) 92 | * [coinbase.com/rm3l](https://www.coinbase.com/rm3l) 93 | 94 | ## License 95 | 96 | The MIT License (MIT) 97 | 98 | Copyright (c) 2022 Armel Soro 99 | 100 | Permission is hereby granted, free of charge, to any person obtaining a copy 101 | of this software and associated documentation files (the "Software"), to deal 102 | in the Software without restriction, including without limitation the rights 103 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 104 | copies of the Software, and to permit persons to whom the Software is 105 | furnished to do so, subject to the following conditions: 106 | 107 | The above copyright notice and this permission notice shall be included in all 108 | copies or substantial portions of the Software. 109 | 110 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 111 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 112 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 113 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 114 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 115 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 116 | SOFTWARE. 117 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rm3l/gh-org-repo-sync 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/cli/go-gh v1.2.1 7 | github.com/cli/safeexec v1.0.1 8 | github.com/cli/shurcooL-graphql v0.0.4 9 | golang.org/x/sync v0.14.0 10 | ) 11 | 12 | require ( 13 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 14 | github.com/henvic/httpretty v0.1.4 // indirect 15 | github.com/kr/text v0.2.0 // indirect 16 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 17 | github.com/mattn/go-isatty v0.0.20 // indirect 18 | github.com/mattn/go-runewidth v0.0.16 // indirect 19 | github.com/muesli/termenv v0.16.0 // indirect 20 | github.com/rivo/uniseg v0.4.7 // indirect 21 | github.com/thlib/go-timezone-local v0.0.6 // indirect 22 | golang.org/x/sys v0.33.0 // indirect 23 | golang.org/x/term v0.32.0 // indirect 24 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 25 | gopkg.in/yaml.v3 v3.0.1 // indirect 26 | ) 27 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= 2 | github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= 3 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= 4 | github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= 5 | github.com/cli/go-gh v1.2.1 h1:xFrjejSsgPiwXFP6VYynKWwxLQcNJy3Twbu82ZDlR/o= 6 | github.com/cli/go-gh v1.2.1/go.mod h1:Jxk8X+TCO4Ui/GarwY9tByWm/8zp4jJktzVZNlTW5VM= 7 | github.com/cli/safeexec v1.0.1 h1:e/C79PbXF4yYTN/wauC4tviMxEV13BwljGj0N9j+N00= 8 | github.com/cli/safeexec v1.0.1/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5zx3Q= 9 | github.com/cli/shurcooL-graphql v0.0.4 h1:6MogPnQJLjKkaXPyGqPRXOI2qCsQdqNfUY1QSJu2GuY= 10 | github.com/cli/shurcooL-graphql v0.0.4/go.mod h1:3waN4u02FiZivIV+p1y4d0Jo1jc6BViMA73C+sZo2fk= 11 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= 15 | github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= 16 | github.com/henvic/httpretty v0.0.6 h1:JdzGzKZBajBfnvlMALXXMVQWxWMF/ofTy8C3/OSUTxs= 17 | github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo= 18 | github.com/henvic/httpretty v0.1.4 h1:Jo7uwIRWVFxkqOnErcoYfH90o3ddQyVrSANeS4cxYmU= 19 | github.com/henvic/httpretty v0.1.4/go.mod h1:Dn60sQTZfbt2dYsdUSNsCljyF4AfdqnuJFDLJA1I4AM= 20 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 21 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 22 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 23 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 24 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 25 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 26 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 27 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 28 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 29 | github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= 30 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 31 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 32 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 33 | github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= 34 | github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 35 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 36 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 37 | github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= 38 | github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= 39 | github.com/muesli/termenv v0.12.0 h1:KuQRUE3PgxRFWhq4gHvZtPSLCGDqM5q/cYr1pZ39ytc= 40 | github.com/muesli/termenv v0.12.0/go.mod h1:WCCv32tusQ/EEZ5S8oUIIrC/nIuBcxCVqlN4Xfkv+7A= 41 | github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= 42 | github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= 43 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 44 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 45 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 46 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 47 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 48 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 49 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 50 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 51 | github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e h1:BuzhfgfWQbX0dWzYzT1zsORLnHRv3bcRcsaUk0VmXA8= 52 | github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI= 53 | github.com/thlib/go-timezone-local v0.0.6 h1:Ii3QJ4FhosL/+eCZl6Hsdr4DDU4tfevNoV83yAEo2tU= 54 | github.com/thlib/go-timezone-local v0.0.6/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI= 55 | golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= 56 | golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 57 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 58 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 59 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 60 | golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 61 | golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 62 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 63 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 64 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 65 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 66 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 67 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 68 | golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= 69 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 70 | golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= 71 | golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= 72 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 73 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 74 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 75 | gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= 76 | gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= 77 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 78 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 79 | -------------------------------------------------------------------------------- /internal/cli/cmd_exec.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/cli/safeexec" 7 | "os" 8 | "os/exec" 9 | ) 10 | 11 | // RunCommandInDir runs any command in the specified working directory 12 | func RunCommandInDir(executable string, workingDir string, env []string, args ...string) (stdOut, stdErr bytes.Buffer, err error) { 13 | executablePath, err := safeexec.LookPath(executable) 14 | if err != nil { 15 | err = fmt.Errorf("error while looking up the command specified: %s: %w", executablePath, err) 16 | return 17 | } 18 | cmd := exec.Command(executablePath, args...) 19 | cmd.Dir = workingDir 20 | cmd.Stdout = &stdOut 21 | cmd.Stderr = &stdErr 22 | if env != nil { 23 | cmd.Env = env 24 | } 25 | err = cmd.Run() 26 | fmt.Print(stdOut.String()) 27 | _, _ = fmt.Fprint(os.Stderr, stdErr.String()) 28 | if err != nil { 29 | err = fmt.Errorf("failed to run command: %s. error: %w", stdErr.String(), err) 30 | return 31 | } 32 | return 33 | } 34 | -------------------------------------------------------------------------------- /internal/github/gh_cli.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "bytes" 5 | "github.com/rm3l/gh-org-repo-sync/internal/cli" 6 | ) 7 | 8 | // RunGhCliInDir runs any gh command in the specified working directory, 9 | // because this is not possible to do with the default gh.Exec function 10 | func RunGhCliInDir(workingDir string, env []string, args ...string) (bytes.Buffer, bytes.Buffer, error) { 11 | return cli.RunCommandInDir("gh", workingDir, env, args...) 12 | } 13 | -------------------------------------------------------------------------------- /internal/github/repo.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "fmt" 5 | "github.com/cli/go-gh" 6 | "github.com/cli/go-gh/pkg/api" 7 | graphql "github.com/cli/shurcooL-graphql" 8 | "log" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | // RepositoryInfo contains few details about a given repository 14 | type RepositoryInfo struct { 15 | // Name is the repository name 16 | Name string 17 | 18 | // IsEmpty determines whether the GitHub repository is empty or not 19 | IsEmpty bool 20 | } 21 | 22 | type organizationResponse struct { 23 | repositories []RepositoryInfo 24 | repositoryCount int 25 | endCursor string 26 | } 27 | 28 | // GetOrganizationRepos returns an aggregated list of all repositories 29 | // within a GitHub organization, either private or public 30 | func GetOrganizationRepos(organization string, query string, batchSize int) ([]RepositoryInfo, error) { 31 | organizationSearchQuery := fmt.Sprintf("org:\"%s\"", organization) 32 | var queryString string 33 | if strings.Contains(query, organizationSearchQuery) { 34 | queryString = query 35 | } else { 36 | if query != "" { 37 | queryString = fmt.Sprintf("%s %s", organizationSearchQuery, query) 38 | } else { 39 | queryString = organizationSearchQuery 40 | } 41 | } 42 | if !strings.Contains(query, "fork:") { 43 | //Include forks by default 44 | queryString += " fork:true" 45 | } 46 | log.Println("[debug] queryString", queryString) 47 | 48 | opts := api.ClientOptions{ 49 | EnableCache: true, 50 | Timeout: 10 * time.Second, 51 | } 52 | client, err := gh.GQLClient(&opts) 53 | if err != nil { 54 | return nil, err 55 | } 56 | var orgRepositories = make([]RepositoryInfo, 0) 57 | organizationResponse, err := getOrganizationRepositories(&client, queryString, batchSize) 58 | if err != nil { 59 | return nil, err 60 | } 61 | orgRepositories = append(orgRepositories, organizationResponse.repositories...) 62 | var after = organizationResponse.endCursor 63 | if organizationResponse.repositoryCount > batchSize { 64 | for { 65 | organizationResponse, err := getOrganizationRepositoriesAfter(&client, queryString, batchSize, after) 66 | if err != nil { 67 | return nil, err 68 | } 69 | orgRepositories = append(orgRepositories, organizationResponse.repositories...) 70 | if organizationResponse.endCursor == "" { 71 | break 72 | } 73 | after = organizationResponse.endCursor 74 | } 75 | } 76 | return orgRepositories, nil 77 | } 78 | 79 | // RepositoryFragment is exported so as to be used in GraphQL APIs 80 | type RepositoryFragment struct { 81 | Name string 82 | IsEmpty bool 83 | } 84 | 85 | func getOrganizationRepositories(client *api.GQLClient, queryString string, batchSize int) (organizationResponse, error) { 86 | /* 87 | search(type: REPOSITORY, query: $query, first: $first) { 88 | pageInfo { 89 | endCursor 90 | startCursor 91 | } 92 | repositoryCount 93 | repos: edges { 94 | repo: node { 95 | ... on Repository { 96 | name 97 | isEmpty 98 | } 99 | } 100 | } 101 | */ 102 | var query struct { 103 | Search struct { 104 | PageInfo struct { 105 | StartCursor string 106 | EndCursor string 107 | } 108 | RepositoryCount int 109 | Repos []struct { 110 | Repo struct { 111 | RepositoryFragment `graphql:"... on Repository"` 112 | } `graphql:"repo: node"` 113 | } `graphql:"repos: edges"` 114 | } `graphql:"search(type: REPOSITORY, query: $query, first: $first)"` 115 | } 116 | variables := map[string]interface{}{ 117 | "query": graphql.String(queryString), 118 | "first": graphql.Int(batchSize), 119 | } 120 | 121 | err := (*client).Query("OrganizationRepositories", &query, variables) 122 | if err != nil { 123 | return organizationResponse{}, err 124 | } 125 | 126 | repositories := make([]RepositoryInfo, 0) 127 | for _, r := range query.Search.Repos { 128 | repositories = append(repositories, RepositoryInfo{ 129 | Name: r.Repo.Name, 130 | IsEmpty: r.Repo.IsEmpty, 131 | }) 132 | } 133 | return organizationResponse{ 134 | repositories: repositories, 135 | repositoryCount: query.Search.RepositoryCount, 136 | endCursor: query.Search.PageInfo.EndCursor, 137 | }, nil 138 | } 139 | 140 | func getOrganizationRepositoriesAfter(client *api.GQLClient, queryString string, batchSize int, after string) (organizationResponse, error) { 141 | var query struct { 142 | Search struct { 143 | PageInfo struct { 144 | StartCursor string 145 | EndCursor string 146 | } 147 | RepositoryCount int 148 | Repos []struct { 149 | Repo struct { 150 | RepositoryFragment `graphql:"... on Repository"` 151 | } `graphql:"repo: node"` 152 | } `graphql:"repos: edges"` 153 | } `graphql:"search(type: REPOSITORY, query: $query, first: $first, after: $after)"` 154 | } 155 | variables := map[string]interface{}{ 156 | "query": graphql.String(queryString), 157 | "first": graphql.Int(batchSize), 158 | "after": graphql.String(after), 159 | } 160 | 161 | err := (*client).Query("OrganizationRepositoriesAfter", &query, variables) 162 | if err != nil { 163 | return organizationResponse{}, err 164 | } 165 | 166 | repositories := make([]RepositoryInfo, 0) 167 | for _, r := range query.Search.Repos { 168 | repositories = append(repositories, RepositoryInfo{ 169 | Name: r.Repo.Name, 170 | IsEmpty: r.Repo.IsEmpty, 171 | }) 172 | } 173 | return organizationResponse{ 174 | repositories: repositories, 175 | repositoryCount: query.Search.RepositoryCount, 176 | endCursor: query.Search.PageInfo.EndCursor, 177 | }, nil 178 | } 179 | -------------------------------------------------------------------------------- /internal/reposync/repo_handler.go: -------------------------------------------------------------------------------- 1 | package reposync 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/cli/go-gh" 12 | "github.com/rm3l/gh-org-repo-sync/internal/cli" 13 | "github.com/rm3l/gh-org-repo-sync/internal/github" 14 | ) 15 | 16 | // CloneProtocol indicates the Git protocol to use for cloning. 17 | // See the constants exported in this package for further details. 18 | type CloneProtocol string 19 | 20 | const ( 21 | // SystemProtocol indicates whether to use the Git protocol configured in the GitHub CLI, 22 | // e.g., via the 'gh config set git_protocol' configuration command 23 | SystemProtocol CloneProtocol = "system" 24 | 25 | // SSHProtocol forces this extension to clone repositories via SSH. 26 | // As such, the Git remote will look like: git@github.com:org/repo.git 27 | SSHProtocol CloneProtocol = "ssh" 28 | 29 | // HTTPSProtocol forces this extension to clone repositories via HTTPS. 30 | // As such, the Git remote will look like: https://github.com/org/repo.git 31 | HTTPSProtocol CloneProtocol = "https" 32 | ) 33 | 34 | // HandleRepository determines whether a directory with the repository name does exist. 35 | // If it does, it checks out its default branch and updates it locally. 36 | // Otherwise, it clones it. 37 | func HandleRepository( 38 | _ context.Context, 39 | dryRun bool, 40 | output, 41 | organization string, 42 | repositoryInfo github.RepositoryInfo, 43 | protocol CloneProtocol, 44 | force bool, 45 | ) error { 46 | repository := repositoryInfo.Name 47 | repoPath, err := safeAbsPath(fmt.Sprintf("%s/%s", output, repository)) 48 | if err != nil { 49 | return err 50 | } 51 | info, err := os.Stat(repoPath) 52 | if err != nil { 53 | if errors.Is(err, os.ErrNotExist) { 54 | if dryRun { 55 | fmt.Printf("=> %s/%s: new clone in '%s'\n", organization, repository, repoPath) 56 | return nil 57 | } 58 | log.Println("[debug] cloning repo because local folder not found:", repoPath) 59 | if err := clone(repoPath, organization, repository, protocol); err != nil { 60 | return err 61 | } 62 | return nil 63 | } 64 | return err 65 | } 66 | if !info.IsDir() { 67 | return fmt.Errorf("expected folder for repository '%s'", repoPath) 68 | } 69 | if dryRun { 70 | fmt.Printf("=> %s/%s: update in '%s'\n", organization, repository, repoPath) 71 | return nil 72 | } 73 | log.Println("[debug] updating local clone for repo:", repoPath) 74 | return updateLocalClone(repoPath, organization, repositoryInfo, force) 75 | } 76 | 77 | func clone(output, organization string, repository string, protocol CloneProtocol) error { 78 | var repoUrl string 79 | switch protocol { 80 | case SystemProtocol: 81 | repoUrl = fmt.Sprintf("%s/%s", organization, repository) 82 | case SSHProtocol: 83 | repoUrl = fmt.Sprintf("git@github.com:%s/%s.git", organization, repository) 84 | case HTTPSProtocol: 85 | repoUrl = fmt.Sprintf("https://github.com/%s/%s.git", organization, repository) 86 | default: 87 | return fmt.Errorf("unknown protocol for cloning: %s", protocol) 88 | } 89 | repoPath, err := safeAbsPath(output) 90 | if err != nil { 91 | return err 92 | } 93 | args := []string{"repo", "clone", repoUrl, repoPath} 94 | _, stdErr, err := gh.Exec(args...) 95 | if stdErrString := stdErr.String(); stdErrString != "" { 96 | fmt.Println(stdErrString) 97 | } 98 | return err 99 | } 100 | 101 | func updateLocalClone(outputPath, organization string, repositoryInfo github.RepositoryInfo, force bool) error { 102 | repository := repositoryInfo.Name 103 | repoPath, err := safeAbsPath(outputPath) 104 | if err != nil { 105 | return err 106 | } 107 | err = fetchAllRemotes(repoPath, force) 108 | if err != nil { 109 | return err 110 | } 111 | if repositoryInfo.IsEmpty { 112 | log.Printf("[warn] skipped syncing empty repo: %s. Only remotes have been fetched\n", repoPath) 113 | return nil 114 | } 115 | args := []string{"repo", "sync", "--source", fmt.Sprintf("%s/%s", organization, repository)} 116 | if force { 117 | args = append(args, "--force") 118 | } 119 | _, _, err = github.RunGhCliInDir(repoPath, nil, args...) 120 | return err 121 | } 122 | 123 | func fetchAllRemotes(outputPath string, force bool) error { 124 | repoPath, err := safeAbsPath(outputPath) 125 | if err != nil { 126 | return err 127 | } 128 | args := []string{"fetch", "--all", "--prune", "--tags", "--recurse-submodules"} 129 | if force { 130 | args = append(args, "--force") 131 | } 132 | _, _, err = cli.RunCommandInDir("git", repoPath, nil, args...) 133 | return err 134 | } 135 | 136 | func safeAbsPath(p string) (string, error) { 137 | return filepath.Abs(filepath.FromSlash(p)) 138 | } 139 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "github.com/rm3l/gh-org-repo-sync/internal/github" 8 | "github.com/rm3l/gh-org-repo-sync/internal/reposync" 9 | "golang.org/x/sync/errgroup" 10 | "log" 11 | "os" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | const defaultBatchSize = 50 17 | 18 | var nbRepos int 19 | 20 | func main() { 21 | start := time.Now() 22 | defer func() { 23 | log.Println("[info] done handling", nbRepos, "repositories in", time.Since(start)) 24 | }() 25 | 26 | var dryRun bool 27 | var query string 28 | var batchSize int 29 | var output string 30 | var protocol string 31 | var force bool 32 | flag.BoolVar(&dryRun, "dry-run", false, 33 | `dry run mode. to display the repos that will get cloned or updated, 34 | without actually performing those actions`) 35 | flag.StringVar(&query, "query", "", 36 | `GitHub search query, to filter the Organization repositories. 37 | Example: "language:Go stars:>10 pushed:>2010-11-12" 38 | See https://bit.ly/3HurHe3 for more details on the search syntax`) 39 | flag.IntVar(&batchSize, "batch-size", defaultBatchSize, 40 | "the number of elements to retrieve at once. Must not exceed 100") 41 | flag.StringVar(&protocol, "protocol", string(reposync.SystemProtocol), 42 | fmt.Sprintf("the protocol to use for cloning. Possible values: %s, %s, %s.", reposync.SystemProtocol, 43 | reposync.SSHProtocol, reposync.HTTPSProtocol)) 44 | flag.StringVar(&output, "output", ".", "the output path") 45 | flag.BoolVar(&force, "force", false, 46 | `whether to force sync repositories. 47 | Caution: this will hard-reset the branch of the destination repository to match the source repository.`) 48 | 49 | flag.Usage = func() { 50 | //goland:noinspection GoUnhandledErrorResult 51 | fmt.Fprintln(os.Stderr, "Usage: gh org-repo-sync [options]") 52 | fmt.Println("Options: ") 53 | flag.PrintDefaults() 54 | } 55 | 56 | if len(os.Args) < 2 { 57 | //goland:noinspection GoUnhandledErrorResult 58 | fmt.Fprintln(os.Stderr, "missing organization") 59 | flag.Usage() 60 | os.Exit(1) 61 | } 62 | 63 | organization := os.Args[1] 64 | 65 | if organization == "-h" || organization == "-help" || organization == "--help" { 66 | flag.Usage() 67 | os.Exit(1) 68 | } else { 69 | // Ignore errors since flag.CommandLine is set for ExitOnError. 70 | _ = flag.CommandLine.Parse(os.Args[2:]) 71 | } 72 | 73 | if batchSize <= 0 || batchSize > 100 { 74 | //goland:noinspection GoUnhandledErrorResult 75 | fmt.Fprintf(os.Stderr, "invalid batch size (%d). Must be strictly higher than 0 and less than 100", 76 | batchSize) 77 | os.Exit(1) 78 | } 79 | cloneProtocol := reposync.CloneProtocol(strings.ToLower(protocol)) 80 | 81 | repositories, err := github.GetOrganizationRepos(organization, query, batchSize) 82 | if err != nil { 83 | log.Fatal(err) 84 | } 85 | nbRepos = len(repositories) 86 | log.Println("[debug] found", nbRepos, "repositories") 87 | if nbRepos == 0 { 88 | return 89 | } 90 | 91 | g, ctx := errgroup.WithContext(context.Background()) 92 | for _, repository := range repositories { 93 | g.Go(func(repo github.RepositoryInfo) func() error { 94 | return func() error { 95 | err := reposync.HandleRepository(ctx, dryRun, output, organization, repo, cloneProtocol, force) 96 | if err != nil { 97 | return fmt.Errorf("error while handling repo %q: %w", repo.Name, err) 98 | } 99 | return nil 100 | } 101 | }(repository)) 102 | } 103 | 104 | if err := g.Wait(); err != nil { 105 | panic(err) 106 | } 107 | } 108 | --------------------------------------------------------------------------------