├── .circleci └── config.yml ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── git-xargs-bug-report.md └── pull_request_template.md ├── .gitignore ├── .gon_amd64.hcl ├── .gon_arm64.hcl ├── .pre-commit-config.yaml ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── LICENSE.txt ├── README.md ├── auth ├── auth.go └── auth_test.go ├── cmd ├── .gitignore ├── git-xargs.go └── git-xargs_test.go ├── common └── common.go ├── config └── config.go ├── data ├── repo-rename-batches │ ├── batch1.txt │ ├── batch2.txt │ ├── batch3.txt │ ├── batch4.txt │ ├── batch5.txt │ └── batch6.txt ├── test-repos.txt ├── test │ ├── _testscripts │ │ ├── add-license.sh │ │ ├── bad-perm.sh │ │ ├── test-env-vars.sh │ │ ├── test-python.py │ │ ├── test-ruby.rb │ │ └── test-stdout-stderr.sh │ ├── bad-repo-address.txt │ ├── bash-commons.txt │ ├── cloud-nuke.txt │ ├── good-test-repos.txt │ ├── mixed-test-repos.txt │ ├── package-openvpn.txt │ ├── terragrunt.txt │ └── test-file-parsing.txt └── tf14-upgrade │ ├── batch1.txt │ ├── batch2.txt │ ├── batch3.txt │ ├── batch4.txt │ ├── batch5.txt │ ├── batch6.txt │ ├── batch7.txt │ ├── batch8.txt │ └── batch9.txt ├── docs ├── git-xargs-banner.png ├── git-xargs-demo.gif └── git-xargs-table.png ├── go.mod ├── go.sum ├── io ├── io.go ├── io_test.go ├── validate-input.go └── validate-input_test.go ├── local └── local.go ├── main.go ├── main_test.go ├── mocks └── mocks.go ├── printer └── printer.go ├── repository ├── fetch-repos.go ├── fetch-repos_test.go ├── process.go ├── process_test.go ├── repo-operations.go ├── repo-operations_test.go ├── select-repos.go └── select-repos_test.go ├── scripts ├── add-or-update-license.sh ├── add-or-update-pr-issue-templates.sh ├── circleci-workflows-version-upgrade.sh ├── error.sh ├── pre-commit-example.sh ├── sleep.sh ├── test-python.py ├── test-ruby.rb ├── tf14.rb └── update-repo-names.sh ├── stats └── stats.go ├── types ├── types.go └── types_test.go └── util └── util.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | env: &env 2 | environment: 3 | TERRATEST_LOG_PARSER_VERSION: NONE 4 | TERRAFORM_VERSION: NONE 5 | TERRAGRUNT_VERSION: NONE 6 | PACKER_VERSION: NONE 7 | GRUNTWORK_INSTALLER_VERSION: v0.0.39 8 | MODULE_CI_VERSION: v0.57.3 9 | GOLANG_VERSION: 1.21.1 10 | GO111MODULE: auto 11 | CGO_ENABLED: 1 12 | defaults: &defaults 13 | docker: 14 | - image: 087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:go1.21-tf1.5-tg39.1-pck1.8-ci50.7 15 | <<: *env 16 | install_gruntwork_utils: &install_gruntwork_utils 17 | name: Install gruntwork utils 18 | command: | 19 | curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}" 20 | gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/terraform-aws-ci" --tag "${MODULE_CI_VERSION}" 21 | configure-environment-for-gruntwork-module \ 22 | --terraform-version ${TERRAFORM_VERSION} \ 23 | --terragrunt-version ${TERRAGRUNT_VERSION} \ 24 | --packer-version ${PACKER_VERSION} \ 25 | --go-version ${GOLANG_VERSION} 26 | orbs: 27 | go: circleci/go@1.7.3 28 | version: 2.1 29 | jobs: 30 | pre-commit: 31 | <<: *defaults 32 | steps: 33 | - checkout 34 | - run: 35 | command: | 36 | pip install pre-commit 37 | go install golang.org/x/tools/cmd/goimports@v0.24.0 38 | export GOPATH=~/go/bin && export PATH=$PATH:$GOPATH 39 | pre-commit install 40 | pre-commit run --all-files 41 | test: 42 | <<: *defaults 43 | steps: 44 | - checkout 45 | - run: 46 | command: | 47 | # The go tests create a disposable local repo at runtime to execute git commands against, so we need to set any arbitrary options here to avoid an error message 48 | git config --global user.email "grunty@gruntwork.io" 49 | git config --global user.name "Grunty" 50 | - run: 51 | name: run git-xargs tests 52 | command: run-go-tests --timeout 5m 53 | no_output_timeout: 45m 54 | when: always 55 | build: 56 | resource_class: large 57 | <<: *defaults 58 | steps: 59 | - checkout 60 | - run: build-go-binaries --app-name git-xargs --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG" 61 | - persist_to_workspace: 62 | root: . 63 | paths: bin 64 | deploy: 65 | <<: *env 66 | macos: 67 | xcode: 15.3.0 68 | resource_class: macos.m1.medium.gen1 69 | steps: 70 | - checkout 71 | - attach_workspace: 72 | at: . 73 | - go/install: 74 | version: "1.20.5" 75 | - run: 76 | name: Install sign-binary-helpers 77 | command: | 78 | curl -Ls https://raw.githubusercontent.com/gruntwork-io/gruntwork-installer/master/bootstrap-gruntwork-installer.sh | bash /dev/stdin --version "${GRUNTWORK_INSTALLER_VERSION}" 79 | gruntwork-install --module-name "gruntwork-module-circleci-helpers" --repo "https://github.com/gruntwork-io/terraform-aws-ci" --tag "${MODULE_CI_VERSION}" 80 | gruntwork-install --module-name "sign-binary-helpers" --repo "https://github.com/gruntwork-io/terraform-aws-ci" --tag "${MODULE_CI_VERSION}" 81 | - run: 82 | name: Compile and sign the binaries 83 | command: | 84 | export AC_PASSWORD=${MACOS_AC_PASSWORD} 85 | export AC_PROVIDER=${MACOS_AC_PROVIDER} 86 | 87 | sign-binary --install-macos-sign-dependencies --os mac .gon_amd64.hcl 88 | sign-binary --os mac .gon_arm64.hcl 89 | echo "Done signing the binary" 90 | 91 | # Replace the files in bin. These are the same file names generated from .gon_amd64.hcl and .gon_arm64.hcl 92 | unzip git-xargs_darwin_amd64.zip 93 | mv git-xargs_darwin_amd64 bin/ 94 | 95 | unzip git-xargs_darwin_arm64.zip 96 | mv git-xargs_darwin_arm64 bin/ 97 | - run: 98 | name: Run SHA256SUM 99 | command: | 100 | brew install coreutils 101 | cd bin && sha256sum * > SHA256SUMS 102 | - run: upload-github-release-assets bin/* 103 | workflows: 104 | version: 2 105 | build-and-test: 106 | jobs: 107 | - pre-commit: 108 | filters: 109 | tags: 110 | only: /^v.*/ 111 | context: 112 | - AWS__PHXDEVOPS__circle-ci-test 113 | - GITHUB__PAT__gruntwork-ci 114 | - test: 115 | requires: 116 | - pre-commit 117 | filters: 118 | tags: 119 | only: /^v.*/ 120 | context: 121 | - AWS__PHXDEVOPS__circle-ci-test 122 | - GITHUB__PAT__gruntwork-ci 123 | - build: 124 | requires: 125 | - test 126 | filters: 127 | tags: 128 | only: /^v.*/ 129 | branches: 130 | ignore: /.*/ 131 | context: 132 | - AWS__PHXDEVOPS__circle-ci-test 133 | - GITHUB__PAT__gruntwork-ci 134 | - deploy: 135 | requires: 136 | - build 137 | filters: 138 | tags: 139 | only: /^v.*/ 140 | branches: 141 | ignore: /.*/ 142 | context: 143 | - AWS__PHXDEVOPS__circle-ci-test 144 | - GITHUB__PAT__gruntwork-ci 145 | - APPLE__OSX__code-signing 146 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: gruntwork-io 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a bug report to help us improve. 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | 14 | 15 | **Describe the bug** 16 | A clear and concise description of what the bug is. 17 | 18 | **To Reproduce** 19 | Steps to reproduce the behavior including the relevant Terraform/Terragrunt/Packer version number and any code snippets and module inputs you used. 20 | 21 | ```hcl 22 | // paste code snippets here 23 | ``` 24 | 25 | **Expected behavior** 26 | A clear and concise description of what you expected to happen. 27 | 28 | **Nice to have** 29 | - [ ] Terminal output 30 | - [ ] Screenshots 31 | 32 | **Additional context** 33 | Add any other context about the problem here. 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Submit a feature request for this repo. 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | 14 | 15 | **Describe the solution you'd like** 16 | A clear and concise description of what you want to happen. 17 | 18 | **Describe alternatives you've considered** 19 | A clear and concise description of any alternative solutions or features you've considered. 20 | 21 | **Additional context** 22 | Add any other context or screenshots about the feature request here. 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/git-xargs-bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: git-xargs bug report 3 | about: Report a problem with git-xargs 4 | title: '' 5 | labels: '' 6 | assignees: zackproser 7 | 8 | --- 9 | 10 | **git-xargs version** 11 | What does `git-xargs --version` show? 12 | 13 | **Describe the bug** 14 | A clear and concise description of what the bug is. 15 | 16 | **To Reproduce** 17 | Steps to reproduce the behavior: 18 | 1. Run this command... 19 | 2. Passing these flags 20 | 3. My `--repos` file looks like this... 21 | 4. See error 22 | 23 | **Expected behavior** 24 | A clear and concise description of what you expected to happen. 25 | 26 | **Screenshots** 27 | If applicable, add screenshots to help explain your problem. 28 | 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | Fixes #000. 6 | 7 | 8 | 9 | ## TODOs 10 | 11 | Read the [Gruntwork contribution guidelines](https://gruntwork.notion.site/Gruntwork-Coding-Methodology-02fdcd6e4b004e818553684760bf691e). 12 | 13 | - [ ] Update the docs. 14 | - [ ] Run the relevant tests successfully, including pre-commit checks. 15 | - [ ] Ensure any 3rd party code adheres with our [license policy](https://www.notion.so/gruntwork/Gruntwork-licenses-and-open-source-usage-policy-f7dece1f780341c7b69c1763f22b1378) or delete this line if its not applicable. 16 | - [ ] Include release notes. If this PR is backward incompatible, include a migration guide. 17 | 18 | ## Release Notes (draft) 19 | 20 | 21 | Added / Removed / Updated [X]. 22 | 23 | ### Migration Guide 24 | 25 | 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | test-repo/ 2 | .idea 3 | *.iml 4 | git-xargs 5 | -------------------------------------------------------------------------------- /.gon_amd64.hcl: -------------------------------------------------------------------------------- 1 | # See https://github.com/gruntwork-io/terraform-aws-ci/blob/main/modules/sign-binary-helpers/ 2 | # for further instructions on how to sign the binary + submitting for notarization. 3 | 4 | source = ["./bin/git-xargs_darwin_amd64"] 5 | 6 | bundle_id = "io.gruntwork.app.terragrunt" 7 | 8 | apple_id { 9 | username = "machine.apple@gruntwork.io" 10 | } 11 | 12 | sign { 13 | application_identity = "Developer ID Application: Gruntwork, Inc." 14 | } 15 | 16 | zip { 17 | output_path = "git-xargs_darwin_amd64.zip" 18 | } 19 | -------------------------------------------------------------------------------- /.gon_arm64.hcl: -------------------------------------------------------------------------------- 1 | # See https://github.com/gruntwork-io/terraform-aws-ci/blob/main/modules/sign-binary-helpers/ 2 | # for further instructions on how to sign the binary + submitting for notarization. 3 | 4 | source = ["./bin/git-xargs_darwin_arm64"] 5 | 6 | bundle_id = "io.gruntwork.app.terragrunt" 7 | 8 | apple_id { 9 | username = "machine.apple@gruntwork.io" 10 | } 11 | 12 | sign { 13 | application_identity = "Developer ID Application: Gruntwork, Inc." 14 | } 15 | 16 | zip { 17 | output_path = "git-xargs_darwin_arm64.zip" 18 | } 19 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/gruntwork-io/pre-commit 3 | rev: v0.1.13 4 | hooks: 5 | - id: goimports 6 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @hongil0316 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | support@gruntwork.io. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021 Gruntwork, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /auth/auth.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/google/go-github/v43/github" 9 | "github.com/gruntwork-io/git-xargs/types" 10 | "github.com/gruntwork-io/go-commons/errors" 11 | "golang.org/x/oauth2" 12 | ) 13 | 14 | // The go-github package satisfies this PullRequest service's interface in production 15 | type githubPullRequestService interface { 16 | Create(ctx context.Context, owner string, name string, pr *github.NewPullRequest) (*github.PullRequest, *github.Response, error) 17 | List(ctx context.Context, owner string, repo string, opts *github.PullRequestListOptions) ([]*github.PullRequest, *github.Response, error) 18 | RequestReviewers(ctx context.Context, owner, repo string, number int, reviewers github.ReviewersRequest) (*github.PullRequest, *github.Response, error) 19 | } 20 | 21 | // The go-github package satisfies this Repositories service's interface in production 22 | type githubRepositoriesService interface { 23 | Get(ctx context.Context, owner, repo string) (*github.Repository, *github.Response, error) 24 | ListByOrg(ctx context.Context, org string, opts *github.RepositoryListByOrgOptions) ([]*github.Repository, *github.Response, error) 25 | } 26 | 27 | // GithubClient is the data structure that is common between production code and test code. In production code, 28 | // go-github satisfies the PullRequests and Repositories service interfaces, whereas in test the concrete 29 | // implementations for these same services are mocks that return a static slice of pointers to GitHub repositories, 30 | // or a single pointer to a GitHub repository, as appropriate. This allows us to test the workflow of git-xargs 31 | // without actually making API calls to GitHub when running tests 32 | type GithubClient struct { 33 | PullRequests githubPullRequestService 34 | Repositories githubRepositoriesService 35 | } 36 | 37 | func NewClient(client *github.Client) GithubClient { 38 | return GithubClient{ 39 | PullRequests: client.PullRequests, 40 | Repositories: client.Repositories, 41 | } 42 | } 43 | 44 | // ConfigureGithubClient creates a GitHub API client using the user-supplied GITHUB_OAUTH_TOKEN and returns the configured GitHub client 45 | func ConfigureGithubClient() GithubClient { 46 | // Ensure user provided a GITHUB_OAUTH_TOKEN 47 | GithubOauthToken := os.Getenv("GITHUB_OAUTH_TOKEN") 48 | 49 | ts := oauth2.StaticTokenSource( 50 | &oauth2.Token{AccessToken: GithubOauthToken}, 51 | ) 52 | 53 | tc := oauth2.NewClient(context.Background(), ts) 54 | 55 | var githubClient *github.Client 56 | 57 | if os.Getenv("GITHUB_HOSTNAME") != "" { 58 | GithubHostname := os.Getenv("GITHUB_HOSTNAME") 59 | baseUrl := fmt.Sprintf("https://%s/", GithubHostname) 60 | 61 | githubClient, _ = github.NewEnterpriseClient(baseUrl, baseUrl, tc) 62 | 63 | } else { 64 | githubClient = github.NewClient(tc) 65 | } 66 | 67 | // Wrap the go-github client in a GithubClient struct, which is common between production and test code 68 | client := NewClient(githubClient) 69 | 70 | return client 71 | } 72 | 73 | // EnsureGithubOauthTokenSet is a sanity check that a value is exported for GITHUB_OAUTH_TOKEN 74 | func EnsureGithubOauthTokenSet() error { 75 | if os.Getenv("GITHUB_OAUTH_TOKEN") == "" { 76 | return errors.WithStackTrace(types.NoGithubOauthTokenProvidedErr{}) 77 | } 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /auth/auth_test.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestConfigureGithubClient performs a sanity check that you can configure a production GitHub API client 11 | // If GITHUB_HOSTNAME, use the github.NewEnterpriseClient 12 | func TestConfigureGithubClient(t *testing.T) { 13 | t.Parallel() 14 | 15 | t.Run("returns github client", func(t *testing.T) { 16 | client := ConfigureGithubClient() 17 | assert.NotNil(t, client) 18 | }) 19 | t.Run("returns github client with GithubHostname", func(t *testing.T) { 20 | GithubHostname := "ghe.my-domain.com" 21 | os.Setenv("GITHUB_HOSTNAME", GithubHostname) 22 | 23 | client := ConfigureGithubClient() 24 | assert.NotNil(t, client) 25 | 26 | }) 27 | 28 | } 29 | 30 | // TestNoGithubOauthTokenPassed temporarily drops the existing GITHUB_OAUTH_TOKEN env var to ensure that the validation 31 | // code throws an error when it is missing. It then replaces it. This is therefore the one test that cannot be run in 32 | // parallel. 33 | func TestNoGithubOAuthTokenPassed(t *testing.T) { 34 | token := os.Getenv("GITHUB_OAUTH_TOKEN") 35 | defer os.Setenv("GITHUB_OAUTH_TOKEN", token) 36 | 37 | os.Setenv("GITHUB_OAUTH_TOKEN", "") 38 | 39 | err := EnsureGithubOauthTokenSet() 40 | assert.Error(t, err) 41 | } 42 | -------------------------------------------------------------------------------- /cmd/.gitignore: -------------------------------------------------------------------------------- 1 | cover.out 2 | -------------------------------------------------------------------------------- /cmd/git-xargs.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strings" 9 | "time" 10 | 11 | "github.com/gruntwork-io/git-xargs/auth" 12 | "github.com/gruntwork-io/git-xargs/common" 13 | "github.com/gruntwork-io/git-xargs/config" 14 | gitxargs_io "github.com/gruntwork-io/git-xargs/io" 15 | "github.com/gruntwork-io/git-xargs/repository" 16 | "github.com/gruntwork-io/git-xargs/types" 17 | "github.com/gruntwork-io/go-commons/errors" 18 | "github.com/gruntwork-io/go-commons/logging" 19 | "github.com/urfave/cli" 20 | ) 21 | 22 | // parseGitXargsConfig accepts a urfave cli context and binds its values 23 | // to an internal representation of the data supplied by the user 24 | func parseGitXargsConfig(c *cli.Context) (*config.GitXargsConfig, error) { 25 | config := config.NewGitXargsConfig() 26 | config.Draft = c.Bool("draft") 27 | config.DryRun = c.Bool("dry-run") 28 | config.SkipPullRequests = c.Bool("skip-pull-requests") 29 | config.SkipArchivedRepos = c.Bool("skip-archived-repos") 30 | config.BranchName = c.String("branch-name") 31 | config.BaseBranchName = c.String("base-branch-name") 32 | config.CommitMessage = c.String("commit-message") 33 | config.PullRequestTitle = c.String("pull-request-title") 34 | config.PullRequestDescription = c.String("pull-request-description") 35 | config.Reviewers = c.StringSlice("reviewers") 36 | config.TeamReviewers = c.StringSlice("team-reviewers") 37 | config.ReposFile = c.String("repos") 38 | config.GithubOrg = c.String("github-org") 39 | config.RepoSlice = c.StringSlice("repo") 40 | config.MaxConcurrentRepos = c.Int("max-concurrent-repos") 41 | config.SecondsToSleepBetweenPRs = c.Int("seconds-between-prs") 42 | config.PullRequestRetries = c.Int("max-pr-retries") 43 | config.SecondsToSleepWhenRateLimited = c.Int("seconds-to-wait-when-rate-limited") 44 | maxConcurrentClones := c.Int("max-concurrent-clones") 45 | if maxConcurrentClones > 0 { 46 | config.CloneJobsLimiter = make(chan struct{}, maxConcurrentClones) 47 | } 48 | 49 | config.NoSkipCI = c.Bool("no-skip-ci") 50 | config.RetainLocalRepos = c.Bool("keep-cloned-repositories") 51 | // By default, prepend "[skip ci]" to commit messages, unless the user passed --no-skip-ci 52 | if config.NoSkipCI == false { 53 | commitMsgWithCISkip := fmt.Sprintf("%s %s", "[skip ci]", config.CommitMessage) 54 | config.CommitMessage = commitMsgWithCISkip 55 | } 56 | 57 | // A non-positive ticker value won't work, so set to the default minimum if user passed a bad value 58 | tickerVal := c.Int("seconds-between-prs") 59 | if tickerVal < 1 { 60 | tickerVal = common.DefaultSecondsBetweenPRs 61 | } 62 | 63 | config.Ticker = time.NewTicker(time.Duration(tickerVal) * time.Second) 64 | config.Args = c.Args() 65 | 66 | shouldReadStdIn, err := dataBeingPipedToStdIn() 67 | if err != nil { 68 | return nil, err 69 | } 70 | if shouldReadStdIn { 71 | repos, err := parseSliceFromStdIn() 72 | if err != nil { 73 | return nil, err 74 | } 75 | config.RepoFromStdIn = repos 76 | } 77 | 78 | return config, nil 79 | } 80 | 81 | // Return true if there is data being piped to stdin and false otherwise 82 | // Based on https://stackoverflow.com/a/26567513/483528. 83 | func dataBeingPipedToStdIn() (bool, error) { 84 | stat, err := os.Stdin.Stat() 85 | if err != nil { 86 | return false, err 87 | } 88 | 89 | return stat.Mode()&os.ModeCharDevice == 0, nil 90 | } 91 | 92 | // Read the data being passed to stdin and parse it as a slice of strings, where we assume strings are separated by 93 | // whitespace or newlines. All extra whitespace and empty lines are ignored. 94 | func parseSliceFromStdIn() ([]string, error) { 95 | return parseSliceFromReader(os.Stdin) 96 | } 97 | 98 | // Read the data from the given reader and parse it as a slice of strings, where we assume strings are separated by 99 | // whitespace or newlines. All extra whitespace and empty lines are ignored. 100 | func parseSliceFromReader(reader io.Reader) ([]string, error) { 101 | out := []string{} 102 | 103 | scanner := bufio.NewScanner(reader) 104 | for scanner.Scan() { 105 | words := strings.Fields(scanner.Text()) 106 | for _, word := range words { 107 | text := strings.TrimSpace(word) 108 | if text != "" { 109 | out = append(out, text) 110 | } 111 | } 112 | } 113 | 114 | err := scanner.Err() 115 | return out, errors.WithStackTrace(err) 116 | } 117 | 118 | // handleRepoProcessing encapsulates the main processing logic for the supplied repos and printing the run report that 119 | // is built up throughout the processing 120 | func handleRepoProcessing(config *config.GitXargsConfig) error { 121 | // Track whether pull requests were skipped 122 | config.Stats.SetSkipPullRequests(config.SkipPullRequests) 123 | 124 | // Update raw command supplied 125 | config.Stats.SetCommand(config.Args) 126 | 127 | if err := repository.OperateOnRepos(config); err != nil { 128 | return err 129 | } 130 | 131 | // Once all processing is complete, print out the summary of what was done 132 | config.Stats.PrintReport() 133 | 134 | return nil 135 | } 136 | 137 | // sanityCheckInputs performs validation on the user-supplied inputs to ensure we have everything we need: 138 | // 1. An exported GITHUB_OAUTH_TOKEN 139 | // 2. Arguments passed to the binary itself which should be executed against the targeted repos 140 | // 3. At least one of the three valid methods for selecting repositories 141 | func sanityCheckInputs(config *config.GitXargsConfig) error { 142 | if err := auth.EnsureGithubOauthTokenSet(); err != nil { 143 | return err 144 | } 145 | 146 | if len(config.Args) < 1 { 147 | return errors.WithStackTrace(types.NoArgumentsPassedErr{}) 148 | } 149 | 150 | if err := gitxargs_io.EnsureValidOptionsPassed(config); err != nil { 151 | return errors.WithStackTrace(err) 152 | } 153 | 154 | return nil 155 | } 156 | 157 | // RunGitXargs is the urfave cli app's Action that is called when the user executes the binary 158 | func RunGitXargs(c *cli.Context) error { 159 | // If someone calls us with no args at all, show the help text and exit 160 | if !c.Args().Present() { 161 | return cli.ShowAppHelp(c) 162 | } 163 | 164 | logger := logging.GetLogger("git-xargs") 165 | 166 | logger.Info("git-xargs running...") 167 | 168 | config, err := parseGitXargsConfig(c) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | if err := sanityCheckInputs(config); err != nil { 174 | return err 175 | } 176 | 177 | // If DryRun is enabled, notify user that no file changes will be made 178 | if config.DryRun { 179 | logger.Info("Dry run setting enabled. No local branches will be pushed and no PRs will be opened in Github") 180 | } 181 | 182 | return handleRepoProcessing(config) 183 | } 184 | -------------------------------------------------------------------------------- /cmd/git-xargs_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/gruntwork-io/git-xargs/config" 8 | "github.com/gruntwork-io/git-xargs/mocks" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | // A smoke test that you can define a basic config and pass it all the way through the main processing routine without 15 | // any errors 16 | func TestHandleRepoProcessing(t *testing.T) { 17 | t.Parallel() 18 | 19 | testConfig := config.NewGitXargsTestConfig() 20 | testConfig.ReposFile = "../data/test/good-test-repos.txt" 21 | testConfig.BranchName = "test-branch-name" 22 | testConfig.CommitMessage = "test-commit-name" 23 | testConfig.Args = []string{"touch", "test.txt"} 24 | testConfig.GithubClient = mocks.ConfigureMockGithubClient() 25 | testConfig.PullRequestRetries = 0 26 | testConfig.SecondsToSleepBetweenPRs = 1 27 | 28 | err := handleRepoProcessing(testConfig) 29 | assert.NoError(t, err) 30 | } 31 | 32 | func TestParseSliceFromReader(t *testing.T) { 33 | t.Parallel() 34 | 35 | testCases := []struct { 36 | name string 37 | input string 38 | expected []string 39 | }{ 40 | {"empty string", "", []string{}}, 41 | {"one string", "foo", []string{"foo"}}, 42 | {"one string with whitespace", " foo\t\t\t", []string{"foo"}}, 43 | {"multiple strings separated by whitespace", "foo bar baz\t\tblah", []string{"foo", "bar", "baz", "blah"}}, 44 | {"multiple strings separated by newlines", "foo\nbar\nbaz\nblah", []string{"foo", "bar", "baz", "blah"}}, 45 | {"multiple strings separated by newlines, with extra newlines", "\n\nfoo\nbar\n\nbaz\nblah\n\n\n", []string{"foo", "bar", "baz", "blah"}}, 46 | } 47 | 48 | for _, testCase := range testCases { 49 | // The following is necessary to make sure testCase's values don't 50 | // get updated due to concurrency within the scope of t.Run(..) below 51 | testCase := testCase 52 | 53 | t.Run(testCase.name, func(t *testing.T) { 54 | t.Parallel() 55 | 56 | actual, err := parseSliceFromReader(strings.NewReader(testCase.input)) 57 | require.NoError(t, err) 58 | require.Equal(t, testCase.expected, actual) 59 | }) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/urfave/cli" 4 | 5 | const ( 6 | GithubOrgFlagName = "github-org" 7 | DraftPullRequestFlagName = "draft" 8 | DryRunFlagName = "dry-run" 9 | SkipPullRequestsFlagName = "skip-pull-requests" 10 | SkipArchivedReposFlagName = "skip-archived-repos" 11 | RepoFlagName = "repo" 12 | ReposFileFlagName = "repos" 13 | CommitMessageFlagName = "commit-message" 14 | BranchFlagName = "branch-name" 15 | BaseBranchFlagName = "base-branch-name" 16 | PullRequestTitleFlagName = "pull-request-title" 17 | PullRequestDescriptionFlagName = "pull-request-description" 18 | PullRequestReviewersFlagName = "reviewers" 19 | PullRequestTeamReviewersFlagName = "team-reviewers" 20 | SecondsToWaitBetweenPrsFlagName = "seconds-between-prs" 21 | DefaultCommitMessage = "git-xargs programmatic commit" 22 | DefaultPullRequestTitle = "git-xargs programmatic pull request" 23 | DefaultPullRequestDescription = "git-xargs programmatic pull request" 24 | MaxPullRequestRetriesFlagName = "max-pr-retries" 25 | SecondsToWaitWhenRateLimitedFlagName = "seconds-to-wait-when-rate-limited" 26 | MaxConcurrentClonesFlagName = "max-concurrent-clones" 27 | NoSkipCIFlagName = "no-skip-ci" 28 | KeepClonedRepositoriesFlagName = "keep-cloned-repositories" 29 | DefaultMaxConcurrentClones = 4 30 | DefaultSecondsBetweenPRs = 1 31 | DefaultMaxPullRequestRetries = 3 32 | DefaultSecondsToWaitWhenRateLimited = 60 33 | ) 34 | 35 | var ( 36 | GenericGithubOrgFlag = cli.StringFlag{ 37 | Name: GithubOrgFlagName, 38 | Usage: "The Github organization to fetch all repositories from.", 39 | } 40 | GenericDraftPullRequestFlag = cli.BoolFlag{ 41 | Name: DraftPullRequestFlagName, 42 | Usage: "Whether to open pull requests in draft mode", 43 | } 44 | GenericDryRunFlag = cli.BoolFlag{ 45 | Name: DryRunFlagName, 46 | Usage: "When dry-run is set to true, no local branch changes will pushed and no pull requests will be opened.", 47 | } 48 | GenericSkipPullRequestFlag = cli.BoolFlag{ 49 | Name: SkipPullRequestsFlagName, 50 | Usage: "When skip-pull-requests is set to true, no pull requests will be opened. All changes will be committed and pushed to the specified branch directly.", 51 | } 52 | GenericSkipArchivedReposFlag = cli.BoolFlag{ 53 | Name: SkipArchivedReposFlagName, 54 | Usage: "Used in conjunction with github-org, will exclude archived repositories.", 55 | } 56 | GenericRepoFlag = cli.StringSliceFlag{ 57 | Name: RepoFlagName, 58 | Usage: "A single repo name to run the command on in the format of . Can be invoked multiple times with different repo names", 59 | } 60 | GenericRepoFileFlag = cli.StringFlag{ 61 | Name: ReposFileFlagName, 62 | Usage: "The path to a file containing repos, one per line in the format of ", 63 | } 64 | GenericBranchFlag = cli.StringFlag{ 65 | Name: BranchFlagName, 66 | Usage: "The name of the branch on which changes will be made", 67 | } 68 | GenericBaseBranchFlag = cli.StringFlag{ 69 | Name: BaseBranchFlagName, 70 | Usage: "The base branch that changes should be merged into", 71 | } 72 | GenericCommitMessageFlag = cli.StringFlag{ 73 | Name: CommitMessageFlagName, 74 | Usage: "The commit message to use when creating commits from changes introduced by your command or script", 75 | Value: DefaultCommitMessage, 76 | } 77 | GenericPullRequestTitleFlag = cli.StringFlag{ 78 | Name: PullRequestTitleFlagName, 79 | Usage: "The title to add to pull requests opened by git-xargs", 80 | Value: DefaultPullRequestTitle, 81 | } 82 | GenericPullRequestDescriptionFlag = cli.StringFlag{ 83 | Name: PullRequestDescriptionFlagName, 84 | Usage: "The description to add to pull requests opened by git-xargs", 85 | Value: DefaultPullRequestDescription, 86 | } 87 | GenericPullRequestReviewersFlag = cli.StringSliceFlag{ 88 | Name: PullRequestReviewersFlagName, 89 | Usage: "A list of GitHub usernames to request reviews from", 90 | } 91 | GenericPullRequestTeamReviewersFlag = cli.StringSliceFlag{ 92 | Name: PullRequestTeamReviewersFlagName, 93 | Usage: "A list of GitHub team names to request reviews from", 94 | } 95 | GenericSecondsToWaitFlag = cli.IntFlag{ 96 | Name: SecondsToWaitBetweenPrsFlagName, 97 | Usage: "The number of seconds to sleep between pull requests in order to respect GitHub API rate limits. Increase this number if you are being rate limited regularly. Defaults to 12 seconds.", 98 | Value: DefaultSecondsBetweenPRs, 99 | } 100 | GenericMaxPullRequestRetriesFlag = cli.IntFlag{ 101 | Name: MaxPullRequestRetriesFlagName, 102 | Usage: "The number of times to retry a pull request that failed due to rate limiting. Defaults to 3.", 103 | Value: DefaultMaxPullRequestRetries, 104 | } 105 | GenericSecondsToWaitWhenRateLimitedFlag = cli.IntFlag{ 106 | Name: SecondsToWaitWhenRateLimitedFlagName, 107 | Usage: "The number of additional seconds to sleep before attempting to open a PR again, when rate limited by GitHub. Defaults to 60.", 108 | Value: DefaultSecondsToWaitWhenRateLimited, 109 | } 110 | GenericMaxConcurrentClonesFlag = cli.IntFlag{ 111 | Name: MaxConcurrentClonesFlagName, 112 | Usage: "The maximum number of concurrent clones to run at once. Defaults to 4. If set to 0 no limit will be applied.", 113 | Value: DefaultMaxConcurrentClones, 114 | } 115 | GenericNoSkipCIFlag = cli.BoolFlag{ 116 | Name: NoSkipCIFlagName, 117 | Usage: "By default, git-xargs prepends \"[skip ci]\" to its commit messages. Pass this flag to prevent \"[skip ci]\" from being prepending to commit messages.", 118 | } 119 | GenericKeepClonedRepositoriesFlag = cli.BoolFlag{ 120 | Name: KeepClonedRepositoriesFlagName, 121 | Usage: "By default, git-xargs deletes the cloned repositories from the temp directory after the command has finished running, to save space on your machine. Pass this flag to prevent git-xargs from deleting the cloned repositories.", 122 | } 123 | ) 124 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/gruntwork-io/git-xargs/auth" 8 | "github.com/gruntwork-io/git-xargs/common" 9 | "github.com/gruntwork-io/git-xargs/local" 10 | "github.com/gruntwork-io/git-xargs/stats" 11 | "github.com/gruntwork-io/git-xargs/types" 12 | "github.com/gruntwork-io/git-xargs/util" 13 | ) 14 | 15 | // GitXargsConfig is the internal representation of a given git-xargs run as specified by the user 16 | type GitXargsConfig struct { 17 | Draft bool 18 | DryRun bool 19 | SkipPullRequests bool 20 | SkipArchivedRepos bool 21 | MaxConcurrentRepos int 22 | BranchName string 23 | BaseBranchName string 24 | CommitMessage string 25 | PullRequestTitle string 26 | PullRequestDescription string 27 | Reviewers []string 28 | TeamReviewers []string 29 | ReposFile string 30 | GithubOrg string 31 | RepoSlice []string 32 | RepoFromStdIn []string 33 | Args []string 34 | GithubClient auth.GithubClient 35 | GitClient local.GitClient 36 | Stats *stats.RunStats 37 | PRChan chan types.OpenPrRequest 38 | SecondsToSleepBetweenPRs int 39 | PullRequestRetries int 40 | SecondsToSleepWhenRateLimited int 41 | CloneJobsLimiter chan struct{} 42 | NoSkipCI bool 43 | RetainLocalRepos bool 44 | Ticker *time.Ticker 45 | } 46 | 47 | // NewGitXargsConfig sets reasonable defaults for a GitXargsConfig and returns a pointer to the config 48 | func NewGitXargsConfig() *GitXargsConfig { 49 | return &GitXargsConfig{ 50 | Draft: false, 51 | DryRun: false, 52 | SkipPullRequests: false, 53 | SkipArchivedRepos: false, 54 | MaxConcurrentRepos: 0, 55 | BranchName: "", 56 | BaseBranchName: "", 57 | CommitMessage: common.DefaultCommitMessage, 58 | PullRequestTitle: common.DefaultPullRequestTitle, 59 | PullRequestDescription: common.DefaultPullRequestDescription, 60 | Reviewers: []string{}, 61 | TeamReviewers: []string{}, 62 | ReposFile: "", 63 | GithubOrg: "", 64 | RepoSlice: []string{}, 65 | RepoFromStdIn: []string{}, 66 | Args: []string{}, 67 | GithubClient: auth.ConfigureGithubClient(), 68 | GitClient: local.NewGitClient(local.GitProductionProvider{}), 69 | Stats: stats.NewStatsTracker(), 70 | PRChan: make(chan types.OpenPrRequest), 71 | SecondsToSleepBetweenPRs: common.DefaultSecondsBetweenPRs, 72 | SecondsToSleepWhenRateLimited: common.DefaultSecondsToWaitWhenRateLimited, 73 | PullRequestRetries: common.DefaultMaxPullRequestRetries, 74 | CloneJobsLimiter: make(chan struct{}, common.DefaultMaxConcurrentClones), 75 | NoSkipCI: false, 76 | RetainLocalRepos: false, 77 | } 78 | } 79 | 80 | func NewGitXargsTestConfig() *GitXargsConfig { 81 | config := NewGitXargsConfig() 82 | 83 | uniqueID := util.RandStringBytes(9) 84 | config.BranchName = fmt.Sprintf("test-branch-%s", uniqueID) 85 | config.CommitMessage = fmt.Sprintf("commit-message-%s", uniqueID) 86 | config.GitClient = local.NewGitClient(local.MockGitProvider{}) 87 | 88 | config.Ticker = time.NewTicker(time.Duration(1) * time.Second) 89 | 90 | return config 91 | } 92 | 93 | func (c *GitXargsConfig) HasReviewers() bool { 94 | return len(c.Reviewers) > 0 || len(c.TeamReviewers) > 0 95 | } 96 | -------------------------------------------------------------------------------- /data/repo-rename-batches/batch1.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/aws-service-catalog -------------------------------------------------------------------------------- /data/repo-rename-batches/batch2.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-aws-ecs 2 | gruntwork-io/terraform-aws-vpc 3 | gruntwork-io/terraform-aws-server 4 | gruntwork-io/terraform-aws-ci 5 | gruntwork-io/terraform-aws-data-storage 6 | gruntwork-io/terraform-aws-monitoring 7 | gruntwork-io/terraform-aws-openvpn 8 | gruntwork-io/terraform-aws-asg 9 | gruntwork-io/terraform-aws-security 10 | gruntwork-io/terraform-aws-cache -------------------------------------------------------------------------------- /data/repo-rename-batches/batch3.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terratest 2 | gruntwork-io/infrastructure-as-code-training 3 | gruntwork-io/fetch 4 | gruntwork-io/gruntwork-installer 5 | gruntwork-io/terragrunt 6 | gruntwork-io/usage-patterns 7 | gruntwork-io/boilerplate 8 | gruntwork-io/gruntkms 9 | gruntwork-io/intro-to-terraform 10 | gruntwork-io/module-ci-update-terraform-variable-test 11 | gruntwork-io/gruntwork-cli 12 | gruntwork-io/terraform-aws-load-balancer 13 | gruntwork-io/terraform-aws-mongodb 14 | gruntwork-io/terraform-aws-utilities 15 | gruntwork-io/terraform-aws-static-assets 16 | gruntwork-io/cloud-nuke 17 | gruntwork-io/terraform-aws-lambda 18 | gruntwork-io/terraform-aws-messaging 19 | gruntwork-io/terragrunt-infrastructure-modules-example 20 | gruntwork-io/terragrunt-infrastructure-live-example 21 | -------------------------------------------------------------------------------- /data/repo-rename-batches/batch4.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-aws-zookeeper 2 | gruntwork-io/terraform-aws-kafka 3 | gruntwork-io/private-tls-cert 4 | gruntwork-io/gruntwork 5 | gruntwork-io/terraform-aws-sam 6 | gruntwork-io/circle-ci-docker-images 7 | gruntwork-io/health-checker 8 | gruntwork-io/infrastructure-modules 9 | gruntwork-io/infrastructure-live 10 | gruntwork-io/terraform-aws-couchbase 11 | gruntwork-io/bash-commons 12 | gruntwork-io/terraform-aws-elk 13 | gruntwork-io/kafka-health-check 14 | gruntwork-io/pre-commit 15 | gruntwork-io/terraform-module-in-root-for-terragrunt-test 16 | gruntwork-io/prototypes 17 | gruntwork-io/terraform-aws-influx 18 | gruntwork-io/terraform-aws-beanstalk 19 | gruntwork-io/terraform-google-gke 20 | gruntwork-io/terraform-google-network 21 | -------------------------------------------------------------------------------- /data/repo-rename-batches/batch5.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-aws-eks 2 | gruntwork-io/kubergrunt 3 | gruntwork-io/terraform-kubernetes-helm 4 | gruntwork-io/helm-kubernetes-services 5 | gruntwork-io/terraform-google-sql 6 | gruntwork-io/infrastructure-live-google 7 | gruntwork-io/infrastructure-modules-google 8 | gruntwork-io/terratest-helm-testing-example 9 | gruntwork-io/terraform-google-static-assets 10 | gruntwork-io/helmcharts 11 | gruntwork-io/terraform-google-load-balancer 12 | gruntwork-io/terraform-google-influx 13 | gruntwork-io/dogfood-infrastructure-modules 14 | gruntwork-io/dogfood-infrastructure-live 15 | gruntwork-io/usage-patterns-google 16 | gruntwork-io/terraform-aws-cis-service-catalog 17 | gruntwork-io/terraform-helm-gke-exts 18 | gruntwork-io/terraform-google-security 19 | gruntwork-io/team-product 20 | gruntwork-io/fork-repos 21 | -------------------------------------------------------------------------------- /data/repo-rename-batches/batch6.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-google-ci 2 | gruntwork-io/infrastructure-as-code-testing-talk 3 | gruntwork-io/company 4 | gruntwork-io/terraform-aws-ci-pipeline-example 5 | gruntwork-io/aws-sample-app 6 | gruntwork-io/aperture 7 | gruntwork-io/g7k-events 8 | gruntwork-io/terraform-aws-architecture-catalog 9 | gruntwork-io/terraform-kubernetes-namespace -------------------------------------------------------------------------------- /data/test-repos.txt: -------------------------------------------------------------------------------- 1 | zack-test-org/terraform-aws-asg 2 | zack-test-org/terraform-aws-vpc 3 | zack-test-org/terraform-aws-security 4 | zack-test-org/terraform-aws-eks 5 | zack-test-org/circleci-test-1 6 | -------------------------------------------------------------------------------- /data/test/_testscripts/add-license.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | YEAR=$(date +"%Y") 4 | FULLNAME="Gruntwork, LLC" 5 | 6 | function create_license { 7 | cat << EOF > LICENSE.txt 8 | MIT License 9 | 10 | Copyright (c) $YEAR, $FULLNAME 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy 13 | of this software and associated documentation files (the "Software"), to deal 14 | in the Software without restriction, including without limitation the rights 15 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 | copies of the Software, and to permit persons to whom the Software is 17 | furnished to do so, subject to the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be included in all 20 | copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 | SOFTWARE. 29 | EOF 30 | } 31 | 32 | echo "Writing license file to repo..." 33 | 34 | create_license 35 | -------------------------------------------------------------------------------- /data/test/_testscripts/bad-perm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "I am a script with bad permissions for use in testing" 4 | -------------------------------------------------------------------------------- /data/test/_testscripts/test-env-vars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script writes some text to stdout and stderr and then exits. 3 | # This is used to test that git-xargs registers environment variables based on flags and arguments. 4 | 5 | echo "XARGS_DRY_RUN=$XARGS_DRY_RUN" 6 | echo "XARGS_REPO_NAME=$XARGS_REPO_NAME" 7 | echo "XARGS_REPO_OWNER=$XARGS_REPO_OWNER" -------------------------------------------------------------------------------- /data/test/_testscripts/test-python.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | print("Python test script running") 3 | -------------------------------------------------------------------------------- /data/test/_testscripts/test-ruby.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | puts "Ruby test script running..." 3 | -------------------------------------------------------------------------------- /data/test/_testscripts/test-stdout-stderr.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script writes some text to stdout and stderr and then exits with an error. This is used to test that git-xargs 3 | # always logs the stdout and stderr from scripts, even if those scripts exit with an error. 4 | 5 | echo 'Hello, from STDOUT' 6 | >&2 echo 'Hello, from STDERR' 7 | exit 1 -------------------------------------------------------------------------------- /data/test/bad-repo-address.txt: -------------------------------------------------------------------------------- 1 | zack-test-org/me-not-really-a-valid-repo 2 | -------------------------------------------------------------------------------- /data/test/bash-commons.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/bash-commons 2 | -------------------------------------------------------------------------------- /data/test/cloud-nuke.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/cloud-nuke 2 | -------------------------------------------------------------------------------- /data/test/good-test-repos.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terragrunt 2 | -------------------------------------------------------------------------------- /data/test/mixed-test-repos.txt: -------------------------------------------------------------------------------- 1 | i don't feel so good 2 | gruntwork-io/cloud-nuke 3 | gruntwork-io/fetch 4 | imposter 5 | gruntwork-io/bash-commons 6 | heynowbrowncow/ 7 | -------------------------------------------------------------------------------- /data/test/package-openvpn.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-aws-openvpn 2 | -------------------------------------------------------------------------------- /data/test/terragrunt.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terragrunt 2 | -------------------------------------------------------------------------------- /data/test/test-file-parsing.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/fetch, 2 | gruntwork-io/cloud-nuke, 3 | gruntwork-io/bash-commons 4 | -------------------------------------------------------------------------------- /data/tf14-upgrade/batch1.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-aws-utilities -------------------------------------------------------------------------------- /data/tf14-upgrade/batch2.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-aws-vpc 2 | gruntwork-io/module-asg 3 | gruntwork-io/module-server -------------------------------------------------------------------------------- /data/tf14-upgrade/batch3.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/package-lambda 2 | gruntwork-io/module-security 3 | gruntwork-io/module-load-balancer 4 | gruntwork-io/terragrunt-infrastructure-modules-example 5 | gruntwork-io/terragrunt-infrastructure-live-example -------------------------------------------------------------------------------- /data/tf14-upgrade/batch4.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/module-data-storage 2 | gruntwork-io/module-cache 3 | gruntwork-io/package-messaging 4 | gruntwork-io/package-static-assets 5 | gruntwork-io/terraform-aws-monitoring -------------------------------------------------------------------------------- /data/tf14-upgrade/batch5.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/package-openvpn 2 | gruntwork-io/module-ecs 3 | gruntwork-io/module-ci 4 | gruntwork-io/terraform-aws-eks -------------------------------------------------------------------------------- /data/tf14-upgrade/batch6.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/package-zookeeper 2 | gruntwork-io/package-kafka 3 | gruntwork-io/package-elk -------------------------------------------------------------------------------- /data/tf14-upgrade/batch7.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/terraform-kubernetes-helm 2 | gruntwork-io/cis-compliance-aws 3 | gruntwork-io/cis-infrastructure-modules-acme 4 | gruntwork-io/usage-patterns 5 | gruntwork-io/infrastructure-modules-acme 6 | gruntwork-io/infrastructure-modules-multi-account-acme 7 | gruntwork-io/aws-service-catalog 8 | gruntwork-io/aws-architecture-catalog -------------------------------------------------------------------------------- /data/tf14-upgrade/batch8.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/package-sam 2 | gruntwork-io/terraform-aws-couchbase 3 | hashicorp/terraform-aws-consul 4 | hashicorp/terraform-aws-vault 5 | hashicorp/terraform-aws-nomad 6 | -------------------------------------------------------------------------------- /data/tf14-upgrade/batch9.txt: -------------------------------------------------------------------------------- 1 | gruntwork-io/infrastructure-as-code-testing-talk 2 | gruntwork-io/infrastructure-as-code-training 3 | gruntwork-io/intro-to-terraform -------------------------------------------------------------------------------- /docs/git-xargs-banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gruntwork-io/git-xargs/74f5c7530d3172c7f8a6ca35bab27f5117bf3063/docs/git-xargs-banner.png -------------------------------------------------------------------------------- /docs/git-xargs-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gruntwork-io/git-xargs/74f5c7530d3172c7f8a6ca35bab27f5117bf3063/docs/git-xargs-demo.gif -------------------------------------------------------------------------------- /docs/git-xargs-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gruntwork-io/git-xargs/74f5c7530d3172c7f8a6ca35bab27f5117bf3063/docs/git-xargs-table.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gruntwork-io/git-xargs 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.7 6 | 7 | require ( 8 | github.com/go-git/go-git/v5 v5.13.0 9 | github.com/google/go-github/v43 v43.0.0 10 | github.com/gruntwork-io/go-commons v0.8.2 11 | github.com/pterm/pterm v0.12.42 12 | github.com/sirupsen/logrus v1.9.0 13 | github.com/stretchr/testify v1.10.0 14 | github.com/urfave/cli v1.22.5 15 | golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 16 | ) 17 | 18 | require ( 19 | atomicgo.dev/cursor v0.1.1 // indirect 20 | atomicgo.dev/keyboard v0.2.8 // indirect 21 | dario.cat/mergo v1.0.0 // indirect 22 | github.com/Microsoft/go-winio v0.6.1 // indirect 23 | github.com/ProtonMail/go-crypto v1.1.3 // indirect 24 | github.com/cloudflare/circl v1.3.7 // indirect 25 | github.com/containerd/console v1.0.3 // indirect 26 | github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect 27 | github.com/cyphar/filepath-securejoin v0.2.5 // indirect 28 | github.com/davecgh/go-spew v1.1.1 // indirect 29 | github.com/emirpasic/gods v1.18.1 // indirect 30 | github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect 31 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect 32 | github.com/go-git/go-billy/v5 v5.6.0 // indirect 33 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 34 | github.com/golang/protobuf v1.5.0 // indirect 35 | github.com/google/go-querystring v1.1.0 // indirect 36 | github.com/gookit/color v1.5.0 // indirect 37 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 38 | github.com/kevinburke/ssh_config v1.2.0 // indirect 39 | github.com/lithammer/fuzzysearch v1.1.5 // indirect 40 | github.com/mattn/go-runewidth v0.0.13 // indirect 41 | github.com/pjbgf/sha1cd v0.3.0 // indirect 42 | github.com/pmezard/go-difflib v1.0.0 // indirect 43 | github.com/rivo/uniseg v0.2.0 // indirect 44 | github.com/russross/blackfriday/v2 v2.0.1 // indirect 45 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect 46 | github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect 47 | github.com/skeema/knownhosts v1.3.0 // indirect 48 | github.com/xanzy/ssh-agent v0.3.3 // indirect 49 | github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect 50 | golang.org/x/crypto v0.36.0 // indirect 51 | golang.org/x/mod v0.17.0 // indirect 52 | golang.org/x/net v0.38.0 // indirect 53 | golang.org/x/sync v0.12.0 // indirect 54 | golang.org/x/sys v0.31.0 // indirect 55 | golang.org/x/term v0.30.0 // indirect 56 | golang.org/x/text v0.23.0 // indirect 57 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect 58 | google.golang.org/appengine v1.6.7 // indirect 59 | google.golang.org/protobuf v1.33.0 // indirect 60 | gopkg.in/warnings.v0 v0.1.2 // indirect 61 | gopkg.in/yaml.v3 v3.0.1 // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /io/io.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | "strings" 7 | 8 | "github.com/gruntwork-io/git-xargs/types" 9 | "github.com/gruntwork-io/git-xargs/util" 10 | "github.com/gruntwork-io/go-commons/logging" 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // ProcessAllowedRepos accepts a path to the flat file in which the user has defined their explicitly allowed repos. 15 | // It expects repos to be defined one per line in the following format: `gruntwork-io/cloud-nuke` with optional commas. 16 | // Stray single and double quotes are also handled and stripped out if they are encountered, and spacing is irrelevant. 17 | func ProcessAllowedRepos(filepath string) ([]*types.AllowedRepo, error) { 18 | logger := logging.GetLogger("git-xargs") 19 | 20 | var allowedRepos []*types.AllowedRepo 21 | 22 | filepath = strings.TrimSpace(strings.Trim(filepath, "\n")) 23 | 24 | file, err := os.Open(filepath) 25 | 26 | if err != nil { 27 | logger.WithFields(logrus.Fields{ 28 | "Error": err, 29 | "Filepath": filepath, 30 | }).Debug("Could not open") 31 | 32 | return allowedRepos, err 33 | } 34 | 35 | // By wrapping the file.Close in a deferred anonymous function, we are able to avoid a nasty edge-case where 36 | // an actual closeErr would not be checked or handled properly in the more common `defer file.Close()` 37 | defer func() { 38 | closeErr := file.Close() 39 | if closeErr != nil { 40 | logger.WithFields(logrus.Fields{ 41 | "Error": closeErr, 42 | }).Debug("Error closing allowed repos file") 43 | } 44 | }() 45 | 46 | // Read through the file line by line, extracting the repo organization and name by splitting on the / char 47 | scanner := bufio.NewScanner(file) 48 | for scanner.Scan() { 49 | 50 | allowedRepo := util.ConvertStringToAllowedRepo(scanner.Text()) 51 | 52 | if allowedRepo != nil { 53 | allowedRepos = append(allowedRepos, allowedRepo) 54 | } 55 | } 56 | 57 | if err := scanner.Err(); err != nil { 58 | logger.WithFields(logrus.Fields{ 59 | "Error": err, 60 | }).Debug("Error parsing line from allowed repos file") 61 | } 62 | 63 | return allowedRepos, nil 64 | } 65 | -------------------------------------------------------------------------------- /io/io_test.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestProcessAllowedReposErrsOnBadFilepath(t *testing.T) { 10 | t.Parallel() 11 | 12 | intentionallyBadFilepath := "../data/test/i-am-not-really-here.sh" 13 | allowedRepos, err := ProcessAllowedRepos(intentionallyBadFilepath) 14 | 15 | assert.Error(t, err) 16 | assert.Equal(t, len(allowedRepos), 0) 17 | } 18 | 19 | func TestProcessAllowedReposCorrectlyParsesValidReposFile(t *testing.T) { 20 | t.Parallel() 21 | 22 | filepathToValidReposFile := "../data/test/test-file-parsing.txt" 23 | allowedRepos, err := ProcessAllowedRepos(filepathToValidReposFile) 24 | 25 | assert.NoError(t, err) 26 | assert.Equal(t, len(allowedRepos), 3) 27 | 28 | // Test that repo names are correctly parsed from the flat file by initially setting a map of each repo name 29 | // to false, and then updating each entry to true as we find them in the flat file. At the end, all map entries should be true / seen 30 | mapOfExpectedRepoNames := make(map[string]bool) 31 | mapOfExpectedRepoNames["fetch"] = false 32 | mapOfExpectedRepoNames["cloud-nuke"] = false 33 | mapOfExpectedRepoNames["bash-commons"] = false 34 | 35 | // ensure all test repos have the correct gruntwork-io org 36 | for _, repo := range allowedRepos { 37 | assert.Equal(t, repo.Organization, "gruntwork-io") 38 | // Update the map as having "seen" the repo 39 | mapOfExpectedRepoNames[repo.Name] = true 40 | } 41 | 42 | for _, v := range mapOfExpectedRepoNames { 43 | assert.True(t, v) 44 | } 45 | } 46 | 47 | func TestProcessAllowedReposCorrectlyFiltersMalformedInput(t *testing.T) { 48 | t.Parallel() 49 | 50 | filepathToReposFileWithSomeMalformedRepos := "../data/test/mixed-test-repos.txt" 51 | 52 | allowedRepos, err := ProcessAllowedRepos(filepathToReposFileWithSomeMalformedRepos) 53 | assert.NoError(t, err) 54 | 55 | // There are 3 valid repos defined in this test file, and 3 intentionally malformed repos, so only 3 should 56 | // be returned by the function as valid repos to operate on 57 | assert.Equal(t, len(allowedRepos), 3) 58 | 59 | // Test that repo names are correctly parsed from the flat file by initially setting a map of each repo name 60 | // to false, and then updating each entry to true as we find them in the flat file. At the end, all map entries should be true / seen 61 | mapOfExpectedRepoNames := make(map[string]bool) 62 | mapOfExpectedRepoNames["fetch"] = false 63 | mapOfExpectedRepoNames["cloud-nuke"] = false 64 | mapOfExpectedRepoNames["bash-commons"] = false 65 | 66 | // ensure all test repos have the correct gruntwork-io org 67 | for _, repo := range allowedRepos { 68 | assert.Equal(t, repo.Organization, "gruntwork-io") 69 | // Update the map as having "seen" the repo 70 | mapOfExpectedRepoNames[repo.Name] = true 71 | } 72 | 73 | for _, v := range mapOfExpectedRepoNames { 74 | assert.True(t, v) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /io/validate-input.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | import ( 4 | "github.com/gruntwork-io/git-xargs/config" 5 | "github.com/gruntwork-io/git-xargs/types" 6 | "github.com/gruntwork-io/go-commons/errors" 7 | ) 8 | 9 | // EnsureValidOptionsPassed checks that user has provided one valid method for selecting repos to operate on 10 | func EnsureValidOptionsPassed(config *config.GitXargsConfig) error { 11 | if len(config.RepoSlice) < 1 && config.ReposFile == "" && config.GithubOrg == "" && len(config.RepoFromStdIn) == 0 { 12 | return errors.WithStackTrace(types.NoRepoSelectionsMadeErr{}) 13 | } 14 | if config.BranchName == "" { 15 | return errors.WithStackTrace(types.NoBranchNameErr{}) 16 | } 17 | return nil 18 | } 19 | -------------------------------------------------------------------------------- /io/validate-input_test.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/git-xargs/config" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestEnsureValidOptionsPassedRejectsEmptySelectors(t *testing.T) { 11 | t.Parallel() 12 | emptyTestConfig := &config.GitXargsConfig{} 13 | 14 | err := EnsureValidOptionsPassed(emptyTestConfig) 15 | assert.Error(t, err) 16 | } 17 | 18 | func TestEnsureValidOptionsPassedAcceptsValidGithubOrg(t *testing.T) { 19 | t.Parallel() 20 | testConfigWithGithubOrg := &config.GitXargsConfig{ 21 | BranchName: "test-branch", 22 | GithubOrg: "gruntwork-io", 23 | } 24 | 25 | err := EnsureValidOptionsPassed(testConfigWithGithubOrg) 26 | assert.NoError(t, err) 27 | } 28 | 29 | func TestEnsureValidOptionsPassedAcceptsValidReposFile(t *testing.T) { 30 | t.Parallel() 31 | testConfigWithReposFile := &config.GitXargsConfig{ 32 | BranchName: "test-branch", 33 | ReposFile: "./my-repos.txt", 34 | } 35 | 36 | err := EnsureValidOptionsPassed(testConfigWithReposFile) 37 | assert.NoError(t, err) 38 | } 39 | 40 | func TestEnsureValidOptionsPassedAcceptedValidSingleRepo(t *testing.T) { 41 | t.Parallel() 42 | testConfigWithExplicitRepos := &config.GitXargsConfig{ 43 | BranchName: "test-branch", 44 | RepoSlice: []string{"gruntwork-io/cloud-nuke"}, 45 | } 46 | 47 | err := EnsureValidOptionsPassed(testConfigWithExplicitRepos) 48 | assert.NoError(t, err) 49 | } 50 | 51 | func TestEnsureValidOptionsPassedAcceptsAllFlagsSimultaneously(t *testing.T) { 52 | t.Parallel() 53 | testConfigWithAllSelectionCriteria := &config.GitXargsConfig{ 54 | BranchName: "test-branch", 55 | ReposFile: "./my-repos.txt", 56 | RepoSlice: []string{"gruntwork-io/cloud-nuke", "gruntwork-io/fetch"}, 57 | GithubOrg: "github-org", 58 | RepoFromStdIn: []string{"gruntwork-io/terragrunt"}, 59 | } 60 | 61 | err := EnsureValidOptionsPassed(testConfigWithAllSelectionCriteria) 62 | assert.NoError(t, err) 63 | } 64 | -------------------------------------------------------------------------------- /local/local.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import "github.com/go-git/go-git/v5" 4 | 5 | type GitProvider interface { 6 | PlainClone(path string, isBare bool, o *git.CloneOptions) (*git.Repository, error) 7 | } 8 | 9 | type GitProductionProvider struct{} 10 | 11 | func (g GitProductionProvider) PlainClone(path string, isBare bool, o *git.CloneOptions) (*git.Repository, error) { 12 | return git.PlainClone(path, isBare, o) 13 | } 14 | 15 | type MockGitProvider struct{} 16 | 17 | func (g MockGitProvider) PlainClone(path string, isBare bool, o *git.CloneOptions) (*git.Repository, error) { 18 | 19 | // Intercept the provided clone options and point to the locally checked out copy of github.com/gruntwork-io/fetch 20 | // to prevent any actual cloning or pushing being done to a real remote repo during testing 21 | o.URL = "../data/test/test-repo" 22 | 23 | return git.PlainClone(path, isBare, o) 24 | } 25 | 26 | type GitClient struct { 27 | GitProvider 28 | } 29 | 30 | func NewGitClient(provider GitProvider) GitClient { 31 | return GitClient{ 32 | provider, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gruntwork-io/git-xargs/cmd" 5 | "github.com/gruntwork-io/git-xargs/common" 6 | "github.com/gruntwork-io/go-commons/entrypoint" 7 | "github.com/gruntwork-io/go-commons/errors" 8 | "github.com/gruntwork-io/go-commons/logging" 9 | "github.com/sirupsen/logrus" 10 | "github.com/urfave/cli" 11 | ) 12 | 13 | // VERSION is set at build time using -ldflags parameters. For example, we typically set this flag in circle.yml 14 | // to the latest Git tag when building our Go apps: 15 | // 16 | // build-go-binaries --app-name my-app --dest-path bin --ld-flags "-X main.VERSION=$CIRCLE_TAG" 17 | // 18 | // For more info, see: http://stackoverflow.com/a/11355611/483528 19 | var VERSION string 20 | 21 | var LogLevelFlag = cli.StringFlag{ 22 | Name: "loglevel", 23 | Value: logrus.InfoLevel.String(), 24 | } 25 | 26 | // initCli initializes the CLI app before any command is actually executed. This function will handle all the setup 27 | // code, such as setting up the logger with the appropriate log level. 28 | func initCli(cliContext *cli.Context) error { 29 | // Set logging level 30 | logLevel := cliContext.String(LogLevelFlag.Name) 31 | level, err := logrus.ParseLevel(logLevel) 32 | if err != nil { 33 | return errors.WithStackTrace(err) 34 | } 35 | logging.SetGlobalLogLevel(level) 36 | return nil 37 | } 38 | 39 | func setupApp() *cli.App { 40 | app := entrypoint.NewApp() 41 | entrypoint.HelpTextLineWidth = 120 42 | 43 | // Override the CLI FlagEnvHinter, so it only returns the Usage text of the Flag and doesn't append the envVar text. Original func https://github.com/urfave/cli/blob/master/flag.go#L652 44 | cli.FlagEnvHinter = func(envVar, str string) string { 45 | return str 46 | } 47 | 48 | app.Name = "git-xargs" 49 | app.Author = "Gruntwork " 50 | 51 | app.Description = "git-xargs is a command-line tool (CLI) for making updates across multiple Github repositories with a single command." 52 | 53 | // Set the version number from your app from the VERSION variable that is passed in at build time 54 | app.Version = VERSION 55 | 56 | app.EnableBashCompletion = true 57 | 58 | app.Before = initCli 59 | 60 | app.Flags = []cli.Flag{ 61 | LogLevelFlag, 62 | common.GenericGithubOrgFlag, 63 | common.GenericDraftPullRequestFlag, 64 | common.GenericDryRunFlag, 65 | common.GenericSkipPullRequestFlag, 66 | common.GenericSkipArchivedReposFlag, 67 | common.GenericRepoFlag, 68 | common.GenericRepoFileFlag, 69 | common.GenericBranchFlag, 70 | common.GenericBaseBranchFlag, 71 | common.GenericCommitMessageFlag, 72 | common.GenericPullRequestTitleFlag, 73 | common.GenericPullRequestDescriptionFlag, 74 | common.GenericPullRequestReviewersFlag, 75 | common.GenericPullRequestTeamReviewersFlag, 76 | common.GenericSecondsToWaitFlag, 77 | common.GenericMaxPullRequestRetriesFlag, 78 | common.GenericSecondsToWaitWhenRateLimitedFlag, 79 | common.GenericMaxConcurrentClonesFlag, 80 | common.GenericNoSkipCIFlag, 81 | common.GenericKeepClonedRepositoriesFlag, 82 | } 83 | 84 | app.Action = cmd.RunGitXargs 85 | 86 | return app 87 | } 88 | 89 | // main should only setup the CLI flags and help texts. 90 | func main() { 91 | app := setupApp() 92 | 93 | entrypoint.RunApp(app) 94 | } 95 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/gruntwork-io/git-xargs/cmd" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/urfave/cli" 11 | ) 12 | 13 | func TestSetupApp(t *testing.T) { 14 | app := setupApp() 15 | assert.NotNil(t, app) 16 | } 17 | 18 | func TestGitXargsShowsHelpTextForEmptyArgs(t *testing.T) { 19 | app := setupApp() 20 | 21 | // Capture the app's stdout 22 | var stdout strings.Builder 23 | app.Writer = &stdout 24 | 25 | emptyFlagSet := flag.NewFlagSet("git-xargs-test", flag.ContinueOnError) 26 | emptyTestContext := cli.NewContext(app, emptyFlagSet, nil) 27 | 28 | err := cmd.RunGitXargs(emptyTestContext) 29 | 30 | // Make sure we see the help text 31 | assert.NoError(t, err) 32 | assert.Contains(t, stdout.String(), app.Description) 33 | } 34 | -------------------------------------------------------------------------------- /mocks/mocks.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/google/go-github/v43/github" 8 | "github.com/gruntwork-io/git-xargs/auth" 9 | ) 10 | 11 | // Mock *github.Repository slice that is returned from the mock Repositories service in test 12 | var ownerName = "gruntwork-io" 13 | 14 | var ( 15 | repoName1 = "terragrunt" 16 | repoName2 = "terratest" 17 | repoName3 = "fetch" 18 | repoName4 = "terraform-kubernetes-helm" 19 | repoName5 = "terraform-google-load-balancer" 20 | ) 21 | 22 | var ( 23 | repoURL1 = "https://github.com/gruntwork-io/terragrunt" 24 | repoURL2 = "https://github.com/gruntwork-io/terratest" 25 | repoURL3 = "https://github.com/gruntwork-io/fetch" 26 | repoURL4 = "https://github.com/gruntwork-io/terraform-kubernetes-helm" 27 | repoURL5 = "https://github.com/gruntwork-io/terraform-google-load-balancer" 28 | ) 29 | 30 | var archivedFlag = true 31 | 32 | var MockGithubRepositories = []*github.Repository{ 33 | { 34 | Owner: &github.User{ 35 | Login: &ownerName, 36 | }, 37 | Name: &repoName1, 38 | HTMLURL: &repoURL1, 39 | }, 40 | { 41 | Owner: &github.User{ 42 | Login: &ownerName, 43 | }, 44 | Name: &repoName2, 45 | HTMLURL: &repoURL2, 46 | }, 47 | { 48 | Owner: &github.User{ 49 | Login: &ownerName, 50 | }, 51 | Name: &repoName3, 52 | HTMLURL: &repoURL3, 53 | }, 54 | { 55 | Owner: &github.User{ 56 | Login: &ownerName, 57 | }, 58 | Name: &repoName4, 59 | HTMLURL: &repoURL4, 60 | Archived: &archivedFlag, 61 | }, 62 | { 63 | Owner: &github.User{ 64 | Login: &ownerName, 65 | }, 66 | Name: &repoName5, 67 | HTMLURL: &repoURL5, 68 | Archived: &archivedFlag, 69 | }, 70 | } 71 | 72 | // This mocks the PullRequest service in go-github that is used in production to call the associated GitHub endpoint 73 | type mockGithubPullRequestService struct { 74 | PullRequest *github.PullRequest 75 | Response *github.Response 76 | } 77 | 78 | func (m mockGithubPullRequestService) Create(ctx context.Context, owner, name string, pr *github.NewPullRequest) (*github.PullRequest, *github.Response, error) { 79 | return m.PullRequest, m.Response, nil 80 | } 81 | 82 | func (m mockGithubPullRequestService) List(ctx context.Context, owner string, repo string, opts *github.PullRequestListOptions) ([]*github.PullRequest, *github.Response, error) { 83 | return []*github.PullRequest{m.PullRequest}, m.Response, nil 84 | } 85 | 86 | func (m mockGithubPullRequestService) RequestReviewers(ctx context.Context, owner, repo string, number int, reviewers github.ReviewersRequest) (*github.PullRequest, *github.Response, error) { 87 | return m.PullRequest, m.Response, nil 88 | } 89 | 90 | // This mocks the Repositories service in go-github that is used in production to call the associated GitHub endpoint 91 | type mockGithubRepositoriesService struct { 92 | Repository *github.Repository 93 | Repositories []*github.Repository 94 | Response *github.Response 95 | } 96 | 97 | func (m mockGithubRepositoriesService) Get(ctx context.Context, owner, repo string) (*github.Repository, *github.Response, error) { 98 | return m.Repository, m.Response, nil 99 | } 100 | 101 | func (m mockGithubRepositoriesService) ListByOrg(ctx context.Context, org string, opts *github.RepositoryListByOrgOptions) ([]*github.Repository, *github.Response, error) { 102 | return m.Repositories, m.Response, nil 103 | } 104 | 105 | // ConfigureMockGithubClient returns a valid GithubClient configured for testing purposes, complete with the mocked services 106 | func ConfigureMockGithubClient() auth.GithubClient { 107 | // Call the same NewClient method that is used by the actual CLI to obtain a GitHub client that calls the 108 | // GitHub API. In testing, however, we just implement the mock services above to satisfy the interfaces required 109 | // by the GithubClient. GithubClient is used uniformly between production and test code, with the only difference 110 | // being that in test we do not actually execute API calls to GitHub 111 | client := auth.NewClient(github.NewClient(nil)) 112 | 113 | testHTMLUrl := "https://github.com/gruntwork-io/test/pull/1" 114 | 115 | client.Repositories = mockGithubRepositoriesService{ 116 | Repository: MockGithubRepositories[0], 117 | Repositories: MockGithubRepositories, 118 | Response: &github.Response{ 119 | Response: &http.Response{ 120 | StatusCode: 200, 121 | }, 122 | 123 | NextPage: 0, 124 | PrevPage: 0, 125 | FirstPage: 0, 126 | LastPage: 0, 127 | 128 | NextPageToken: "dontuseme", 129 | 130 | Rate: github.Rate{}, 131 | }, 132 | } 133 | client.PullRequests = mockGithubPullRequestService{ 134 | PullRequest: &github.PullRequest{ 135 | HTMLURL: &testHTMLUrl, 136 | }, 137 | Response: &github.Response{}, 138 | } 139 | 140 | return client 141 | } 142 | 143 | func GetMockGithubRepo() *github.Repository { 144 | userLogin := "gruntwork-io" 145 | user := &github.User{ 146 | Login: &userLogin, 147 | } 148 | 149 | repoName := "terragrunt" 150 | cloneURL := "https://github.com/gruntwork-io/terragrunt" 151 | 152 | repo := &github.Repository{ 153 | Owner: user, 154 | Name: &repoName, 155 | CloneURL: &cloneURL, 156 | } 157 | 158 | return repo 159 | } 160 | -------------------------------------------------------------------------------- /printer/printer.go: -------------------------------------------------------------------------------- 1 | package printer 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/gruntwork-io/git-xargs/types" 8 | "github.com/pterm/pterm" 9 | ) 10 | 11 | func PrintRepoReport(allEvents []types.AnnotatedEvent, runReport *types.RunReport) { 12 | renderSection(fmt.Sprintf("Git-xargs run summary @ %s", time.Now().UTC())) 13 | 14 | pterm.DefaultBulletList.WithItems([]pterm.BulletListItem{ 15 | {Level: 0, Text: fmt.Sprintf("Runtime in seconds: %d", runReport.RuntimeSeconds)}, 16 | {Level: 0, Text: fmt.Sprintf("Command supplied: %s", runReport.Command)}, 17 | {Level: 0, Text: fmt.Sprintf("Repo selection method: %s", runReport.SelectionMode)}, 18 | }).Render() 19 | 20 | if len(runReport.FileProvidedRepos) > 0 { 21 | renderSection("Repos supplied via --repos file flag") 22 | data := make([][]string, len(runReport.FileProvidedRepos)) 23 | for idx, fileProvidedRepo := range runReport.FileProvidedRepos { 24 | data[idx] = []string{fmt.Sprintf("%s/%s", fileProvidedRepo.Organization, fileProvidedRepo.Name)} 25 | } 26 | renderTableWithHeader([]string{"Repo name"}, data) 27 | } 28 | 29 | // For each event type, print a summary table of the repos in that category 30 | for _, ae := range allEvents { 31 | 32 | var reducedRepos []types.ReducedRepo 33 | 34 | for _, repo := range runReport.Repos[ae.Event] { 35 | rr := types.ReducedRepo{ 36 | Name: repo.GetName(), 37 | URL: repo.GetHTMLURL(), 38 | } 39 | reducedRepos = append(reducedRepos, rr) 40 | } 41 | 42 | if len(reducedRepos) > 0 { 43 | 44 | renderSection(ae.Description) 45 | data := make([][]string, len(reducedRepos)) 46 | for idx, repo := range reducedRepos { 47 | data[idx] = []string{repo.Name, repo.URL} 48 | } 49 | 50 | renderTableWithHeader([]string{"Repo name", "Repo URL"}, data) 51 | } 52 | } 53 | 54 | var pullRequests []types.PullRequest 55 | 56 | for repoName, prURL := range runReport.PullRequests { 57 | pr := types.PullRequest{ 58 | Repo: repoName, 59 | URL: prURL, 60 | } 61 | pullRequests = append(pullRequests, pr) 62 | } 63 | 64 | var draftPullRequests []types.PullRequest 65 | 66 | for repoName, prURL := range runReport.DraftPullRequests { 67 | pr := types.PullRequest{ 68 | Repo: repoName, 69 | URL: prURL, 70 | } 71 | draftPullRequests = append(draftPullRequests, pr) 72 | } 73 | 74 | if len(pullRequests) > 0 { 75 | renderSection("Pull requests opened") 76 | 77 | data := make([][]string, len(pullRequests)) 78 | for idx, pullRequest := range pullRequests { 79 | data[idx] = []string{pullRequest.Repo, pullRequest.URL} 80 | } 81 | 82 | renderTableWithHeader([]string{"Repo name", "Pull request URL"}, data) 83 | } 84 | 85 | if len(draftPullRequests) > 0 { 86 | renderSection("Draft Pull requests opened") 87 | 88 | data := make([][]string, len(draftPullRequests)) 89 | for idx, draftPullRequest := range draftPullRequests { 90 | data[idx] = []string{draftPullRequest.Repo, draftPullRequest.URL} 91 | } 92 | 93 | renderTableWithHeader([]string{"Repo name", "Draft Pull request URL"}, data) 94 | } 95 | } 96 | 97 | func renderSection(sectionTitle string) { 98 | pterm.DefaultSection.Style = pterm.NewStyle(pterm.FgLightCyan) 99 | pterm.DefaultSection.WithLevel(0).Println(sectionTitle) 100 | } 101 | 102 | func renderTableWithHeader(headers []string, data [][]string) { 103 | tableData := pterm.TableData{ 104 | headers, 105 | } 106 | for idx := range data { 107 | tableData = append(tableData, data[idx]) 108 | } 109 | pterm.DefaultTable. 110 | WithHasHeader(). 111 | WithBoxed(true). 112 | WithRowSeparator("-"). 113 | WithData(tableData). 114 | Render() 115 | } 116 | -------------------------------------------------------------------------------- /repository/fetch-repos.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/gruntwork-io/git-xargs/auth" 8 | "github.com/gruntwork-io/git-xargs/config" 9 | "github.com/gruntwork-io/git-xargs/stats" 10 | "github.com/gruntwork-io/git-xargs/types" 11 | "github.com/gruntwork-io/go-commons/errors" 12 | 13 | "github.com/google/go-github/v43/github" 14 | "github.com/gruntwork-io/go-commons/logging" 15 | "github.com/sirupsen/logrus" 16 | ) 17 | 18 | // getFileDefinedRepos converts user-supplied repositories to GitHub API response objects that can be further processed 19 | func getFileDefinedRepos(GithubClient auth.GithubClient, allowedRepos []*types.AllowedRepo, tracker *stats.RunStats) ([]*github.Repository, error) { 20 | logger := logging.GetLogger("git-xargs") 21 | 22 | var allRepos []*github.Repository 23 | 24 | for _, allowedRepo := range allowedRepos { 25 | 26 | logger.WithFields(logrus.Fields{ 27 | "Organization": allowedRepo.Organization, 28 | "Name": allowedRepo.Name, 29 | }).Debug("Looking up filename provided repo") 30 | 31 | repo, resp, err := GithubClient.Repositories.Get(context.Background(), allowedRepo.Organization, allowedRepo.Name) 32 | 33 | if err != nil { 34 | logger.WithFields(logrus.Fields{ 35 | "Error": err, 36 | "Response Status Code": resp.StatusCode, 37 | "AllowedRepoOwner": allowedRepo.Organization, 38 | "AllowedRepoName": allowedRepo.Name, 39 | }).Debug("error getting single repo") 40 | 41 | if resp.StatusCode == 404 { 42 | // This repo does not exist / could not be fetched as named, so we won't include it in the list of repos to process 43 | 44 | // create an empty GitHub repo object to satisfy the stats tracking interface 45 | missingRepo := &github.Repository{ 46 | Owner: &github.User{Login: github.String(allowedRepo.Organization)}, 47 | Name: github.String(allowedRepo.Name), 48 | } 49 | tracker.TrackSingle(stats.RepoNotExists, missingRepo) 50 | continue 51 | } else { 52 | return allRepos, errors.WithStackTrace(err) 53 | } 54 | } 55 | 56 | if resp.StatusCode == 200 { 57 | logger.WithFields(logrus.Fields{ 58 | "Organization": allowedRepo.Organization, 59 | "Name": allowedRepo.Name, 60 | }).Debug("Successfully fetched repo") 61 | 62 | allRepos = append(allRepos, repo) 63 | } 64 | } 65 | return allRepos, nil 66 | } 67 | 68 | // getReposByOrg takes the string name of a GitHub organization and pages through the API to fetch all of its repositories 69 | func getReposByOrg(config *config.GitXargsConfig) ([]*github.Repository, error) { 70 | 71 | logger := logging.GetLogger("git-xargs") 72 | 73 | // Page through all of the organization's repos, collecting them in this slice 74 | var allRepos []*github.Repository 75 | 76 | if config.GithubOrg == "" { 77 | return allRepos, errors.WithStackTrace(types.NoGithubOrgSuppliedErr{}) 78 | } 79 | 80 | opt := &github.RepositoryListByOrgOptions{ 81 | ListOptions: github.ListOptions{ 82 | PerPage: 100, 83 | }, 84 | } 85 | 86 | for { 87 | var reposToAdd []*github.Repository 88 | repos, resp, err := config.GithubClient.Repositories.ListByOrg(context.Background(), config.GithubOrg, opt) 89 | if err != nil { 90 | return allRepos, errors.WithStackTrace(err) 91 | } 92 | 93 | // github.RepositoryListByOrgOptions doesn't seem to be able to filter out archived repos 94 | // So filter the repos list if --skip-archived-repos is passed and the repository is in archived/read-only state 95 | if config.SkipArchivedRepos { 96 | for _, repo := range repos { 97 | if repo.GetArchived() { 98 | logger.WithFields(logrus.Fields{ 99 | "Name": repo.GetFullName(), 100 | }).Debug("Skipping archived repository") 101 | 102 | // Track repos to skip because of archived status for our final run report 103 | config.Stats.TrackSingle(stats.ReposArchivedSkipped, repo) 104 | } else { 105 | reposToAdd = append(reposToAdd, repo) 106 | } 107 | } 108 | } else { 109 | reposToAdd = repos 110 | } 111 | 112 | allRepos = append(allRepos, reposToAdd...) 113 | 114 | if resp.NextPage == 0 { 115 | break 116 | } 117 | opt.Page = resp.NextPage 118 | } 119 | 120 | repoCount := len(allRepos) 121 | 122 | if repoCount == 0 { 123 | return nil, errors.WithStackTrace(types.NoReposFoundErr{GithubOrg: config.GithubOrg}) 124 | } 125 | 126 | logger.WithFields(logrus.Fields{ 127 | "Repo count": repoCount, 128 | }).Debug(fmt.Sprintf("Fetched repos from Github organization: %s", config.GithubOrg)) 129 | 130 | config.Stats.TrackMultiple(stats.FetchedViaGithubAPI, allRepos) 131 | 132 | return allRepos, nil 133 | } 134 | -------------------------------------------------------------------------------- /repository/fetch-repos_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/git-xargs/config" 7 | "github.com/gruntwork-io/git-xargs/mocks" 8 | "github.com/gruntwork-io/git-xargs/types" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | // TestGetFileDefinedRepos provides static allowedRepos input to the getFileDefined repos, ensuring that method returns 13 | // all valid repos passed to it 14 | func TestGetFileDefinedRepos(t *testing.T) { 15 | t.Parallel() 16 | 17 | config := config.NewGitXargsTestConfig() 18 | config.GithubClient = mocks.ConfigureMockGithubClient() 19 | 20 | allowedRepos := []*types.AllowedRepo{ 21 | &types.AllowedRepo{ 22 | Organization: "gruntwork-io", 23 | Name: "cloud-nuke", 24 | }, 25 | &types.AllowedRepo{ 26 | Organization: "gruntwork-io", 27 | Name: "fetch", 28 | }, 29 | &types.AllowedRepo{ 30 | Organization: "gruntwork-io", 31 | Name: "terratest", 32 | }, 33 | } 34 | 35 | githubRepos, reposLookupErr := getFileDefinedRepos(config.GithubClient, allowedRepos, config.Stats) 36 | 37 | assert.Equal(t, len(githubRepos), len(allowedRepos)) 38 | assert.NoError(t, reposLookupErr) 39 | } 40 | 41 | // TestGetReposByOrg ensures that you can pass a configuration specifying repo look up by GitHub Org to getReposByOrg 42 | func TestGetReposByOrg(t *testing.T) { 43 | t.Parallel() 44 | 45 | config := config.NewGitXargsTestConfig() 46 | config.GithubOrg = "gruntwork-io" 47 | config.GithubClient = mocks.ConfigureMockGithubClient() 48 | 49 | githubRepos, reposByOrgLookupErr := getReposByOrg(config) 50 | 51 | assert.Equal(t, len(githubRepos), len(mocks.MockGithubRepositories)) 52 | assert.NoError(t, reposByOrgLookupErr) 53 | } 54 | 55 | // TestSkipArchivedRepos ensures that you can filter out archived repositories 56 | func TestSkipArchivedRepos(t *testing.T) { 57 | t.Parallel() 58 | 59 | config := config.NewGitXargsTestConfig() 60 | config.GithubOrg = "gruntwork-io" 61 | config.SkipArchivedRepos = true 62 | config.GithubClient = mocks.ConfigureMockGithubClient() 63 | 64 | githubRepos, reposByOrgLookupErr := getReposByOrg(config) 65 | 66 | assert.Equal(t, len(githubRepos), len(mocks.MockGithubRepositories)-2) 67 | assert.NoError(t, reposByOrgLookupErr) 68 | } 69 | -------------------------------------------------------------------------------- /repository/process.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | "time" 7 | 8 | "github.com/google/go-github/v43/github" 9 | "github.com/gruntwork-io/git-xargs/config" 10 | "github.com/gruntwork-io/git-xargs/types" 11 | "github.com/gruntwork-io/go-commons/logging" 12 | "github.com/pterm/pterm" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | // openPullRequestsWithThrottling calls the method to open a pull request after waiting on the internal ticker channel, which 17 | // reflects the value of the --seconds-between-prs flag 18 | func openPullRequestsWithThrottling(gitxargsConfig *config.GitXargsConfig, pr types.OpenPrRequest) error { 19 | logger := logging.GetLogger("git-xargs") 20 | logger.Debugf("pullRequestWorker received pull request job. Delay: %d seconds. Retries: %d for repo: %s on branch: %s\n", pr.Delay, pr.Retries, pr.Repo.GetName(), pr.Branch) 21 | 22 | // Space out open PR calls to GitHub API to avoid being aggressively rate-limited. By waiting on the ticker, 23 | // we ensure we're staggering our calls by the number of seconds specified by the seconds-between-prs flag, or 24 | // the default value of 1 second. This behavior is explicitly requested by GitHub API's integrator guidelines 25 | <-gitxargsConfig.Ticker.C 26 | if pr.Delay != 0 { 27 | logger.Debugf("Throttled pull request worker delaying %d seconds before attempting to re-open pr against repo: %s", pr.Delay, pr.Repo.GetName()) 28 | time.Sleep(time.Duration(pr.Delay) * time.Second) 29 | } 30 | // Make pull request. Errors are handled within the method itself 31 | return openPullRequest(gitxargsConfig, pr) 32 | } 33 | 34 | // ProcessRepos loops through every repo we've selected and uses a WaitGroup so that the processing can happen in parallel. 35 | 36 | // We process all work that can be done up to the open pull request API call in parallel 37 | // However, we then separately process all open pull request jobs, through the PRChan. We do this so that 38 | // we can insert a configurable buffer of time between open pull request API calls, which must be staggered to avoid tripping 39 | // the GitHub API's rate limiting mechanisms 40 | // See https://github.com/gruntwork-io/git-xargs/issues/53 for more information 41 | func ProcessRepos(gitxargsConfig *config.GitXargsConfig, repos []*github.Repository) error { 42 | logger := logging.GetLogger("git-xargs") 43 | 44 | p, progressBarErr := pterm.DefaultProgressbar.WithTotal(len(repos)).WithTitle("Processing repos").Start() 45 | if progressBarErr != nil { 46 | return progressBarErr 47 | } 48 | 49 | wg := &sync.WaitGroup{} 50 | wg.Add(len(repos)) 51 | 52 | for _, repo := range repos { 53 | go func(gitxargsConfig *config.GitXargsConfig, repo *github.Repository) error { 54 | defer p.Increment() 55 | defer wg.Done() 56 | 57 | // For each repo, run the supplied command against it and, if it succeeds without error, 58 | // commit the changes, push the local branch to remote and use the GitHub API to open a pr 59 | processErr := processRepo(gitxargsConfig, repo) 60 | if processErr != nil { 61 | logger.WithFields(logrus.Fields{ 62 | "Repo name": repo.GetName(), "Error": processErr, 63 | }).Debug("Error encountered while processing repo") 64 | } 65 | 66 | return processErr 67 | }(gitxargsConfig, repo) 68 | } 69 | wg.Wait() 70 | 71 | return nil 72 | } 73 | 74 | // cleanupTempDir removes the temporary directory that was created for the local clone of the repo 75 | // It logs a debug-level error if the directory could not be removed, but does not return an error 76 | func cleanupTempDir(repositoryDir string) { 77 | logger := logging.GetLogger("git-xargs") 78 | removeErr := os.RemoveAll(repositoryDir) 79 | if removeErr != nil { 80 | logger.WithFields(logrus.Fields{ 81 | "Error": removeErr, 82 | "Temp directory": repositoryDir, 83 | }).Debug("Error encountered while removing temporary directory") 84 | } 85 | } 86 | 87 | // 1. Attempt to clone it to the local filesystem. To avoid conflicts, this generates a new directory for each repo FOR EACH run, so heavy use of this tool may inflate your /tmp/ directory size 88 | // 2. Look up the HEAD ref of the repo, and create a new branch from that ref, specific to this tool so that we can safely make our changes in the branch 89 | // 3. Execute the supplied command against the locally cloned repo 90 | // 4. Look up any worktree changes (deleted files, modified files, new and untracked files) and ADD THEM ALL to the git stage 91 | // 5. Commit these changes with the optionally configurable git commit message, or fall back to the default if it was not provided by the user 92 | // 6. Push the branch containing the new commit to the remote origin 93 | // 7. Via the GitHub API, open a pull request of the newly pushed branch against the main branch of the repo 94 | // 8. Track all successfully opened pull requests via the stats tracker so that we can print them out as part of our final 95 | // run report that is displayed in table format to the operator following each run 96 | func processRepo(config *config.GitXargsConfig, repo *github.Repository) error { 97 | logger := logging.GetLogger("git-xargs") 98 | 99 | // Create a new temporary directory in the default temp directory of the system, but append 100 | // git-xargs- to it so that it's easier to find when you're looking for it 101 | repositoryDir, localRepository, cloneErr := cloneLocalRepository(config, repo) 102 | 103 | // if user did not pass retention flag, defer cleanup of the repositoryDir 104 | if config.RetainLocalRepos == false { 105 | defer cleanupTempDir(repositoryDir) 106 | } 107 | 108 | if cloneErr != nil { 109 | return cloneErr 110 | } 111 | 112 | // Get HEAD ref from the repo 113 | ref, headRefErr := getLocalRepoHeadRef(config, localRepository, repo) 114 | if headRefErr != nil { 115 | return headRefErr 116 | } 117 | 118 | // Get the worktree for the given local repository, so we can examine any changes made by script operations 119 | worktree, worktreeErr := getLocalWorkTree(repositoryDir, localRepository, repo) 120 | 121 | if worktreeErr != nil { 122 | return worktreeErr 123 | } 124 | 125 | // Create a branch in the locally cloned copy of the repo to hold all the changes that may result from script execution 126 | // Also, attempt to pull the latest from the remote branch if it exists 127 | branchName, branchErr := checkoutLocalBranch(config, ref, worktree, repo, localRepository) 128 | if branchErr != nil { 129 | return branchErr 130 | } 131 | 132 | // Run the specified command 133 | commandErr := executeCommand(config, repositoryDir, repo) 134 | if commandErr != nil { 135 | return commandErr 136 | } 137 | 138 | // Commit and push the changes to Git and open a PR 139 | if err := updateRepo(config, repositoryDir, worktree, repo, localRepository, branchName.String()); err != nil { 140 | return err 141 | } 142 | 143 | logger.WithFields(logrus.Fields{ 144 | "Repo name": repo.GetName(), 145 | }).Debug("Repository successfully processed") 146 | 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /repository/process_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "os/exec" 5 | "testing" 6 | 7 | "github.com/gruntwork-io/git-xargs/config" 8 | "github.com/gruntwork-io/git-xargs/mocks" 9 | "github.com/gruntwork-io/git-xargs/util" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | // TestProcessRepo smoke tests the processRepo function with a basic test config - however, the MockGitProvider implemented 14 | // in git_test.go intercepts the call to git.PlainClone to modify the repo URL to the local checkout of gruntwork-io/fetch 15 | // which is bundled in data/test to allow tests to run against an actual repository without making any network calls or pushes to actual remote repositories 16 | func TestProcessRepo(t *testing.T) { 17 | t.Parallel() 18 | 19 | // Hackily create a simple git repo at ./testdata/test-repo if it doesn't already exist 20 | cmd := exec.Command("bash", "-c", "mkdir -p test-repo && cd test-repo && git init && touch README.md && git add README.md && git commit -m \"Add README.md\"") 21 | cmd.Dir = "../data/test/" 22 | cmdOut, err := cmd.CombinedOutput() 23 | if err != nil { 24 | t.Logf("Error creating test git repo at ../data/test/test-repo: +%v\n", err) 25 | t.Log(string(cmdOut)) 26 | } else { 27 | t.Log("TestProcessRepo Successfully created test git repo at ../data/test/test-repo") 28 | } 29 | 30 | testConfig := config.NewGitXargsTestConfig() 31 | testConfig.Args = []string{"touch", util.NewTestFileName()} 32 | testConfig.GithubClient = mocks.ConfigureMockGithubClient() 33 | 34 | // The GitXargsConfig object uses an unbuffered channel to send pull request messages 35 | // so we need to listen for the PR messages in this test so that we don't block the channel 36 | // which would deadlock this test - we also don't need to make the PR requests themselves 37 | // in this test, we can discard them instead 38 | go func() { 39 | for { 40 | select { 41 | case pr := <-testConfig.PRChan: 42 | _ = pr 43 | } 44 | } 45 | }() 46 | 47 | defer close(testConfig.PRChan) 48 | 49 | // Run a command to delete all local branches in the "../data/test/test-repo" repo to avoid the git-xargs repo 50 | // growing in size over time with test data 51 | defer cleanupLocalTestRepoChanges(t, testConfig) 52 | 53 | processErr := processRepo(testConfig, mocks.GetMockGithubRepo()) 54 | assert.NoError(t, processErr) 55 | } 56 | 57 | func cleanupLocalTestRepoChanges(t *testing.T, config *config.GitXargsConfig) { 58 | t.Log("cleanupLocalTestRepoChanges deleting branches in local test repo to avoid bloat...") 59 | // Force delete all of the branches that are not either "master" or "main" 60 | cmd := exec.Command("bash", "-c", "git branch | grep -v 'master' | grep -v '*' | xargs -r git branch -D") 61 | cmd.Dir = "../data/test/test-repo" 62 | cmdOut, err := cmd.CombinedOutput() 63 | t.Log(string(cmdOut)) 64 | if err != nil { 65 | t.Logf("cleanupLocalTestRepoChanges error deleting test branches: %+v\n", err) 66 | } else { 67 | t.Log("cleanupLocalTestRepoChanges successfully deleted branches in local test repo") 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /repository/repo-operations.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "os/exec" 10 | "strings" 11 | "time" 12 | 13 | "github.com/go-git/go-git/v5" 14 | "github.com/go-git/go-git/v5/plumbing" 15 | "github.com/go-git/go-git/v5/plumbing/transport/http" 16 | "github.com/sirupsen/logrus" 17 | 18 | "github.com/google/go-github/v43/github" 19 | 20 | "github.com/gruntwork-io/git-xargs/common" 21 | "github.com/gruntwork-io/git-xargs/config" 22 | "github.com/gruntwork-io/git-xargs/stats" 23 | "github.com/gruntwork-io/git-xargs/types" 24 | "github.com/gruntwork-io/go-commons/errors" 25 | "github.com/gruntwork-io/go-commons/logging" 26 | ) 27 | 28 | // cloneLocalRepository clones a remote GitHub repo via SSH to a local temporary directory so that the supplied command 29 | // can be run against the repo locally and any git changes handled thereafter. The local directory has 30 | // git-xargs- appended to it to make it easier to find when you are looking for it while debugging 31 | func cloneLocalRepository(config *config.GitXargsConfig, repo *github.Repository) (string, *git.Repository, error) { 32 | logger := logging.GetLogger("git-xargs") 33 | config.CloneJobsLimiter <- struct{}{} 34 | 35 | logger.WithFields(logrus.Fields{ 36 | "Repo": repo.GetName(), 37 | }).Debug("Attempting to clone repository using GITHUB_OAUTH_TOKEN") 38 | 39 | repositoryDir, tmpDirErr := ioutil.TempDir("", fmt.Sprintf("git-xargs-%s", repo.GetName())) 40 | if tmpDirErr != nil { 41 | logger.WithFields(logrus.Fields{ 42 | "Error": tmpDirErr, 43 | "Repo": repo.GetName(), 44 | }).Debug("Failed to create temporary directory to hold repo") 45 | return repositoryDir, nil, errors.WithStackTrace(tmpDirErr) 46 | } 47 | 48 | gitProgressBuffer := bytes.NewBuffer(nil) 49 | localRepository, err := config.GitClient.PlainClone(repositoryDir, false, &git.CloneOptions{ 50 | URL: repo.GetCloneURL(), 51 | Progress: gitProgressBuffer, 52 | Auth: &http.BasicAuth{ 53 | Username: repo.GetOwner().GetLogin(), 54 | Password: os.Getenv("GITHUB_OAUTH_TOKEN"), 55 | }, 56 | }) 57 | 58 | logger.WithFields(logrus.Fields{ 59 | "Repo": repo.GetName(), 60 | }).Debug(gitProgressBuffer) 61 | 62 | <-config.CloneJobsLimiter 63 | 64 | if err != nil { 65 | logger.WithFields(logrus.Fields{ 66 | "Error": err, 67 | "Repo": repo.GetName(), 68 | }).Debug("Error cloning repository") 69 | 70 | // Track failure to clone for our final run report 71 | config.Stats.TrackSingle(stats.RepoFailedToClone, repo) 72 | 73 | return repositoryDir, nil, errors.WithStackTrace(err) 74 | } 75 | 76 | config.Stats.TrackSingle(stats.RepoSuccessfullyCloned, repo) 77 | 78 | return repositoryDir, localRepository, nil 79 | } 80 | 81 | // getLocalRepoHeadRef looks up the HEAD reference of the locally cloned git repository, which is required by 82 | // downstream operations such as branching 83 | func getLocalRepoHeadRef(config *config.GitXargsConfig, localRepository *git.Repository, repo *github.Repository) (*plumbing.Reference, error) { 84 | logger := logging.GetLogger("git-xargs") 85 | 86 | ref, headErr := localRepository.Head() 87 | if headErr != nil { 88 | logger.WithFields(logrus.Fields{ 89 | "Error": headErr, 90 | "Repo": repo.GetName(), 91 | }).Debug("Error getting HEAD ref from local repo") 92 | 93 | config.Stats.TrackSingle(stats.GetHeadRefFailed, repo) 94 | 95 | return nil, errors.WithStackTrace(headErr) 96 | } 97 | return ref, nil 98 | } 99 | 100 | // executeCommand runs the user-supplied command against the given repository 101 | func executeCommand(config *config.GitXargsConfig, repositoryDir string, repo *github.Repository) error { 102 | return executeCommandWithLogger(config, repositoryDir, repo, logging.GetLogger("git-xargs")) 103 | } 104 | 105 | // executeCommandWithLogger runs the user-supplied command against the given repository, and sends the log output 106 | // to the given logger 107 | func executeCommandWithLogger(config *config.GitXargsConfig, repositoryDir string, repo *github.Repository, logger *logrus.Logger) error { 108 | if len(config.Args) < 1 { 109 | return errors.WithStackTrace(types.NoCommandSuppliedErr{}) 110 | } 111 | 112 | cmdArgs := config.Args 113 | 114 | cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) 115 | cmd.Dir = repositoryDir 116 | cmd.Env = os.Environ() 117 | cmd.Env = append(cmd.Env, fmt.Sprintf("XARGS_DRY_RUN=%t", config.DryRun)) 118 | cmd.Env = append(cmd.Env, fmt.Sprintf("XARGS_REPO_NAME=%s", repo.GetName())) 119 | cmd.Env = append(cmd.Env, fmt.Sprintf("XARGS_REPO_OWNER=%s", repo.GetOwner().GetLogin())) 120 | 121 | logger.WithFields(logrus.Fields{ 122 | "Repo": repo.GetName(), 123 | "Directory": repositoryDir, 124 | "Command": config.Args, 125 | }).Debug("Executing command against local clone of repo...") 126 | 127 | stdoutStdErr, err := cmd.CombinedOutput() 128 | 129 | logger.Debugf("Output of command %v for repo %s in directory %s:\n%s", config.Args, repo.GetName(), repositoryDir, string(stdoutStdErr)) 130 | 131 | if err != nil { 132 | logger.WithFields(logrus.Fields{ 133 | "Error": err, 134 | }).Debug("Error getting output of command execution") 135 | // Track the command error against the repo 136 | config.Stats.TrackSingle(stats.CommandErrorOccurredDuringExecution, repo) 137 | return errors.WithStackTrace(err) 138 | } 139 | 140 | return nil 141 | } 142 | 143 | // getLocalWorkTree looks up the working tree of the locally cloned repository and returns it if possible, or an error 144 | func getLocalWorkTree(repositoryDir string, localRepository *git.Repository, repo *github.Repository) (*git.Worktree, error) { 145 | logger := logging.GetLogger("git-xargs") 146 | 147 | worktree, worktreeErr := localRepository.Worktree() 148 | 149 | if worktreeErr != nil { 150 | logger.WithFields(logrus.Fields{ 151 | "Error": worktreeErr, 152 | "Repo": repo.GetName(), 153 | "Dir": repositoryDir, 154 | }).Debug("Error looking up local repository's worktree") 155 | 156 | return nil, errors.WithStackTrace(worktreeErr) 157 | } 158 | return worktree, nil 159 | } 160 | 161 | // checkoutLocalBranch creates a local branch specific to this tool in the locally checked out copy of the repo in the /tmp folder 162 | func checkoutLocalBranch(config *config.GitXargsConfig, ref *plumbing.Reference, worktree *git.Worktree, remoteRepository *github.Repository, localRepository *git.Repository) (plumbing.ReferenceName, error) { 163 | logger := logging.GetLogger("git-xargs") 164 | 165 | // BranchName is a global variable that is set in cmd/root.go. It is override-able by the operator via the --branch-name or -b flag. It defaults to "git-xargs" 166 | 167 | branchName := plumbing.NewBranchReferenceName(config.BranchName) 168 | logger.WithFields(logrus.Fields{ 169 | "Branch Name": branchName, 170 | "Repo": remoteRepository.GetName(), 171 | }).Debug("Created branch") 172 | 173 | // Create a branch specific to the multi repo script runner 174 | co := &git.CheckoutOptions{ 175 | Hash: ref.Hash(), 176 | Branch: branchName, 177 | Create: true, 178 | } 179 | 180 | // Attempt to checkout the new tool-specific branch on which the supplied command will be executed 181 | checkoutErr := worktree.Checkout(co) 182 | 183 | if checkoutErr != nil { 184 | if config.SkipPullRequests && 185 | remoteRepository.GetDefaultBranch() == config.BranchName && 186 | strings.Contains(checkoutErr.Error(), "already exists") { 187 | // User has requested pull requess be skipped, meaning they want their commits pushed on their target branch 188 | // If the target branch is also the repo's default branch and therefore already exists, we don't have an error 189 | } else { 190 | logger.WithFields(logrus.Fields{ 191 | "Error": checkoutErr, 192 | "Repo": remoteRepository.GetName(), 193 | }).Debug("Error creating new branch") 194 | 195 | // Track the error checking out the branch 196 | config.Stats.TrackSingle(stats.BranchCheckoutFailed, remoteRepository) 197 | 198 | return branchName, errors.WithStackTrace(checkoutErr) 199 | } 200 | } 201 | 202 | // Pull latest code from remote branch if it exists to avoid fast-forwarding errors 203 | gitProgressBuffer := bytes.NewBuffer(nil) 204 | po := &git.PullOptions{ 205 | RemoteName: "origin", 206 | ReferenceName: branchName, 207 | Auth: &http.BasicAuth{ 208 | Username: remoteRepository.GetOwner().GetLogin(), 209 | Password: os.Getenv("GITHUB_OAUTH_TOKEN"), 210 | }, 211 | Progress: gitProgressBuffer, 212 | } 213 | 214 | logger.WithFields(logrus.Fields{ 215 | "Repo": remoteRepository.GetName(), 216 | }).Debug(gitProgressBuffer) 217 | 218 | pullErr := worktree.Pull(po) 219 | 220 | if pullErr != nil { 221 | 222 | if pullErr == plumbing.ErrReferenceNotFound { 223 | // The supplied branch just doesn't exist yet on the remote - this is not a fatal error and will 224 | // allow the new branch to be pushed in pushLocalBranch 225 | config.Stats.TrackSingle(stats.BranchRemoteDidntExistYet, remoteRepository) 226 | return branchName, nil 227 | } 228 | 229 | if pullErr == git.NoErrAlreadyUpToDate { 230 | // The local branch is already up to date, which is not a fatal error 231 | return branchName, nil 232 | } 233 | 234 | // Track the error pulling the latest from the remote branch 235 | config.Stats.TrackSingle(stats.BranchRemotePullFailed, remoteRepository) 236 | 237 | return branchName, errors.WithStackTrace(pullErr) 238 | } 239 | 240 | return branchName, nil 241 | } 242 | 243 | // updateRepo will check for any changes in worktree as a result of script execution, and if any are present, 244 | // add any untracked, deleted or modified files, create a commit using the supplied or default commit message, 245 | // push the code to the remote repo, and open a pull request. 246 | func updateRepo(config *config.GitXargsConfig, 247 | repositoryDir string, 248 | worktree *git.Worktree, 249 | remoteRepository *github.Repository, 250 | localRepository *git.Repository, 251 | branchName string, 252 | ) error { 253 | logger := logging.GetLogger("git-xargs") 254 | 255 | status, statusErr := worktree.Status() 256 | 257 | if statusErr != nil { 258 | logger.WithFields(logrus.Fields{ 259 | "Error": statusErr, 260 | "Repo": remoteRepository.GetName(), 261 | "Dir": repositoryDir, 262 | }).Debug("Error looking up worktree status") 263 | 264 | // Track the status check failure 265 | config.Stats.TrackSingle(stats.WorktreeStatusCheckFailedCommand, remoteRepository) 266 | return errors.WithStackTrace(statusErr) 267 | } 268 | 269 | // If there are no changes, we log it, track it, and return 270 | if status.IsClean() { 271 | logger.WithFields(logrus.Fields{ 272 | "Repo": remoteRepository.GetName(), 273 | }).Debug("Local repository status is clean - nothing to stage or commit") 274 | 275 | // Track the fact that repo had no file changes post command execution 276 | config.Stats.TrackSingle(stats.WorktreeStatusClean, remoteRepository) 277 | return nil 278 | } 279 | 280 | // Commit any untracked files, modified or deleted files that resulted from script execution 281 | commitErr := commitLocalChanges(status, config, repositoryDir, worktree, remoteRepository, localRepository) 282 | if commitErr != nil { 283 | return commitErr 284 | } 285 | 286 | // Push the local branch containing all of our changes from executing the supplied command 287 | pushBranchErr := pushLocalBranch(config, remoteRepository, localRepository) 288 | if pushBranchErr != nil { 289 | return pushBranchErr 290 | } 291 | 292 | // Create an OpenPrRequest that tracks retries 293 | opr := types.OpenPrRequest{ 294 | Repo: remoteRepository, 295 | Branch: branchName, 296 | Retries: 0, 297 | } 298 | 299 | return openPullRequestsWithThrottling(config, opr) 300 | } 301 | 302 | // commitLocalChanges will check for any changes in worktree as a result of script execution, and if any are present, 303 | // add any untracked, deleted or modified files and create a commit using the supplied or default commit message. 304 | func commitLocalChanges(status git.Status, config *config.GitXargsConfig, repositoryDir string, worktree *git.Worktree, remoteRepository *github.Repository, localRepository *git.Repository) error { 305 | logger := logging.GetLogger("git-xargs") 306 | 307 | // If there are changes, we need to stage, add and commit them 308 | logger.WithFields(logrus.Fields{ 309 | "Repo": remoteRepository.GetName(), 310 | }).Debug("Local repository worktree no longer clean, will stage and add new files and commit changes") 311 | 312 | // Track the fact that worktree changes were made following execution 313 | config.Stats.TrackSingle(stats.WorktreeStatusDirty, remoteRepository) 314 | 315 | for filepath := range status { 316 | if status.IsUntracked(filepath) { 317 | logger.WithFields(logrus.Fields{ 318 | "Filepath": filepath, 319 | }).Debug("Found untracked file. Adding to stage") 320 | 321 | _, addErr := worktree.Add(filepath) 322 | if addErr != nil { 323 | logger.WithFields(logrus.Fields{ 324 | "Error": addErr, 325 | "Filepath": filepath, 326 | }).Debug("Error adding file to git stage") 327 | // Track the file staging failure 328 | config.Stats.TrackSingle(stats.WorktreeAddFileFailed, remoteRepository) 329 | return errors.WithStackTrace(addErr) 330 | } 331 | } 332 | } 333 | 334 | // With all our untracked files staged, we can now create a commit, passing the All 335 | // option when configuring our commit option so that all modified and deleted files 336 | // will have their changes committed 337 | commitOps := &git.CommitOptions{ 338 | All: true, 339 | } 340 | 341 | _, commitErr := worktree.Commit(config.CommitMessage, commitOps) 342 | 343 | if commitErr != nil { 344 | logger.WithFields(logrus.Fields{ 345 | "Error": commitErr, 346 | "Repo": remoteRepository.GetName(), 347 | }) 348 | 349 | // If we reach this point, we were unable to commit our changes, so we'll 350 | // continue rather than attempt to push an empty branch and open an empty PR 351 | config.Stats.TrackSingle(stats.CommitChangesFailed, remoteRepository) 352 | return errors.WithStackTrace(commitErr) 353 | } 354 | 355 | // If --skip-pull-requests was passed, track the repos whose changes were committed directly to the main branch 356 | if config.SkipPullRequests { 357 | config.Stats.TrackSingle(stats.CommitsMadeDirectlyToBranch, remoteRepository) 358 | } 359 | 360 | return nil 361 | } 362 | 363 | // pushLocalBranch pushes the branch in the local clone of the /tmp/ directory repository to the GitHub remote origin 364 | // so that a pull request can be opened against it via the GitHub API 365 | func pushLocalBranch(config *config.GitXargsConfig, remoteRepository *github.Repository, localRepository *git.Repository) error { 366 | logger := logging.GetLogger("git-xargs") 367 | 368 | if config.DryRun { 369 | logger.WithFields(logrus.Fields{ 370 | "Repo": remoteRepository.GetName(), 371 | }).Debug("Skipping branch push to remote origin because --dry-run flag is set") 372 | 373 | config.Stats.TrackSingle(stats.PushBranchSkipped, remoteRepository) 374 | return nil 375 | } 376 | // Push the changes to the remote repo 377 | po := &git.PushOptions{ 378 | RemoteName: "origin", 379 | Auth: &http.BasicAuth{ 380 | Username: remoteRepository.GetOwner().GetLogin(), 381 | Password: os.Getenv("GITHUB_OAUTH_TOKEN"), 382 | }, 383 | } 384 | pushErr := localRepository.Push(po) 385 | 386 | if pushErr != nil { 387 | logger.WithFields(logrus.Fields{ 388 | "Error": pushErr, 389 | "Repo": remoteRepository.GetName(), 390 | }).Debug("Error pushing new branch to remote origin") 391 | 392 | // Track the push failure 393 | config.Stats.TrackSingle(stats.PushBranchFailed, remoteRepository) 394 | return errors.WithStackTrace(pushErr) 395 | } 396 | 397 | logger.WithFields(logrus.Fields{ 398 | "Repo": remoteRepository.GetName(), 399 | }).Debug("Successfully pushed local branch to remote origin") 400 | 401 | // If --skip-pull-requests was passed, track the fact that these changes were pushed directly to the main branch 402 | if config.SkipPullRequests { 403 | config.Stats.TrackSingle(stats.DirectCommitsPushedToRemoteBranch, remoteRepository) 404 | } 405 | 406 | return nil 407 | } 408 | 409 | // Attempt to open a pull request via the GitHub API, of the supplied branch specific to this tool, against the main 410 | // branch for the remote origin 411 | func openPullRequest(config *config.GitXargsConfig, pr types.OpenPrRequest) error { 412 | logger := logging.GetLogger("git-xargs") 413 | 414 | // If the current request has already exhausted the configured number of PR retries, short-circuit 415 | if pr.Retries > config.PullRequestRetries { 416 | config.Stats.TrackSingle(stats.PRFailedAfterMaximumRetriesErr, pr.Repo) 417 | return nil 418 | } 419 | 420 | if config.DryRun || config.SkipPullRequests { 421 | logger.WithFields(logrus.Fields{ 422 | "Repo": pr.Repo.GetName(), 423 | }).Debug("--dry-run and / or --skip-pull-requests is set to true, so skipping opening a pull request!") 424 | return nil 425 | } 426 | 427 | logger.Debugf("openPullRequest received job with retries: %d. Config max retries for this run: %d", pr.Retries, config.PullRequestRetries) 428 | 429 | repoDefaultBranch := config.BaseBranchName 430 | if repoDefaultBranch == "" { 431 | repoDefaultBranch = pr.Repo.GetDefaultBranch() 432 | } 433 | 434 | pullRequestAlreadyExists, err := pullRequestAlreadyExistsForBranch(config, pr.Repo, pr.Branch, repoDefaultBranch) 435 | if err != nil { 436 | logger.WithFields(logrus.Fields{ 437 | "Error": err, 438 | "Head": pr.Branch, 439 | "Base": repoDefaultBranch, 440 | }).Debug("Error listing pull requests") 441 | 442 | // Track pull request open failure 443 | config.Stats.TrackSingle(stats.PullRequestOpenErr, pr.Repo) 444 | return errors.WithStackTrace(err) 445 | } 446 | 447 | if pullRequestAlreadyExists { 448 | logger.WithFields(logrus.Fields{ 449 | "Repo": pr.Repo.GetName(), 450 | "Head": pr.Branch, 451 | "Base": repoDefaultBranch, 452 | }).Debug("Pull request already exists for this branch, so skipping opening a pull request!") 453 | 454 | // Track that we skipped opening a pull request 455 | config.Stats.TrackSingle(stats.PullRequestAlreadyExists, pr.Repo) 456 | return nil 457 | } 458 | 459 | // If the user only supplies a commit message, use that for both the pull request title and descriptions, 460 | // unless they are provided separately 461 | titleToUse := config.PullRequestTitle 462 | descriptionToUse := config.PullRequestDescription 463 | 464 | commitMessage := config.CommitMessage 465 | 466 | if commitMessage != common.DefaultCommitMessage { 467 | if titleToUse == common.DefaultPullRequestTitle { 468 | titleToUse = commitMessage 469 | } 470 | 471 | if descriptionToUse == common.DefaultPullRequestDescription { 472 | descriptionToUse = commitMessage 473 | } 474 | } 475 | 476 | // Configure pull request options that the GitHub client accepts when making calls to open new pull requests 477 | newPR := &github.NewPullRequest{ 478 | Title: github.String(titleToUse), 479 | Head: github.String(pr.Branch), 480 | Base: github.String(repoDefaultBranch), 481 | Body: github.String(descriptionToUse), 482 | MaintainerCanModify: github.Bool(true), 483 | Draft: github.Bool(config.Draft), 484 | } 485 | 486 | // Make a pull request via the Github API 487 | githubPR, resp, err := config.GithubClient.PullRequests.Create(context.Background(), *pr.Repo.GetOwner().Login, pr.Repo.GetName(), newPR) 488 | 489 | // The go-github library's CheckResponse method can return two different types of rate limiting error: 490 | // 1. AbuseRateLimitError which may contain a Retry-After header whose value we can use to slow down, or 491 | // 2. RateLimitError which may contain information about when the rate limit will be removed, that we can also use to slow down 492 | // Therefore, we need to use type assertions to test for each type of error response, and accordingly fetch the data it may contain 493 | // about how long git-xargs should wait before its next attempt to open a pull request 494 | githubErr := github.CheckResponse(resp.Response) 495 | 496 | if githubErr != nil { 497 | 498 | isRateLimited := false 499 | 500 | // Create a new open pull request struct that we'll eventually send on the PRChan 501 | opr := types.OpenPrRequest{ 502 | Repo: pr.Repo, 503 | Branch: pr.Branch, 504 | Retries: 1, 505 | } 506 | 507 | // If this request has been seen before, increment its retries count, taking into account previous iterations 508 | opr.Retries = (pr.Retries + 1) 509 | 510 | // If GitHub returned an error of type RateLimitError, we can attempt to compute the next time to try the request again 511 | // by reading its rate limit information 512 | if rateLimitError, ok := githubErr.(*github.RateLimitError); ok { 513 | isRateLimited = true 514 | retryAfter := time.Until(rateLimitError.Rate.Reset.Time) 515 | opr.Delay = retryAfter 516 | logger.Debugf("git-xargs parsed retryAfter %d from GitHub rate limit error's reset time", retryAfter) 517 | } 518 | 519 | // If GitHub returned a Retry-After header, use its value, otherwise use the default 520 | if abuseRateLimitError, ok := githubErr.(*github.AbuseRateLimitError); ok { 521 | isRateLimited = true 522 | if abuseRateLimitError.RetryAfter != nil { 523 | if abuseRateLimitError.RetryAfter.Seconds() > 0 { 524 | opr.Delay = *abuseRateLimitError.RetryAfter 525 | } 526 | } 527 | } 528 | 529 | if isRateLimited { 530 | // If we couldn't determine a more accurate delay from GitHub API response headers, then fall back to our user-configurable default 531 | if opr.Delay == 0 { 532 | opr.Delay = time.Duration(config.SecondsToSleepWhenRateLimited) 533 | } 534 | 535 | logger.Debugf("Retrying PR for repo: %s again later with %d second delay due to secondary rate limiting.", pr.Repo.GetName(), opr.Delay) 536 | // Put another pull request on the channel so this can effectively be retried after a cooldown 537 | 538 | // Keep track of the repo's PR initially failing due to rate limiting 539 | config.Stats.TrackSingle(stats.PRFailedDueToRateLimitsErr, pr.Repo) 540 | return openPullRequestsWithThrottling(config, opr) 541 | } 542 | } 543 | 544 | // Otherwise, if we reach this point, we can assume we are not rate limited, and hence must do some 545 | // further inspection on the error values returned to us to determine what went wrong 546 | prErrorMessage := "Error opening pull request" 547 | 548 | // Github's API will return HTTP status code 422 for several different errors 549 | // Currently, there are two such errors that git-xargs is concerned with: 550 | // 1. User passes the --draft flag, but the targeted repo does not support draft pull requests 551 | // 2. User passes the --base-branch-name flag, specifying a branch that does not exist in the repo 552 | if err != nil { 553 | if resp.StatusCode == 422 { 554 | switch { 555 | case strings.Contains(err.Error(), "Draft pull requests are not supported"): 556 | prErrorMessage = "Error opening pull request: draft PRs not supported for this repo. See https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests" 557 | config.Stats.TrackSingle(stats.RepoDoesntSupportDraftPullRequestsErr, pr.Repo) 558 | 559 | case strings.Contains(err.Error(), "Field:base Code:invalid"): 560 | prErrorMessage = fmt.Sprintf("Error opening pull request: Base branch name: %s is invalid", config.BaseBranchName) 561 | config.Stats.TrackSingle(stats.BaseBranchTargetInvalidErr, pr.Repo) 562 | 563 | default: 564 | config.Stats.TrackSingle(stats.PullRequestOpenErr, pr.Repo) 565 | } 566 | } 567 | 568 | // If the Github reponse's status code is not 422, fallback to logging and tracking a generic pull request error 569 | config.Stats.TrackSingle(stats.PullRequestOpenErr, pr.Repo) 570 | 571 | logger.WithFields(logrus.Fields{ 572 | "Error": err, 573 | "Head": pr.Branch, 574 | "Base": repoDefaultBranch, 575 | "Body": descriptionToUse, 576 | }).Debug(prErrorMessage) 577 | return errors.WithStackTrace(err) 578 | } 579 | 580 | // There was no error opening the pull request 581 | logger.WithFields(logrus.Fields{ 582 | "Pull Request URL": githubPR.GetHTMLURL(), 583 | }).Debug("Successfully opened pull request") 584 | 585 | reviewersRequest := github.ReviewersRequest{ 586 | NodeID: githubPR.NodeID, 587 | Reviewers: config.Reviewers, 588 | TeamReviewers: config.TeamReviewers, 589 | } 590 | 591 | // If the user supplied reviewer information on the pull request, initiate a separate request to ask for reviews 592 | if config.HasReviewers() { 593 | _, _, reviewRequestErr := config.GithubClient.PullRequests.RequestReviewers(context.Background(), *pr.Repo.GetOwner().Login, pr.Repo.GetName(), githubPR.GetNumber(), reviewersRequest) 594 | if reviewRequestErr != nil { 595 | config.Stats.TrackSingle(stats.RequestReviewersErr, pr.Repo) 596 | } 597 | 598 | } 599 | 600 | if config.Draft { 601 | config.Stats.TrackDraftPullRequest(pr.Repo.GetName(), githubPR.GetHTMLURL()) 602 | } else { 603 | // Track successful opening of the pull request, extracting the HTML url to the PR itself for easier review 604 | config.Stats.TrackPullRequest(pr.Repo.GetName(), githubPR.GetHTMLURL()) 605 | } 606 | return nil 607 | } 608 | 609 | // Returns true if a pull request already exists in the given repo for the given branch 610 | func pullRequestAlreadyExistsForBranch(config *config.GitXargsConfig, repo *github.Repository, branch string, repoDefaultBranch string) (bool, error) { 611 | opts := &github.PullRequestListOptions{ 612 | // Filter pulls by head user or head organization and branch name in the format of user:ref-name or organization:ref-name 613 | // https://docs.github.com/en/rest/reference/pulls#list-pull-requests 614 | Head: fmt.Sprintf("%s:%s", *repo.GetOwner().Login, branch), 615 | Base: repoDefaultBranch, 616 | } 617 | 618 | prs, _, err := config.GithubClient.PullRequests.List(context.Background(), *repo.GetOwner().Login, repo.GetName(), opts) 619 | if err != nil { 620 | return false, errors.WithStackTrace(err) 621 | } 622 | 623 | return len(prs) > 0, nil 624 | } 625 | -------------------------------------------------------------------------------- /repository/repo-operations_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/gruntwork-io/git-xargs/config" 9 | "github.com/sirupsen/logrus" 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/google/go-github/v43/github" 13 | ) 14 | 15 | func getMockGithubRepo() *github.Repository { 16 | userLogin := "gruntwork-io" 17 | user := &github.User{ 18 | Login: &userLogin, 19 | } 20 | 21 | repoName := "terragrunt" 22 | cloneURL := "https://github.com/gruntwork-io/terragrunt" 23 | 24 | repo := &github.Repository{ 25 | Owner: user, 26 | Name: &repoName, 27 | CloneURL: &cloneURL, 28 | } 29 | 30 | return repo 31 | } 32 | 33 | // Test that we can execute a script and that the expected stdout and stderr get written to the logger, even if that 34 | // script exits with an error (exit status 1). 35 | func TestExecuteCommandWithLogger(t *testing.T) { 36 | t.Parallel() 37 | 38 | cfg := config.NewGitXargsConfig() 39 | cfg.Args = []string{"../data/test/_testscripts/test-stdout-stderr.sh"} 40 | repo := getMockGithubRepo() 41 | 42 | var buffer bytes.Buffer 43 | logger := &logrus.Logger{ 44 | Out: &buffer, 45 | Level: logrus.TraceLevel, 46 | Formatter: new(logrus.TextFormatter), 47 | } 48 | 49 | err := executeCommandWithLogger(cfg, ".", repo, logger) 50 | assert.Errorf(t, err, "exit status 1") 51 | assert.Contains(t, buffer.String(), "Hello, from STDOUT") 52 | assert.Contains(t, buffer.String(), "Hello, from STDERR") 53 | } 54 | 55 | // Test that we can execute a script and that the environment variables are set correctly. 56 | func TestExecuteCommandWithLoggerWithEnvVars(t *testing.T) { 57 | t.Parallel() 58 | 59 | cfg := config.NewGitXargsConfig() 60 | cfg.Args = []string{"../data/test/_testscripts/test-env-vars.sh"} 61 | repo := getMockGithubRepo() 62 | 63 | var buffer bytes.Buffer 64 | 65 | // Test whether the lack of --dry-run sets environment variable correctly 66 | cfg.DryRun = false 67 | 68 | logger := &logrus.Logger{ 69 | Out: &buffer, 70 | Level: logrus.TraceLevel, 71 | Formatter: new(logrus.TextFormatter), 72 | } 73 | 74 | err := executeCommandWithLogger(cfg, ".", repo, logger) 75 | assert.NoError(t, err) 76 | assert.Contains(t, buffer.String(), "XARGS_DRY_RUN=false") 77 | assert.Contains(t, buffer.String(), fmt.Sprintf("XARGS_REPO_NAME=%s", *repo.Name)) 78 | assert.Contains(t, buffer.String(), fmt.Sprintf("XARGS_REPO_OWNER=%s", *repo.Owner.Login)) 79 | 80 | // Test whether --dry-run sets environment variable correctly 81 | cfg.DryRun = true 82 | 83 | logger = &logrus.Logger{ 84 | Out: &buffer, 85 | Level: logrus.TraceLevel, 86 | Formatter: new(logrus.TextFormatter), 87 | } 88 | 89 | err = executeCommandWithLogger(cfg, ".", repo, logger) 90 | assert.NoError(t, err) 91 | assert.Contains(t, buffer.String(), "XARGS_DRY_RUN=true") 92 | assert.Contains(t, buffer.String(), fmt.Sprintf("XARGS_REPO_NAME=%s", *repo.Name)) 93 | assert.Contains(t, buffer.String(), fmt.Sprintf("XARGS_REPO_OWNER=%s", *repo.Owner.Login)) 94 | } 95 | -------------------------------------------------------------------------------- /repository/select-repos.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "github.com/google/go-github/v43/github" 5 | "github.com/gruntwork-io/git-xargs/auth" 6 | "github.com/gruntwork-io/git-xargs/config" 7 | "github.com/gruntwork-io/git-xargs/io" 8 | "github.com/gruntwork-io/git-xargs/stats" 9 | "github.com/gruntwork-io/git-xargs/types" 10 | "github.com/gruntwork-io/git-xargs/util" 11 | "github.com/gruntwork-io/go-commons/errors" 12 | "github.com/gruntwork-io/go-commons/logging" 13 | 14 | "github.com/sirupsen/logrus" 15 | ) 16 | 17 | type RepoSelectionCriteria string 18 | 19 | const ( 20 | ReposViaStdIn RepoSelectionCriteria = "repo-stdin" 21 | ExplicitReposOnCommandLine RepoSelectionCriteria = "repo-flag" 22 | ReposFilePath RepoSelectionCriteria = "repos-file" 23 | GithubOrganization RepoSelectionCriteria = "github-org" 24 | ) 25 | 26 | // getPreferredOrderOfRepoSelections codifies the order in which flags will be preferred when the user supplied more 27 | // than one: 28 | // 1. --github-org is a string representing the GitHub org to page through via API for all repos. 29 | // 2. --repos is a string representing a filepath to a repos file 30 | // 3. --repo is a string slice flag that can be called multiple times 31 | // 4. stdin allows you to pipe repos in from other CLI tools 32 | func getPreferredOrderOfRepoSelections(config *config.GitXargsConfig) RepoSelectionCriteria { 33 | if config.GithubOrg != "" { 34 | return GithubOrganization 35 | } 36 | if config.ReposFile != "" { 37 | return ReposFilePath 38 | } 39 | if len(config.RepoSlice) > 0 { 40 | return ExplicitReposOnCommandLine 41 | } 42 | return ReposViaStdIn 43 | } 44 | 45 | // RepoSelection is a struct that presents a uniform interface to present to OperateRepos that converts 46 | // user-supplied repos in the format of / to GitHub API response objects that we actually 47 | // pass into processRepos which does the git cloning, command execution, committing and pull request opening 48 | type RepoSelection struct { 49 | SelectionType RepoSelectionCriteria 50 | AllowedRepos []*types.AllowedRepo 51 | GithubOrganizationName string 52 | } 53 | 54 | func (r RepoSelection) GetCriteria() RepoSelectionCriteria { 55 | return r.SelectionType 56 | } 57 | 58 | func (r RepoSelection) GetAllowedRepos() []*types.AllowedRepo { 59 | return r.AllowedRepos 60 | } 61 | 62 | func (r RepoSelection) GetGithubOrg() string { 63 | return r.GithubOrganizationName 64 | } 65 | 66 | // selectReposViaInput will examine the various repo and github-org flags to determine which should be selected and processed (only one at a time is used) 67 | func selectReposViaInput(config *config.GitXargsConfig) (*RepoSelection, error) { 68 | 69 | def := &RepoSelection{ 70 | SelectionType: GithubOrganization, 71 | AllowedRepos: []*types.AllowedRepo{}, 72 | GithubOrganizationName: config.GithubOrg, 73 | } 74 | switch getPreferredOrderOfRepoSelections(config) { 75 | case ExplicitReposOnCommandLine: 76 | config.Stats.SetSelectionMode(string(ExplicitReposOnCommandLine)) 77 | 78 | allowedRepos, malformedRepos, err := selectReposViaRepoFlag(config.RepoSlice) 79 | if err != nil { 80 | return def, err 81 | } 82 | 83 | trackMalformedUserSuppliedRepoNames(config, malformedRepos) 84 | 85 | return &RepoSelection{ 86 | SelectionType: ExplicitReposOnCommandLine, 87 | AllowedRepos: allowedRepos, 88 | GithubOrganizationName: "", 89 | }, nil 90 | 91 | case ReposFilePath: 92 | 93 | config.Stats.SetSelectionMode(string(ReposFilePath)) 94 | 95 | allowedRepos, err := io.ProcessAllowedRepos(config.ReposFile) 96 | if err != nil { 97 | return def, err 98 | } 99 | 100 | return &RepoSelection{ 101 | SelectionType: ReposFilePath, 102 | AllowedRepos: allowedRepos, 103 | GithubOrganizationName: "", 104 | }, nil 105 | 106 | case GithubOrganization: 107 | 108 | config.Stats.SetSelectionMode(string(GithubOrganization)) 109 | 110 | return def, nil 111 | 112 | case ReposViaStdIn: 113 | config.Stats.SetSelectionMode(string(ReposViaStdIn)) 114 | 115 | allowedRepos, malformedRepos, err := selectReposViaRepoFlag(config.RepoFromStdIn) 116 | if err != nil { 117 | return def, err 118 | } 119 | 120 | trackMalformedUserSuppliedRepoNames(config, malformedRepos) 121 | 122 | return &RepoSelection{ 123 | SelectionType: ReposViaStdIn, 124 | AllowedRepos: allowedRepos, 125 | GithubOrganizationName: "", 126 | }, nil 127 | 128 | default: 129 | return def, nil 130 | } 131 | } 132 | 133 | // trackMalformedUserSuppliedRepoNames will add any malformed repositories supplied by the user via --repo or STDIN 134 | // to the final report, explaining that the repos could not be used as supplied (usually due to missing org prefix) 135 | func trackMalformedUserSuppliedRepoNames(config *config.GitXargsConfig, malformedRepos []string) { 136 | // If any repos supplied via --repo flags were not parsed successfully, probably because they were malformed, 137 | // then add them to the final run report so the operator understands why they were not processed 138 | for _, m := range malformedRepos { 139 | mr := &github.Repository{ 140 | Name: github.String(m), 141 | } 142 | config.Stats.TrackSingle(stats.RepoFlagSuppliedRepoMalformed, mr) 143 | } 144 | } 145 | 146 | // selectReposViaRepoFlag converts the string slice of repo flags provided via stdin or by invocations of the --repo 147 | // flag into the internal representation of AllowedRepo that we use prior to fetching the corresponding repo from 148 | // GitHub 149 | func selectReposViaRepoFlag(inputRepos []string) ([]*types.AllowedRepo, []string, error) { 150 | var allowedRepos []*types.AllowedRepo 151 | var malformedRepos []string 152 | 153 | for _, repoInput := range inputRepos { 154 | allowedRepo := util.ConvertStringToAllowedRepo(repoInput) 155 | if allowedRepo != nil { 156 | allowedRepos = append(allowedRepos, allowedRepo) 157 | } else { 158 | malformedRepos = append(malformedRepos, repoInput) 159 | } 160 | } 161 | 162 | if len(allowedRepos) < 1 { 163 | return allowedRepos, malformedRepos, errors.WithStackTrace(types.NoRepoFlagTargetsValid{}) 164 | } 165 | 166 | return allowedRepos, malformedRepos, nil 167 | } 168 | 169 | // fetchUserProvidedReposViaGithub converts repos provided as strings, already validated as being well-formed, into GitHub API repo objects that can be further processed 170 | func fetchUserProvidedReposViaGithubAPI(githubClient auth.GithubClient, rs RepoSelection, stats *stats.RunStats) ([]*github.Repository, error) { 171 | ar := rs.GetAllowedRepos() 172 | return getFileDefinedRepos(githubClient, ar, stats) 173 | 174 | } 175 | 176 | // OperateOnRepos acts as a switch, depending upon whether the user provided an explicit list of repos to operate. 177 | // 178 | // There are three ways to select repos to operate on via this tool: 179 | // 1. the --repo flag, which specifies a single repo, and which can be passed multiple times, e.g., --repo gruntwork-io/fetch --repo gruntwork-io/cloud-nuke, etc. 180 | // 2. the --repos flag which specifies the path to the user-defined flat file of repos in the format of 'gruntwork-io/cloud-nuke', one repo per line. 181 | // 3. the --github-org flag which specifies the GitHub organization that should have all its repos fetched via API. 182 | // 183 | // However, even though there are two methods for users to select repos, we still only want a single uniform interface 184 | // for dealing with a repo throughout this tool, and that is the *github.Repository type provided by the go-github 185 | // library. Therefore, this function serves the purpose of creating that uniform interface, by looking up flat file-provided 186 | // repos via go-github, so that we're only ever dealing with pointers to github.Repositories going forward. 187 | func OperateOnRepos(config *config.GitXargsConfig) error { 188 | 189 | logger := logging.GetLogger("git-xargs") 190 | 191 | // The set of GitHub repositories the tool will actually process 192 | var reposToIterate []*github.Repository 193 | 194 | // repoSelection is a representations of the user-supplied input, containing the repo organization and name 195 | repoSelection, err := selectReposViaInput(config) 196 | 197 | if err != nil { 198 | return err 199 | } 200 | 201 | switch repoSelection.GetCriteria() { 202 | 203 | case GithubOrganization: 204 | // If githubOrganization is set, the user did not provide a flat file or explicit repos via the -repo(s) flags, so we're just looking up all the GitHub 205 | // repos via their Organization name via the GitHub API 206 | reposFetchedFromGithubAPI, err := getReposByOrg(config) 207 | if err != nil { 208 | logger.WithFields(logrus.Fields{ 209 | "Error": err, 210 | "Organization": config.GithubOrg, 211 | }).Debug("Failure looking up repos for organization") 212 | return err 213 | } 214 | // We gather all the repos by fetching them from the GitHub API, paging through the results of the supplied organization 215 | reposToIterate = reposFetchedFromGithubAPI 216 | 217 | logger.Debugf("Using Github org: %s as source of repositories. Paging through Github API for repos.", config.GithubOrg) 218 | 219 | case ReposFilePath: 220 | githubRepos, err := fetchUserProvidedReposViaGithubAPI(config.GithubClient, *repoSelection, config.Stats) 221 | if err != nil { 222 | return err 223 | } 224 | 225 | reposToIterate = githubRepos 226 | 227 | // Update count of number of repos the tool read in from the provided file 228 | config.Stats.SetFileProvidedRepos(repoSelection.GetAllowedRepos()) 229 | 230 | case ExplicitReposOnCommandLine, ReposViaStdIn: 231 | githubRepos, err := fetchUserProvidedReposViaGithubAPI(config.GithubClient, *repoSelection, config.Stats) 232 | if err != nil { 233 | return err 234 | } 235 | 236 | reposToIterate = githubRepos // Update the count of number of repos the tool read in from explicit --repo flags 237 | config.Stats.SetRepoFlagProvidedRepos(repoSelection.GetAllowedRepos()) 238 | 239 | default: 240 | // We've got no repos to iterate on, so return an error 241 | return errors.WithStackTrace(types.NoValidReposFoundAfterFilteringErr{}) 242 | } 243 | 244 | // Track the repos selected for processing 245 | config.Stats.TrackMultiple(stats.ReposSelected, reposToIterate) 246 | 247 | // Print out the repos that we've filtered for processing in debug mode 248 | for _, repo := range reposToIterate { 249 | logger.WithFields(logrus.Fields{ 250 | "Repository": repo.GetName(), 251 | }).Debug("Repo will have all targeted scripts run against it") 252 | } 253 | // Now that we've gathered the repos we're going to operate on, do the actual processing by running the 254 | // user-defined scripts against each repo and handling the resulting git operations that follow 255 | if err := ProcessRepos(config, reposToIterate); err != nil { 256 | return err 257 | } 258 | 259 | return nil 260 | } 261 | -------------------------------------------------------------------------------- /repository/select-repos_test.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/git-xargs/config" 7 | "github.com/gruntwork-io/git-xargs/mocks" 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | // TestSelectReposViaInput ensures the selectReposViaInput function correctly returns the correct repo target type 14 | // given the 3 different ways to target repos for processing 15 | func TestSelectReposViaInput(t *testing.T) { 16 | t.Parallel() 17 | 18 | testConfig := config.NewGitXargsTestConfig() 19 | testConfig.RepoSlice = []string{"gruntwork-io/terratest", "gruntwork-io/cloud-nuke"} 20 | 21 | repoSelection, err := selectReposViaInput(testConfig) 22 | 23 | require.NoError(t, err) 24 | require.NotNil(t, repoSelection) 25 | assert.Equal(t, repoSelection.SelectionType, ExplicitReposOnCommandLine) 26 | 27 | configOrg := config.NewGitXargsTestConfig() 28 | configOrg.GithubOrg = "gruntwork-io" 29 | 30 | repoSelectionByOrg, orgErr := selectReposViaInput(configOrg) 31 | 32 | require.NoError(t, orgErr) 33 | require.NotNil(t, repoSelectionByOrg) 34 | assert.Equal(t, repoSelectionByOrg.SelectionType, GithubOrganization) 35 | 36 | configStdin := config.NewGitXargsTestConfig() 37 | configStdin.RepoFromStdIn = []string{"gruntwork-io/terratest", "gruntwork-io/cloud-nuke"} 38 | 39 | repoSelectionByStdin, stdInErr := selectReposViaInput(configStdin) 40 | 41 | require.NoError(t, stdInErr) 42 | require.NotNil(t, repoSelectionByStdin) 43 | assert.Equal(t, repoSelectionByStdin.SelectionType, ReposViaStdIn) 44 | } 45 | 46 | // TestOperateOnRepos smoke tests the OperateOnRepos method 47 | func TestOperateOnRepos(t *testing.T) { 48 | t.Parallel() 49 | 50 | testConfig := config.NewGitXargsTestConfig() 51 | testConfig.GithubOrg = "gruntwork-io" 52 | testConfig.GithubClient = mocks.ConfigureMockGithubClient() 53 | 54 | err := OperateOnRepos(testConfig) 55 | assert.NoError(t, err) 56 | 57 | configReposOnCommandLine := config.NewGitXargsTestConfig() 58 | configReposOnCommandLine.GithubClient = mocks.ConfigureMockGithubClient() 59 | 60 | configReposOnCommandLine.RepoSlice = []string{"gruntwork-io/fetch", "gruntwork-io/cloud-nuke"} 61 | 62 | cmdLineErr := OperateOnRepos(configReposOnCommandLine) 63 | assert.NoError(t, cmdLineErr) 64 | } 65 | 66 | // TestGetPreferredOrderOfRepoSelections ensures the getPreferredOrderOfRepoSelections returns the expected method 67 | // for fetching repos given the three possible means of targeting repositories for processing 68 | func TestGetPreferredOrderOfRepoSelections(t *testing.T) { 69 | t.Parallel() 70 | 71 | testConfig := config.NewGitXargsTestConfig() 72 | 73 | testConfig.GithubOrg = "gruntwork-io" 74 | testConfig.ReposFile = "repos.txt" 75 | testConfig.RepoSlice = []string{"github.com/gruntwork-io/fetch", "github.com/gruntwork-io/cloud-nuke"} 76 | testConfig.RepoFromStdIn = []string{"github.com/gruntwork-io/terragrunt", "github.com/gruntwork-io/terratest"} 77 | 78 | assert.Equal(t, GithubOrganization, getPreferredOrderOfRepoSelections(testConfig)) 79 | 80 | testConfig.GithubOrg = "" 81 | 82 | assert.Equal(t, ReposFilePath, getPreferredOrderOfRepoSelections(testConfig)) 83 | 84 | testConfig.ReposFile = "" 85 | 86 | assert.Equal(t, ExplicitReposOnCommandLine, getPreferredOrderOfRepoSelections(testConfig)) 87 | 88 | testConfig.RepoSlice = []string{} 89 | 90 | assert.Equal(t, ReposViaStdIn, getPreferredOrderOfRepoSelections(testConfig)) 91 | } 92 | -------------------------------------------------------------------------------- /scripts/add-or-update-license.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | YEAR=$(date +"%Y") 4 | FULLNAME="Gruntwork, LLC" 5 | 6 | function create_license { 7 | cat << EOF > LICENSE.txt 8 | MIT License 9 | 10 | Copyright (c) 2016 to $YEAR, $FULLNAME 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy 13 | of this software and associated documentation files (the "Software"), to deal 14 | in the Software without restriction, including without limitation the rights 15 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 | copies of the Software, and to permit persons to whom the Software is 17 | furnished to do so, subject to the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be included in all 20 | copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 | SOFTWARE. 29 | EOF 30 | } 31 | 32 | # Copyrights should be declared as "$CREATION_YEAR to $CURRENT_YEAR" 33 | # Therefore, this sed command will look to update the date immediately following the word "to" 34 | function update_license_copyright_year { 35 | echo "Updating license copyright year to $(date +%Y)..." 36 | sed -i "s|to \([1-9][0-9][0-9][0-9]\)|to $(date +%Y)|" LICENSE.txt 37 | if [ $? -eq 0 ]; then 38 | echo "Success!" 39 | else 40 | echo "Error!" 41 | fi 42 | } 43 | 44 | # if the repo does not contain a LICENSE.txt file, then create one with the correct year 45 | if [ ! -f "LICENSE.txt" ]; then 46 | echo "Could not find LICENSE.txt at root of repo, so adding one..." 47 | create_license 48 | else 49 | update_license_copyright_year 50 | fi 51 | 52 | 53 | -------------------------------------------------------------------------------- /scripts/add-or-update-pr-issue-templates.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script will ensure a repo has the necessary pull request and issue templates as 4 | # part of our contributing guide: https://doc.gruntwork.io/guides/contributing/ 5 | # 6 | # It creates or replaces the following files: 7 | # .github/ISSUE_TEMPLATE/bug_report.md 8 | # .github/ISSUE_TEMPLATE/feature_request.md 9 | # .github/pull_request_template.md 10 | 11 | function create_bug_issue_template { 12 | cat << "EOF" > .github/ISSUE_TEMPLATE/bug_report.md 13 | --- 14 | name: Bug report 15 | about: Create a bug report to help us improve. 16 | title: '' 17 | labels: bug 18 | assignees: '' 19 | 20 | --- 21 | 22 | 26 | 27 | **Describe the bug** 28 | A clear and concise description of what the bug is. 29 | 30 | **To Reproduce** 31 | Steps to reproduce the behavior including the relevant Terraform/Terragrunt/Packer version number and any code snippets and module inputs you used. 32 | 33 | ```hcl 34 | // paste code snippets here 35 | ``` 36 | 37 | **Expected behavior** 38 | A clear and concise description of what you expected to happen. 39 | 40 | **Nice to have** 41 | - [ ] Terminal output 42 | - [ ] Screenshots 43 | 44 | **Additional context** 45 | Add any other context about the problem here. 46 | EOF 47 | } 48 | 49 | function create_feature_issue_template { 50 | cat << "EOF" > .github/ISSUE_TEMPLATE/feature_request.md 51 | --- 52 | name: Feature request 53 | about: Submit a feature request for this repo. 54 | title: '' 55 | labels: enhancement 56 | assignees: '' 57 | 58 | --- 59 | 60 | 64 | 65 | **Describe the solution you'd like** 66 | A clear and concise description of what you want to happen. 67 | 68 | **Describe alternatives you've considered** 69 | A clear and concise description of any alternative solutions or features you've considered. 70 | 71 | **Additional context** 72 | Add any other context or screenshots about the feature request here. 73 | EOF 74 | } 75 | 76 | function create_pr_template { 77 | cat << "EOF" > .github/pull_request_template.md 78 | 83 | 84 | ## Description 85 | 86 | 87 | 88 | ### Documentation 89 | 90 | 98 | 99 | 100 | 101 | ## TODOs 102 | 103 | Please ensure all of these TODOs are completed before asking for a review. 104 | 105 | - [ ] Ensure the branch is named correctly with the issue number. e.g: `feature/new-vpc-endpoints-955` or `bug/missing-count-param-434`. 106 | - [ ] Update the docs. 107 | - [ ] Keep the changes backward compatible where possible. 108 | - [ ] Run the pre-commit checks successfully. 109 | - [ ] Run the relevant tests successfully. 110 | - [ ] Ensure any 3rd party code adheres with our [license policy](https://www.notion.so/gruntwork/Gruntwork-licenses-and-open-source-usage-policy-f7dece1f780341c7b69c1763f22b1378) or delete this line if its not applicable. 111 | 112 | 113 | ## Related Issues 114 | 115 | 121 | EOF 122 | } 123 | 124 | # Ensure the GitHub template directories exist 125 | mkdir -p .github 126 | mkdir -p .github/ISSUE_TEMPLATE 127 | 128 | # if the repo does not contain a bug_report.md file, then create one or replace the existing one 129 | if [[ ! -f ".github/ISSUE_TEMPLATE/bug_report.md" ]]; then 130 | echo "Could not find file at .github/ISSUE_TEMPLATE/bug_report.md, so adding one..." 131 | create_bug_issue_template 132 | else 133 | echo "Found file at .github/ISSUE_TEMPLATE/bug_report.md, so replacing it..." 134 | rm .github/ISSUE_TEMPLATE/bug_report.md 135 | create_bug_issue_template 136 | fi 137 | 138 | # if the repo does not contain a feature_request.md file, then create one or replace the existing one 139 | if [[ ! -f ".github/ISSUE_TEMPLATE/feature_request.md" ]]; then 140 | echo "Could not find file at .github/ISSUE_TEMPLATE/feature_request.md, so adding one..." 141 | create_feature_issue_template 142 | else 143 | echo "Found file at .github/ISSUE_TEMPLATE/feature_request.md, so replacing it..." 144 | rm .github/ISSUE_TEMPLATE/feature_request.md 145 | create_feature_issue_template 146 | fi 147 | 148 | # if the repo does not contain a pull_request_template.md file, then create one or replace the existing one 149 | if [[ ! -f ".github/pull_request_template.md" ]]; then 150 | echo "Could not find file at .github/pull_request_template.md, so adding one..." 151 | create_pr_template 152 | else 153 | echo "Found file at .github/pull_request_template.md, so replacing it..." 154 | rm .github/pull_request_template.md 155 | create_pr_template 156 | fi 157 | -------------------------------------------------------------------------------- /scripts/circleci-workflows-version-upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Upgrading CircleCI workflows syntax to 2..." 4 | 5 | yq w -i .circleci/config.yml 'workflows.version' 2 6 | 7 | # Remove stray merge tags that yq adds to the final output 8 | sed -i '/!!merge /d' .circleci/config.yml 9 | -------------------------------------------------------------------------------- /scripts/error.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script intentionally errors out during execution to simulate the behavior of a real script that encounters a runtime error 4 | # This is useful for verifying the error handling behavior of the tool 5 | echo "error.sh intentionally exiting with 1 status" 6 | exit 1 7 | -------------------------------------------------------------------------------- /scripts/pre-commit-example.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function add_precommit { 4 | cat << EOF > .pre-commit-config.yaml 5 | repos: 6 | - repo: https://github.com/gruntwork-io/pre-commit 7 | rev: # Get the latest from: https://github.com/gruntwork-io/pre-commit/releases 8 | hooks: 9 | - id: terraform-fmt 10 | - id: terraform-validate 11 | - id: tflint 12 | - id: shellcheck 13 | - id: gofmt 14 | - id: golint 15 | EOF 16 | } 17 | 18 | echo "Running pre-commit example.sh..." 19 | 20 | if [[ ! -f .pre-commit-config.yaml ]]; then 21 | echo ".pre-commit-config.yaml file does not already exist. Adding it now..." 22 | add_precommit 23 | else 24 | echo "Found existing .pre-commit-config.yaml file. Nothing to do." 25 | 26 | fi 27 | 28 | 29 | -------------------------------------------------------------------------------- /scripts/sleep.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # I'm used for testing parallelization! 4 | echo "sleeping 5 seconds..." 5 | sleep 5 6 | echo "writing slept file" 7 | touch slept.txt 8 | -------------------------------------------------------------------------------- /scripts/test-python.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | 4 | print("Python test script running") 5 | 6 | os.system('touch python-was-here.py') 7 | -------------------------------------------------------------------------------- /scripts/test-ruby.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | puts "Ruby test script running..." 3 | 4 | puts `touch ruby-was-here.rb` 5 | -------------------------------------------------------------------------------- /scripts/tf14.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # This script updates a repo to use Terraform 0.14. We are implementing the steps outlined in 3 | # https://www.notion.so/gruntwork/Terraform-0-14-Upgrade-87f06063b1bd46789a6e0089168674d6 4 | 5 | require 'json' 6 | require 'open3' 7 | 8 | DEFAULT_TERRAFORM_VERSION = "0.14.8" 9 | 10 | REQUIRED_PROVIDERS_REGEX = /^\s*required_providers\s*{\s*$/ 11 | 12 | def get_root_folder 13 | `git rev-parse --show-toplevel`.strip 14 | end 15 | 16 | # Updates the Terraform version in CircleCI config file 17 | def update_circleci_build_to_tf14 root_folder 18 | puts "Updating Terraform's version..." 19 | current_image = `yq eval '.defaults.docker[0].image' #{root_folder}/.circleci/config.yml` 20 | 21 | if current_image.strip != "null" 22 | `yq eval '.defaults.docker[0].image="087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:go1.16-go111module"' -i #{root_folder}/.circleci/config.yml` 23 | elsif File.readlines("#{root_folder}/.circleci/config.yml").grep(/TERRAFORM_VERSION:/).size > 0 24 | circle_ci_search_and_replace root_folder, /TERRAFORM_VERSION: .+/, "TERRAFORM_VERSION: #{DEFAULT_TERRAFORM_VERSION}" 25 | circle_ci_search_and_replace root_folder, /^(.*)GOLANG_VERSION: .+(\n.*GO111MODULE: auto.*)?/, '\1GOLANG_VERSION: 1.16\n\1GO111MODULE: auto' 26 | circle_ci_search_and_replace root_folder, '\n', "\n" 27 | circle_ci_search_and_replace root_folder, /^(.*)MODULE_CI_VERSION: .+/, '\1MODULE_CI_VERSION: v0.31.0' 28 | else 29 | raise "Did not find either a Docker image nor TERRAFORM_VERSION in CircleCI config" 30 | end 31 | end 32 | 33 | # Updates Go's version in every folder that has a go.mod 34 | def update_terratest_in_go_mod root_folder 35 | paths_with_golang = Dir.glob "#{root_folder}/**/go.mod" 36 | 37 | paths_with_golang.each do |path| 38 | if File.readlines(path).grep(/terratest/).size > 0 39 | puts "Updating Terratest's version at #{path}..." 40 | 41 | dir = File.dirname(path) 42 | `cd #{dir} && go get -u github.com/gruntwork-io/terratest@v0.31.3` 43 | `cd #{dir} && go mod tidy` 44 | else 45 | puts "Found #{path} but it does not include Terratest as a dependency. Skipping." 46 | end 47 | end 48 | end 49 | 50 | # Updates the Terragrunt version in CircleCI config file 51 | def update_circleci_build_to_tg27 root_folder 52 | terragrunt_version = `yq eval '.env.environment.TERRAGRUNT_VERSION' #{root_folder}/.circleci/config.yml` 53 | 54 | if (terragrunt_version.strip != "null") && (terragrunt_version.strip != "NONE") 55 | puts "Updating Terragrunt's version..." 56 | `yq eval -P '.env.environment.TERRAGRUNT_VERSION="v0.27.1"' -i #{root_folder}/.circleci/config.yml` 57 | else 58 | puts "Did not find a Terragrunt version (or it is set to NONE) in the CircleCi config. Skipping." 59 | end 60 | end 61 | 62 | # Removes `!!merge` tag left over from yq in the circleci/config.yml file 63 | def remove_unused_yq_tags root_folder 64 | circle_ci_search_and_replace root_folder, "!!merge ", "" 65 | end 66 | 67 | # Updates the CircleCI config.yml file, replacing a given string to a new one. 68 | def circle_ci_search_and_replace root_folder, old_str, new_str 69 | path_to_circle_config_yml = "#{root_folder}/.circleci/config.yml" 70 | circle_ci_config = File.read(path_to_circle_config_yml) 71 | sanitised_circle_ci_config = circle_ci_config.gsub(old_str, new_str) 72 | 73 | if sanitised_circle_ci_config != circle_ci_config 74 | File.open(path_to_circle_config_yml, "w") do |file| 75 | file.write(sanitised_circle_ci_config) 76 | end 77 | 78 | true 79 | else 80 | false 81 | end 82 | end 83 | 84 | # Add .terraform.lock.hcl to .gitignore, as our repos only have example code, and we want the latest providers on them, 85 | # so there's no need to lock the versions. 86 | def ignore_terraform_lock_file root_folder 87 | gitignore_path = "#{root_folder}/.gitignore" 88 | new_gitignore_value = %{ 89 | # Ignore Terraform lock files, as we want to test the Terraform code in these repos with the latest provider 90 | # versions. 91 | .terraform.lock.hcl 92 | } 93 | 94 | if File.readlines(gitignore_path).grep(/.terraform.lock.hcl/).size == 0 95 | puts "Adding .terraform.lock.hcl to .gitignore" 96 | File.open(gitignore_path, 'a') do |file| 97 | file.write(new_gitignore_value) 98 | end 99 | else 100 | puts ".terraform.lock.hcl is already in .gitignore. Skipping." 101 | end 102 | end 103 | 104 | # Update the required_version constraint in the Terraform code. Also, our code has several flavors of comment blocks 105 | # that talk about what version of Terraform the code works with or is tested with, so we update or clean up those 106 | # comment blocks here too. 107 | def update_tf_version_in_code root_folder 108 | regex_to_update = [ 109 | { 110 | # Remove out of date comment blocks we forgot about during the 0.13 upgrade. These look something like: 111 | # 112 | # ----------------------------------------------------------------------------------------------- 113 | # REQUIRE A SPECIFIC TERRAFORM VERSION OR HIGHER 114 | # This module uses HCL2 syntax, which means it is not compatible with any versions below 0.12. 115 | # ----------------------------------------------------------------------------------------------- 116 | :regex => /#\s*-+\s*\n# REQUIRE A SPECIFIC TERRAFORM VERSION OR HIGHER\s*\n# This module uses HCL2 syntax, which means it is not compatible with any versions below 0.12.*\n#\s*-+\s*\n/, 117 | :replacement => '' 118 | }, 119 | { 120 | # Remove out of date comment blocks we forgot about during the 0.13 upgrade. These look something like: 121 | # 122 | # ----------------------------------------------------------------------------------------------- 123 | # REQUIRE A SPECIFIC TERRAFORM VERSION OR HIGHER 124 | # This module has been updated with 0.12 syntax, which means it is no longer compatible with any versions below 0.12. 125 | # ----------------------------------------------------------------------------------------------- 126 | :regex => /#\s*-+\s*\n# REQUIRE A SPECIFIC TERRAFORM VERSION OR HIGHER\s*\n# This module has been updated with 0.12 syntax, which means it is no longer compatible with any versions below 0.12.*\n#\s*-+\s*\n/, 127 | :replacement => '' 128 | }, 129 | { 130 | # Remove out of date comment blocks we forgot about during the 0.13 upgrade. 131 | :regex => /# This module has been updated with 0.12 syntax, which means it is no longer compatible with any versions below 0.12.*\n/, 132 | :replacement => '' 133 | }, 134 | { 135 | # Update the first sentence in a comment block we added during the 0.13 upgrade to now talk about 0.14. These 136 | # look something like: 137 | # 138 | # This module is now only being tested with Terraform 0.13.x. However, to make upgrading easier, we are setting 139 | # 0.12.26 as the minimum version, as that version added support for required_providers with source URLs, making it 140 | # forwards compatible with 0.13.x code. 141 | :regex => /This module is now only being tested with Terraform 0.13.x/, 142 | :replacement => 'This module is now only being tested with Terraform 0.14.x' 143 | }, 144 | { 145 | # Update the last sentence in a comment block we added during the 0.13 upgrade to now talk about 0.14. These 146 | # look something like: 147 | # 148 | # This module is now only being tested with Terraform 0.13.x. However, to make upgrading easier, we are setting 149 | # 0.12.26 as the minimum version, as that version added support for required_providers with source URLs, making it 150 | # forwards compatible with 0.13.x code. 151 | :regex => /forwards compatible with 0.13.x code/, 152 | :replacement => 'forwards compatible with 0.14.x code' 153 | } 154 | ] 155 | 156 | paths_with_terraform = Dir.glob "#{root_folder}/**/*.tf" 157 | paths_with_terraform.each do |path| 158 | contents = File.read(path) 159 | updated_contents = regex_to_update.reduce(contents) do |current_contents, to_update| 160 | current_contents.gsub(to_update[:regex], to_update[:replacement]) 161 | end 162 | 163 | updated_contents = update_required_tf_version updated_contents, path 164 | updated_contents = ensure_comment_block_present updated_contents, path 165 | 166 | if contents != updated_contents 167 | puts "Updating Terraform version in comments in #{path}" 168 | File.write(path, updated_contents) 169 | else 170 | puts "Did not find Terraform version in any comments to update in #{path}" 171 | end 172 | end 173 | end 174 | 175 | # Ensure that the required_version in the given contents (which are assumed to be a Terraform file at the given path) 176 | # is set to the value we are currently using (>= 0.12.26). 177 | def update_required_tf_version contents, path 178 | expected_version_constraint = ">= 0.12.26" 179 | version_constraint_regex = /required_version\s*=\s*"(.+?)"/ 180 | 181 | required_version = contents.match(version_constraint_regex) 182 | if required_version 183 | version_constraint = required_version.captures.first 184 | if version_constraint != expected_version_constraint 185 | puts "Updating version constraint in #{path} from #{version_constraint} to #{expected_version_constraint}" 186 | return contents.gsub(version_constraint_regex, "required_version = \"#{expected_version_constraint}\"") 187 | end 188 | end 189 | 190 | contents 191 | end 192 | 193 | # Ensure that we have a consistent comment block above the required_version constraint in the given contents (which 194 | # are assumed to be a Terraform file at the given path) 195 | def ensure_comment_block_present contents, path 196 | comment_block = %{ 197 | # This module is now only being tested with Terraform 0.14.x. However, to make upgrading easier, we are setting 198 | # 0.12.26 as the minimum version, as that version added support for required_providers with source URLs, making it 199 | # forwards compatible with 0.14.x code. 200 | } 201 | 202 | version_constraint_prev_line_regex = /^(.*)\n(\s*required_version\s*=\s*".+?")/ 203 | 204 | required_version_and_prev_line = contents.match(version_constraint_prev_line_regex) 205 | if required_version_and_prev_line 206 | prev_line, required_version = required_version_and_prev_line.captures 207 | if prev_line.strip != "# forwards compatible with 0.14.x code." 208 | puts "Adding missing comment block above required_version constraint in #{path}" 209 | return contents.gsub(version_constraint_prev_line_regex, "#{prev_line}#{comment_block}#{required_version}") 210 | end 211 | end 212 | 213 | contents 214 | end 215 | 216 | 217 | # In TF 0.14, a provider { ... } block with a version = "" is deprecated and needs to be replaced with a 218 | # required_providers block inside of a terraform { ... } block. This function uses a simple regex to look for these 219 | # version constraints and exit with an error if it finds one so that the human operator can go and fix them. 220 | def update_provider_constraints root_folder 221 | puts "Switching to Terraform #{DEFAULT_TERRAFORM_VERSION}" 222 | `tfenv install #{DEFAULT_TERRAFORM_VERSION}` 223 | `tfenv use #{DEFAULT_TERRAFORM_VERSION}` 224 | 225 | paths_with_terraform = Dir.glob "#{root_folder}/**/*.tf" 226 | folders_with_terraform = paths_with_terraform.map { |path| File.dirname(path) }.uniq 227 | 228 | folders_with_terraform.each do |folder| 229 | # We make the (hopefully not too inaccurate) assumption that all provider and terraform blocks in our modules are 230 | # defined in a main.tf file. If there's no main.tf file, we just skip it. 231 | main_tf_path = File.join(folder, "main.tf") 232 | if !File.exist?(main_tf_path) 233 | next 234 | end 235 | 236 | main_tf_contents = IO.read(main_tf_path) 237 | 238 | blocks = hcledit(["block", "list"], main_tf_contents).split("\n") 239 | 240 | has_terraform_block = blocks.any? { |block| block == "terraform" } 241 | if !has_terraform_block 242 | main_tf_contents = add_terraform_block(main_tf_contents, main_tf_path) 243 | end 244 | 245 | providers = blocks.select { |block| block.start_with?("provider.") }.map{ |block| block.gsub(/^provider./, "") } 246 | providers.each do |provider| 247 | main_tf_contents = update_provider_constraint(provider, main_tf_contents, main_tf_path) 248 | end 249 | 250 | IO.write(main_tf_path, main_tf_contents) 251 | end 252 | end 253 | 254 | def update_provider_constraint(provider, main_tf_contents, main_tf_path) 255 | version_constraint = hcledit(["attribute", "get", "provider.#{provider}.version"], main_tf_contents).strip 256 | if version_constraint.length == 0 257 | return main_tf_contents 258 | end 259 | 260 | puts "Removing version constraint for provider #{provider} in #{main_tf_path}" 261 | main_tf_contents = hcledit(["attribute", "rm", "provider.#{provider}.version"], main_tf_contents) 262 | 263 | has_required_providers_constraint = hcledit(["attribute", "get", "terraform.required_providers.#{provider}"], main_tf_contents) 264 | if has_required_providers_constraint.length > 0 265 | puts "There is already a required_providers version constraint for provider #{provider} in #{main_tf_path}, so will not add another one." 266 | else 267 | terraform_block = hcledit(["block", "get", "terraform"], main_tf_contents) 268 | if !REQUIRED_PROVIDERS_REGEX.match(terraform_block) 269 | puts "Adding required_providers block to #{main_tf_path}" 270 | main_tf_contents = hcledit(["block", "append", "terraform", "required_providers", "--newline"], main_tf_contents) 271 | end 272 | 273 | puts "Adding version constraint for provider #{provider} to required_providers block in #{main_tf_path}" 274 | version_constraint_block = """{ 275 | source = \"hashicorp/#{provider}\" 276 | version = #{version_constraint} 277 | } 278 | """ 279 | main_tf_contents = hcledit(["attribute", "append", "terraform.required_providers.#{provider}", version_constraint_block], main_tf_contents) 280 | end 281 | 282 | main_tf_contents 283 | end 284 | 285 | # Add a new terraform { ... } block. If the file starts with a comment block, we try to put the terraform { ... } block 286 | # after that comment block; otherwise, we put it at the top of the file. 287 | def add_terraform_block(main_tf_contents, main_tf_path) 288 | puts "Adding terraform { ... } block to #{main_tf_path}" 289 | 290 | lines = main_tf_contents.split("\n") 291 | line_no = 0 292 | while line_no < lines.length && lines[line_no].start_with?("#") 293 | line_no = line_no + 1 294 | end 295 | 296 | terraform_block = """ 297 | terraform { 298 | # This module is now only being tested with Terraform 0.14.x. However, to make upgrading easier, we are setting 299 | # 0.12.26 as the minimum version, as that version added support for required_providers with source URLs, making it 300 | # forwards compatible with 0.14.x code. 301 | required_version = \">= 0.12.26\" 302 | } 303 | """ 304 | 305 | lines.insert(line_no, terraform_block) 306 | lines.join("\n") 307 | end 308 | 309 | # Run hcledit with the given args and the given stdin. Return stdout. 310 | def hcledit(args, stdin) 311 | out, err, status = Open3.capture3("hcledit", *args, stdin_data: stdin) 312 | if status != 0 313 | raise "hcledit exited with exit code #{status}: #{err}" 314 | end 315 | out 316 | end 317 | 318 | def update_readme_badge root_folder 319 | # In root/README.md or root/README.adoc: 320 | # replace string https://img.shields.io/badge/tf-%3E%3D0.12.0-blue.svg 321 | # with string https://img.shields.io/badge/tf-%3E%3D0.14.0-blue.svg 322 | 323 | tf_12_badge = 'https://img.shields.io/badge/tf-%3E%3D0.12.0-blue.svg' 324 | tf_14_badge = 'https://img.shields.io/badge/tf-%3E%3D0.14.0-blue.svg' 325 | readme_adoc_path = root_folder + '/README.adoc' 326 | readme_md_path = root_folder + '/README.md' 327 | 328 | for readme_path in [readme_adoc_path, readme_md_path] 329 | if File.exist?(readme_path) 330 | puts "Found " + readme_path 331 | readme_contents = File.read(readme_path) 332 | if readme_contents.include? tf_12_badge 333 | puts "Found TF 12 badge, replacing with TF 14 badge." 334 | readme_contents = readme_contents.gsub(tf_12_badge, tf_14_badge) 335 | IO.write(readme_path, readme_contents) 336 | else 337 | puts "Did not find TF 12 badge. Skipping." 338 | end 339 | else 340 | puts "Did not find " + readme_path + '. Skipping.' 341 | end 342 | end 343 | end 344 | 345 | def terraform_format root_folder 346 | `cd #{root_folder} && terraform fmt -recursive .` 347 | end 348 | 349 | root_folder = get_root_folder 350 | update_circleci_build_to_tf14 root_folder 351 | update_terratest_in_go_mod root_folder 352 | update_circleci_build_to_tg27 root_folder 353 | remove_unused_yq_tags root_folder 354 | ignore_terraform_lock_file root_folder 355 | update_tf_version_in_code root_folder 356 | update_provider_constraints root_folder 357 | update_readme_badge root_folder 358 | terraform_format root_folder 359 | 360 | -------------------------------------------------------------------------------- /scripts/update-repo-names.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # We renamed a ton of our repos to the Terraform Registry naming format. This script uses grep and sed to search for 3 | # references to the old names and replace them with the new names. For more context on the rename, see 4 | # https://gruntwork-io.slack.com/archives/C0PJF332B/p1611068513028500. 5 | 6 | # Bash doesn't have a good way to escape quotes in strings. The official solution is to list multiple strings next 7 | # to each other (https://stackoverflow.com/a/28786747/483528), but that becomes unreadable, especially with regex. 8 | # Therefore, to make our regex more readable, we are declaring clearly-named variables for the special characters we 9 | # want to match, and including those in a string with no need for escaping or crazy concatenation 10 | readonly DOUBLE_QUOTE='"' 11 | readonly SINGLE_QUOTE="'" 12 | readonly BACKTICK='`' 13 | readonly START_OF_LINE='^' 14 | readonly END_OF_LINE='$' 15 | readonly FORWARD_SLASH='\/' 16 | readonly DOT='\.' 17 | readonly WHITESPACE='\s' 18 | readonly OPEN_BRACKET='\[' 19 | readonly CLOSE_BRACKET='\]' 20 | readonly OPEN_PAREN='\(' 21 | readonly CLOSE_PAREN='\)' 22 | readonly OPEN_CURLY_BRACE='\{' 23 | readonly CLOSE_CURLY_BRACE='\}' 24 | 25 | # When replacing old names with new, these are regular expressions for the characters we allow before a name or after. 26 | # We check these characters explicitly to make sure we don't accidentally replace one of the names when it just happens 27 | # to appear as a substring in some unrelated word. E.g., "module-ci" should NOT be replaced in 28 | # "gruntwork-module-circleci-helpers". 29 | readonly ALLOWED_CHARS_BEFORE="($START_OF_LINE|$WHITESPACE|$FORWARD_SLASH|$DOUBLE_QUOTE|$SINGLE_QUOTE|$BACKTICK|$OPEN_BRACKET|$CLOSE_BRACKET|$OPEN_PAREN|$CLOSE_PAREN|$OPEN_CURLY_BRACE|$CLOSE_CURLY_BRACE)" 30 | readonly ALLOWED_CHARS_AFTER="($END_OF_LINE|$WHITESPACE|$FORWARD_SLASH|$DOUBLE_QUOTE|$SINGLE_QUOTE|$BACKTICK|$OPEN_BRACKET|$CLOSE_BRACKET|$OPEN_PAREN|$CLOSE_PAREN|$OPEN_CURLY_BRACE|$CLOSE_CURLY_BRACE|$DOT)" 31 | 32 | # The list of repos to replace, in pairs, where the first entry is the old name and the second entry is the second name 33 | # (we use this ugly array instead of a map because the old Bash version on Mac doesn't support maps). 34 | readonly REPLACEMENT_PAIRS=( 35 | "module-vpc" "terraform-aws-vpc" 36 | "module-aws-monitoring" "terraform-aws-monitoring" 37 | "package-directory-services" "terraform-aws-directory-services" 38 | "module-file-storage" "terraform-aws-file-storage" 39 | "module-ecs" "terraform-aws-ecs" 40 | "module-security" "terraform-aws-security" 41 | "cis-compliance-aws" "terraform-aws-cis-service-catalog" 42 | "aws-service-catalog" "terraform-aws-service-catalog" 43 | "aws-architecture-catalog" "terraform-aws-architecture-catalog" 44 | "package-terraform-utilities" "terraform-aws-utilities" 45 | "module-ci" "terraform-aws-ci" 46 | "module-asg" "terraform-aws-asg" 47 | "module-server" "terraform-aws-server" 48 | "package-beanstalk" "terraform-aws-beanstalk" 49 | "package-openvpn" "terraform-aws-openvpn" 50 | "module-data-storage" "terraform-aws-data-storage" 51 | "module-load-balancer" "terraform-aws-load-balancer" 52 | "package-zookeeper" "terraform-aws-zookeeper" 53 | "package-kafka" "terraform-aws-kafka" 54 | "package-messaging" "terraform-aws-messaging" 55 | "module-cache" "terraform-aws-cache" 56 | "package-static-assets" "terraform-aws-static-assets" 57 | "package-elk" "terraform-aws-elk" 58 | "package-mongodb" "terraform-aws-mongodb" 59 | "package-lambda" "terraform-aws-lambda" 60 | "package-datadog" "terraform-aws-datadog" 61 | "package-waf" "terraform-aws-waf" 62 | "package-sam" "terraform-aws-sam" 63 | "module-ci-pipeline-example" "terraform-aws-ci-pipeline-example" 64 | ) 65 | 66 | # Finds all files in the repo to replace, taking care to exclude the .git folder, Terraform & Terragrunt scratch 67 | # folders, binary files, and other files we shouldn't be touching. Note that we also skip .go, go.mod, and go.sum 68 | # because our Go code refers to skip versions of our repos, and if we try the new names with the old version numbers, 69 | # Go gives a "module declares its path as: " error. So the only way to use the new names with Go code is to 70 | # publish new versions, but to do that, we'd have to build a dependency graph, update repos and publish new versions in 71 | # the right order, update the code to use these new versions, and fix any backwards incompatibilities in these new 72 | # versions, which is far work than we're willing to do for a version number bump right now. 73 | function find_files_to_update { 74 | find . \ 75 | -not -iwholename '*.git*' \ 76 | -not -iwholename '*.terragrunt-cache*' \ 77 | -not -iwholename '*.terraform*' \ 78 | -not -iwholename '*.png' \ 79 | -not -iwholename '*.jpg' \ 80 | -not -iwholename '*.jpeg' \ 81 | -not -iwholename '*.gif' \ 82 | -not -iwholename '*.bmp' \ 83 | -not -iwholename '*.tiff' \ 84 | -not -iwholename '*.DS_Store*' \ 85 | -not -iwholename '*.go' \ 86 | -not -iwholename '*go.mod' \ 87 | -not -iwholename '*go.sum' \ 88 | -type f 89 | } 90 | 91 | # Format a regex replacement string for use with perl. The returned value will be of the format: 92 | # 93 | # s///g; s///g; s///g; ... 94 | # 95 | # This string will allow us to replace multiple values in a single call to Perl. 96 | # 97 | # https://stackoverflow.com/a/8934117/483528 98 | function format_replacement_string { 99 | local replacements="" 100 | local i old_name new_name 101 | for ((i=0;i<"${#REPLACEMENT_PAIRS[@]}";i+=2)); do 102 | old_name="${REPLACEMENT_PAIRS[i]}" 103 | new_name="${REPLACEMENT_PAIRS[i+1]}" 104 | 105 | # This is the sed-like regex for the replacements we'll be doing. To help create this regex, I used this handy 106 | # online regex tester, that not only gives you nice highlighting, but even lets you define a bunch of test cases to 107 | # check against! 108 | # 109 | # https://regexr.com/5l8n4 110 | # 111 | replacements+=" s/$ALLOWED_CHARS_BEFORE$old_name$ALLOWED_CHARS_AFTER/\$1$new_name\$2/g;" 112 | done 113 | 114 | # Strip extra space at start of string: https://stackoverflow.com/a/16623897/483528 115 | echo "${replacements# }" 116 | } 117 | 118 | # The main entrypoint for this script 119 | function update_all_repo_names { 120 | local replacements 121 | replacements=$(format_replacement_string) 122 | 123 | local files_to_update 124 | files_to_update=($(find_files_to_update)) 125 | 126 | local file 127 | for file in "${files_to_update[@]}"; do 128 | # I originally used sed, but on Mac, sed added an unnecessary newline at the end of every single file it touched, 129 | # so I switched to Perl. This also has the added benefit of allowing us to process multiple replacements in a 130 | # single call. 131 | perl -i -pe "${replacements[@]}" "$file" 132 | done 133 | } 134 | 135 | update_all_repo_names -------------------------------------------------------------------------------- /stats/stats.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/google/go-github/v43/github" 8 | "github.com/gruntwork-io/git-xargs/printer" 9 | "github.com/gruntwork-io/git-xargs/types" 10 | ) 11 | 12 | const ( 13 | // DryRunSet denotes a repo will not have any file changes, branches made or PRs opened because the dry-run flag was set to true 14 | DryRunSet types.Event = "dry-run-set-no-changes-made" 15 | // ReposSelected denotes all the repositories that were targeted for processing by this tool AFTER filtering was applied to determine valid repos 16 | ReposSelected types.Event = "repos-selected-pre-processing" 17 | // ReposArchivedSkipped denotes all the repositories that were skipped from the list of repos to clone because the skip-archived-repos was set to true 18 | ReposArchivedSkipped types.Event = "repos-archived-skipped" 19 | // TargetBranchNotFound denotes the special branch used by this tool to make changes on was not found on lookup, suggesting it should be created 20 | TargetBranchNotFound types.Event = "target-branch-not-found" 21 | // TargetBranchAlreadyExists denotes the special branch used by this tool was already found (so it was likely already created by a previous run) 22 | TargetBranchAlreadyExists types.Event = "target-branch-already-exists" 23 | // TargetBranchLookupErr denotes an issue performing the lookup via GitHub API for the target branch - an API call failure 24 | TargetBranchLookupErr types.Event = "target-branch-lookup-err" 25 | // TargetBranchSuccessfullyCreated denotes a repo for which the target branch was created via GitHub API call 26 | TargetBranchSuccessfullyCreated types.Event = "target-branch-successfully-created" 27 | // FetchedViaGithubAPI denotes a repo was successfully listed by calling the GitHub API 28 | FetchedViaGithubAPI types.Event = "fetch-via-github-api" 29 | // RepoSuccessfullyCloned denotes a repo that was cloned to the local filesystem of the operator's machine 30 | RepoSuccessfullyCloned types.Event = "repo-successfully-cloned" 31 | // RepoFailedToClone denotes that for whatever reason we were unable to clone the repo to the local system 32 | RepoFailedToClone types.Event = "repo-failed-to-clone" 33 | // BranchCheckoutFailed denotes a failure to checkout a new tool specific branch in the given repo 34 | BranchCheckoutFailed types.Event = "branch-checkout-failed" 35 | // GetHeadRefFailed denotes a repo for which the HEAD git reference could not be obtained 36 | GetHeadRefFailed types.Event = "get-head-ref-failed" 37 | // CommandErrorOccurredDuringExecution denotes a repo for which the supplied command failed to be executed 38 | CommandErrorOccurredDuringExecution types.Event = "command-error-during-execution" 39 | // WorktreeStatusCheckFailed denotes a repo whose git status command failed post command execution 40 | WorktreeStatusCheckFailed types.Event = "worktree-status-check-failed" 41 | // WorktreeStatusCheckFailedCommand denotes a repo whose git status command failed following command execution 42 | WorktreeStatusCheckFailedCommand = "worktree-status-check-failed-command" 43 | // WorktreeStatusDirty denotes repos that had local file changes following execution of all their targeted 44 | WorktreeStatusDirty types.Event = "worktree-status-dirty" 45 | // WorktreeStatusClean denotes a repo that did not have any local file changes following command execution 46 | WorktreeStatusClean types.Event = "worktree-status-clean" 47 | // WorktreeAddFileFailed denotes a failure to add at least one file to the git stage following command execution 48 | WorktreeAddFileFailed types.Event = "worktree-add-file-failed" 49 | // CommitChangesFailed denotes an error git committing our file changes to the local repo 50 | CommitChangesFailed types.Event = "commit-changes-failed" 51 | // PushBranchFailed denotes a repo whose new tool-specific branch could not be pushed to remote origin 52 | PushBranchFailed types.Event = "push-branch-failed" 53 | // PushBranchSkipped denotes a repo whose local branch was not pushed due to the --dry-run flag being set 54 | PushBranchSkipped types.Event = "push-branch-skipped" 55 | // RepoNotExists denotes a repo + org combo that was supplied via file but could not be successfully looked up via the GitHub API (returned a 404) 56 | RepoNotExists types.Event = "repo-not-exists" 57 | // PullRequestOpenErr denotes a repo whose pull request containing config changes could not be made successfully 58 | PullRequestOpenErr types.Event = "pull-request-open-error" 59 | // PullRequestAlreadyExists denotes a repo where the pull request already exists for the requested branch, so we didn't open a new one 60 | PullRequestAlreadyExists types.Event = "pull-request-already-exists" 61 | // CommitsMadeDirectlyToBranch denotes a repo whose local worktree changes were committed directly to the specified branch because the --skip-pull-requests flag was passed 62 | CommitsMadeDirectlyToBranch types.Event = "commits-made-directly-to-branch" 63 | // DirectCommitsPushedToRemoteBranch denotes a repo whose changes were pushed to the remote specified branch because the --skip-pull-requests flag was passed 64 | DirectCommitsPushedToRemoteBranch types.Event = "direct-commits-pushed-to-remote" 65 | // BranchRemotePullFailed denotes a repo whose remote branch could not be fetched successfully 66 | BranchRemotePullFailed types.Event = "branch-remote-pull-failed" 67 | // BranchRemoteDidntExistYet denotes a repo whose specified branch didn't exist remotely yet and so was just created locally to begin with 68 | BranchRemoteDidntExistYet types.Event = "branch-remote-didnt-exist-yet" 69 | // RepoFlagSuppliedRepoMalformed denotes a repo passed via the --repo flag that was malformed (perhaps missing it's Github org prefix) and therefore unprocessable 70 | RepoFlagSuppliedRepoMalformed types.Event = "repo-flag-supplied-repo-malformed" 71 | // RepoDoesntSupportDraftPullRequestsErr denotes a repo that is incompatible with the submitted pull request configuration 72 | RepoDoesntSupportDraftPullRequestsErr types.Event = "repo-not-compatible-with-pull-config" 73 | // BaseBranchTargetInvalidErr denotes a repo that does not have the base branch specified by the user 74 | BaseBranchTargetInvalidErr types.Event = "base-branch-target-invalid" 75 | // PRFailedDueToRateLimits denotes a repo whose initial pull request failed as a result of being rate limited by GitHub 76 | PRFailedDueToRateLimitsErr types.Event = "pr-failed-due-to-rate-limits" 77 | // PRFailedAfterMaximumRetriesErr denotes a repo whose pull requests all failed to be created via GitHub following the maximum number of retries 78 | PRFailedAfterMaximumRetriesErr types.Event = "pr-failed-after-maximum-retries" 79 | // RequestReviewersErr denotes a repo whose follow up request to add reviewers to the opened pull request failed 80 | RequestReviewersErr types.Event = "request-reviewers-error" 81 | ) 82 | 83 | var allEvents = []types.AnnotatedEvent{ 84 | {Event: FetchedViaGithubAPI, Description: "Repos successfully fetched via Github API"}, 85 | {Event: DryRunSet, Description: "Repos that were not modified in any way because this was a dry-run"}, 86 | {Event: ReposSelected, Description: "All repos that were targeted for processing after filtering missing / malformed repos"}, 87 | {Event: ReposArchivedSkipped, Description: "All repos that were filtered out with the --skip-archived-repos flag"}, 88 | {Event: TargetBranchNotFound, Description: "Repos whose target branch was not found"}, 89 | {Event: TargetBranchAlreadyExists, Description: "Repos whose target branch already existed"}, 90 | {Event: TargetBranchLookupErr, Description: "Repos whose target branches could not be looked up due to an API error"}, 91 | {Event: RepoSuccessfullyCloned, Description: "Repos that were successfully cloned to the local filesystem"}, 92 | {Event: RepoFailedToClone, Description: "Repos that were unable to be cloned to the local filesystem"}, 93 | {Event: BranchCheckoutFailed, Description: "Repos for which checking out a new tool-specific branch failed"}, 94 | {Event: GetHeadRefFailed, Description: "Repos for which the HEAD git reference could not be obtained"}, 95 | {Event: CommandErrorOccurredDuringExecution, Description: "Repos for which the supplied command raised an error during execution"}, 96 | {Event: WorktreeStatusCheckFailed, Description: "Repos for which the git status command failed following command execution"}, 97 | {Event: WorktreeStatusDirty, Description: "Repos that showed file changes to their working directory following command execution"}, 98 | {Event: WorktreeStatusClean, Description: "Repos that showed no file changes to their working directory following command execution"}, 99 | {Event: CommitChangesFailed, Description: "Repos whose file changes failed to be committed for some reason"}, 100 | {Event: PushBranchFailed, Description: "Repos whose tool-specific branch containing changes failed to push to remote origin"}, 101 | {Event: PushBranchSkipped, Description: "Repos whose local branch was not pushed because the --dry-run flag was set"}, 102 | {Event: RepoNotExists, Description: "Repos that were supplied by user but don't exist (404'd) via Github API"}, 103 | {Event: PullRequestOpenErr, Description: "Repos against which pull requests failed to be opened"}, 104 | {Event: PullRequestAlreadyExists, Description: "Repos where opening a pull request was skipped because a pull request was already open"}, 105 | {Event: CommitsMadeDirectlyToBranch, Description: "Repos whose local changes were committed directly to the specified branch because --skip-pull-requests was passed"}, 106 | {Event: DirectCommitsPushedToRemoteBranch, Description: "Repos whose changes were pushed directly to the remote branch because --skip-pull-requests was passed"}, 107 | {Event: BranchRemotePullFailed, Description: "Repos whose remote branches could not be successfully pulled"}, 108 | {Event: BranchRemoteDidntExistYet, Description: "Repos whose specified branches did not exist on the remote, and so were first created locally"}, 109 | {Event: RepoFlagSuppliedRepoMalformed, Description: "Repos passed via the --repo flag that were malformed (missing their Github org prefix?) and therefore unprocessable"}, 110 | {Event: RepoDoesntSupportDraftPullRequestsErr, Description: "Repos that do not support Draft PRs (--draft flag was passed)"}, 111 | {Event: BaseBranchTargetInvalidErr, Description: "Repos that did not have the branch specified by --base-branch-name"}, 112 | {Event: PRFailedDueToRateLimitsErr, Description: "Repos whose initial Pull Request failed to be created due to GitHub rate limits"}, 113 | {Event: PRFailedAfterMaximumRetriesErr, Description: "Repos whose Pull Request failed to be created after the maximum number of retries"}, 114 | {Event: RequestReviewersErr, Description: "Repos whose request to add reviewers to the opened pull request failed"}, 115 | } 116 | 117 | // RunStats will be a stats-tracker class that keeps score of which repos were touched, which were considered for update, which had branches made, PRs made, which were missing workflows or contexts, or had out of date workflows syntax values, etc 118 | type RunStats struct { 119 | selectionMode string 120 | repos map[types.Event][]*github.Repository 121 | skippedArchivedRepos map[types.Event][]*github.Repository 122 | pulls map[string]string 123 | draftpulls map[string]string 124 | command []string 125 | fileProvidedRepos []*types.AllowedRepo 126 | repoFlagProvidedRepos []*types.AllowedRepo 127 | startTime time.Time 128 | skipPullRequests bool 129 | mutex *sync.Mutex 130 | } 131 | 132 | // NewStatsTracker initializes a tracker struct that is capable of keeping tabs on which repos were handled and how 133 | func NewStatsTracker() *RunStats { 134 | var fileProvidedRepos []*types.AllowedRepo 135 | var repoFlagProvidedRepos []*types.AllowedRepo 136 | 137 | t := &RunStats{ 138 | repos: make(map[types.Event][]*github.Repository), 139 | skippedArchivedRepos: make(map[types.Event][]*github.Repository), 140 | pulls: make(map[string]string), 141 | draftpulls: make(map[string]string), 142 | command: []string{}, 143 | fileProvidedRepos: fileProvidedRepos, 144 | repoFlagProvidedRepos: repoFlagProvidedRepos, 145 | startTime: time.Now(), 146 | skipPullRequests: false, 147 | mutex: &sync.Mutex{}, 148 | } 149 | return t 150 | } 151 | 152 | // SetSelectionMode accepts a string representing the method by which repos were selected for this run - in order to print a human-legible description in the final report 153 | func (r *RunStats) SetSelectionMode(mode string) { 154 | r.selectionMode = mode 155 | } 156 | 157 | // GetSelectionMode returns the currently set repo selection method 158 | func (r *RunStats) GetSelectionMode() string { 159 | return r.selectionMode 160 | } 161 | 162 | // GetTotalRunSeconds returns the total time it took, in seconds, to run all the selected commands against all the targeted repos 163 | func (r *RunStats) GetTotalRunSeconds() int { 164 | s := time.Since(r.startTime).Seconds() 165 | return int(s) % 60 166 | } 167 | 168 | // GetRepos returns the inner map of events to *github.Repositories that the RunStats maintains throughout the lifecycle of a given command run 169 | func (r *RunStats) GetRepos() map[types.Event][]*github.Repository { 170 | return r.repos 171 | } 172 | 173 | // GetSkippedArchivedRepos returns the inner map of events to *github.Repositories that are excluded from the targeted repos list 174 | func (r *RunStats) GetSkippedArchivedRepos() map[types.Event][]*github.Repository { 175 | return r.skippedArchivedRepos 176 | } 177 | 178 | // GetPullRequests returns the inner representation of the pull requests that were opened during the lifecycle of a given run 179 | func (r *RunStats) GetPullRequests() map[string]string { 180 | return r.pulls 181 | } 182 | 183 | // GetDraftPullRequests returns the inner representation of the draft pull requests that were opened during the lifecycle of a given run 184 | func (r *RunStats) GetDraftPullRequests() map[string]string { 185 | return r.draftpulls 186 | } 187 | 188 | // SetFileProvidedRepos sets the number of repos that were provided via file by the user on startup (as opposed to looked up via GitHub API via the --github-org flag) 189 | func (r *RunStats) SetFileProvidedRepos(fileProvidedRepos []*types.AllowedRepo) { 190 | for _, ar := range fileProvidedRepos { 191 | r.fileProvidedRepos = append(r.fileProvidedRepos, ar) 192 | } 193 | } 194 | 195 | // GetFileProvidedRepos returns a slice of the repos that were provided via the --repos flag (as opposed to looked up via the GitHub API via the --github-org flag) 196 | func (r *RunStats) GetFileProvidedRepos() []*types.AllowedRepo { 197 | return r.fileProvidedRepos 198 | } 199 | 200 | // SetRepoFlagProvidedRepos sets the number of repos that were provided via a single or multiple invocations of the --repo flag 201 | func (r *RunStats) SetRepoFlagProvidedRepos(repoFlagProvidedRepos []*types.AllowedRepo) { 202 | for _, ar := range repoFlagProvidedRepos { 203 | r.repoFlagProvidedRepos = append(r.repoFlagProvidedRepos, ar) 204 | } 205 | } 206 | 207 | // SetSkipPullRequests tracks whether the user specified that pull requests should be skipped (in favor of committing and pushing directly to the specified branch) 208 | func (r *RunStats) SetSkipPullRequests(skipPullRequests bool) { 209 | r.skipPullRequests = skipPullRequests 210 | } 211 | 212 | // SetCommand sets the user-supplied command to be run against the targeted repos 213 | func (r *RunStats) SetCommand(c []string) { 214 | r.command = c 215 | } 216 | 217 | // GetMultiple returns the slice of pointers to GitHub repositories filed under the provided event's key 218 | func (r *RunStats) GetMultiple(event types.Event) []*github.Repository { 219 | return r.repos[event] 220 | } 221 | 222 | // TrackSingle accepts a types.Event to associate with the supplied repo so that a final report can be generated at the end of each run 223 | func (r *RunStats) TrackSingle(event types.Event, repo *github.Repository) { 224 | // TrackSingle is called from multiple concurrent writing goroutines, so we need to lock access to the underlying map 225 | defer r.mutex.Unlock() 226 | r.mutex.Lock() 227 | r.repos[event] = TrackEventIfMissing(r.repos[event], repo) 228 | } 229 | 230 | // TrackEventIfMissing prevents the addition of duplicates to the tracking slices. Repos may end up with file changes 231 | // for example, from multiple command runs, so we don't need the same repo repeated multiple times in the final report 232 | func TrackEventIfMissing(slice []*github.Repository, repo *github.Repository) []*github.Repository { 233 | for _, existingRepo := range slice { 234 | if existingRepo.GetName() == repo.GetName() { 235 | // We've already tracked this repo under this event, return the existing slice to avoid adding 236 | // it a second time 237 | return slice 238 | } 239 | } 240 | return append(slice, repo) 241 | } 242 | 243 | // TrackPullRequest stores the successful PR opening for the supplied Repo, at the supplied PR URL 244 | // This function is safe to call from concurrent goroutines 245 | func (r *RunStats) TrackPullRequest(repoName, prURL string) { 246 | defer r.mutex.Unlock() 247 | r.mutex.Lock() 248 | r.pulls[repoName] = prURL 249 | } 250 | 251 | // TrackDraftPullRequest stores the successful Draft PR opening for the supplied Repo, at the supplied PR URL 252 | // This function is safe to call from concurrent goroutines 253 | func (r *RunStats) TrackDraftPullRequest(repoName, prURL string) { 254 | defer r.mutex.Unlock() 255 | r.mutex.Lock() 256 | r.draftpulls[repoName] = prURL 257 | } 258 | 259 | // TrackMultiple accepts a types.Event and a slice of pointers to GitHub repos that will all be associated with that event 260 | func (r *RunStats) TrackMultiple(event types.Event, repos []*github.Repository) { 261 | for _, repo := range repos { 262 | r.TrackSingle(event, repo) 263 | } 264 | } 265 | 266 | // GenerateRunReport creates a struct that contains all the information necessary to print a final summary report 267 | func (r *RunStats) GenerateRunReport() *types.RunReport { 268 | return &types.RunReport{ 269 | Repos: r.GetRepos(), 270 | SkippedRepos: r.GetSkippedArchivedRepos(), 271 | Command: r.command, 272 | SelectionMode: r.selectionMode, 273 | RuntimeSeconds: r.GetTotalRunSeconds(), FileProvidedRepos: r.GetFileProvidedRepos(), 274 | PullRequests: r.GetPullRequests(), 275 | DraftPullRequests: r.GetDraftPullRequests(), 276 | } 277 | } 278 | 279 | // PrintReport renders to STDOUT a summary of each repo that was considered by this tool and what happened to it during processing 280 | func (r *RunStats) PrintReport() { 281 | printer.PrintRepoReport(allEvents, r.GenerateRunReport()) 282 | } 283 | -------------------------------------------------------------------------------- /types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/google/go-github/v43/github" 8 | ) 9 | 10 | // Event is a generic tracking occurrence that RunStats manages 11 | type Event string 12 | 13 | // ReducedRepo is a simplified form of the github.Repository struct 14 | type ReducedRepo struct { 15 | Name string `header:"Repo name"` 16 | URL string `header:"Repo url"` 17 | } 18 | 19 | type RunReport struct { 20 | Repos map[Event][]*github.Repository 21 | SkippedRepos map[Event][]*github.Repository 22 | Command []string 23 | SelectionMode string 24 | RuntimeSeconds int 25 | FileProvidedRepos []*AllowedRepo 26 | PullRequests map[string]string 27 | DraftPullRequests map[string]string 28 | } 29 | 30 | // AnnotatedEvent is used in printing the final report. It contains the info to print a section's table - both its Event for looking up the tagged repos, and the human-legible description for printing above the table 31 | type AnnotatedEvent struct { 32 | Event Event 33 | Description string 34 | } 35 | 36 | // AllowedRepo represents a single repository under a GitHub organization that this tool may operate on 37 | type AllowedRepo struct { 38 | Organization string `header:"Organization name"` 39 | Name string `header:"URL"` 40 | } 41 | 42 | type OpenPrRequest struct { 43 | Repo *github.Repository 44 | Branch string 45 | Delay time.Duration 46 | Retries int 47 | } 48 | 49 | // PullRequest is a simple two column representation of the repo name and its PR url 50 | type PullRequest struct { 51 | Repo string `header:"Repo name"` 52 | URL string `header:"PR URL"` 53 | } 54 | 55 | type NoArgumentsPassedErr struct{} 56 | 57 | func (NoArgumentsPassedErr) Error() string { 58 | return fmt.Sprint("You must pass a valid command or script path to git-xargs") 59 | } 60 | 61 | type NoGithubOrgSuppliedErr struct{} 62 | 63 | func (NoGithubOrgSuppliedErr) Error() string { 64 | return fmt.Sprint("You must pass a valid Github organization name") 65 | } 66 | 67 | type NoRepoSelectionsMadeErr struct{} 68 | 69 | func (NoRepoSelectionsMadeErr) Error() string { 70 | return fmt.Sprint("You must target some repos for processing either via stdin or by providing one of the --github-org, --repos, or --repo flags") 71 | } 72 | 73 | type NoRepoFlagTargetsValid struct{} 74 | 75 | func (NoRepoFlagTargetsValid) Error() string { 76 | return fmt.Sprint("None of the repos specified via the --repo flag are valid. Please double-check you have included the Github org prefix for each - e.g. --repo gruntwork-io/git-xargs") 77 | } 78 | 79 | type NoBranchNameErr struct{} 80 | 81 | func (NoBranchNameErr) Error() string { 82 | return fmt.Sprint("You must pass a branch name to use via the --branch-name flag") 83 | } 84 | 85 | type NoReposFoundErr struct { 86 | GithubOrg string 87 | } 88 | 89 | func (err NoReposFoundErr) Error() string { 90 | return fmt.Sprintf("No repos found for the organization supplied via --github-org: %s", err.GithubOrg) 91 | } 92 | 93 | type NoValidReposFoundAfterFilteringErr struct{} 94 | 95 | func (NoValidReposFoundAfterFilteringErr) Error() string { 96 | return fmt.Sprint("No valid repos were found after filtering out malformed input") 97 | } 98 | 99 | type NoCommandSuppliedErr struct{} 100 | 101 | func (NoCommandSuppliedErr) Error() string { 102 | return fmt.Sprintf("You must supply a valid command or script to execute") 103 | } 104 | 105 | type NoGithubOauthTokenProvidedErr struct{} 106 | 107 | func (NoGithubOauthTokenProvidedErr) Error() string { 108 | return fmt.Sprintf("You must export a valid Github personal access token as GITHUB_OAUTH_TOKEN") 109 | } 110 | -------------------------------------------------------------------------------- /types/types_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | // TestCustomErrorStatements performs quick sanity checks on the custom error types to ensure they return the expected messages 10 | func TestCustomErrorStatements(t *testing.T) { 11 | t.Parallel() 12 | 13 | err := &NoArgumentsPassedErr{} 14 | assert.Equal(t, "You must pass a valid command or script path to git-xargs", err.Error()) 15 | 16 | errNoGithubOrg := &NoGithubOrgSuppliedErr{} 17 | assert.Equal(t, "You must pass a valid Github organization name", errNoGithubOrg.Error()) 18 | 19 | errNoRepoSelected := &NoRepoSelectionsMadeErr{} 20 | assert.Equal(t, "You must target some repos for processing either via stdin or by providing one of the --github-org, --repos, or --repo flags", errNoRepoSelected.Error()) 21 | 22 | errNoReposFound := &NoReposFoundErr{GithubOrg: "gruntwork-io"} 23 | assert.Equal(t, "No repos found for the organization supplied via --github-org: gruntwork-io", errNoReposFound.Error()) 24 | 25 | errNoValidReposFoundAfterFiltering := NoValidReposFoundAfterFilteringErr{} 26 | assert.Equal(t, "No valid repos were found after filtering out malformed input", errNoValidReposFoundAfterFiltering.Error()) 27 | 28 | errNoCommandSupplied := NoCommandSuppliedErr{} 29 | assert.Equal(t, "You must supply a valid command or script to execute", errNoCommandSupplied.Error()) 30 | 31 | errNoGithubOauthTokenProvided := NoGithubOauthTokenProvidedErr{} 32 | assert.Equal(t, "You must export a valid Github personal access token as GITHUB_OAUTH_TOKEN", errNoGithubOauthTokenProvided.Error()) 33 | 34 | } 35 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/gruntwork-io/git-xargs/types" 10 | "github.com/gruntwork-io/go-commons/logging" 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 15 | 16 | // ConvertStringToAllowedRepo accepts a user-supplied repo in the format of /. 17 | // It trims out stray characters that we might expect in a repos file that was copy-pasted from json or an array, 18 | // and it only returns an AllowedRepo if the user-supplied input looks valid. Note this does not actually look 19 | // up the repo via the GitHub API because that's slow, and we do it later when converting repo names to GitHub response structs. 20 | func ConvertStringToAllowedRepo(repoInput string) *types.AllowedRepo { 21 | 22 | logger := logging.GetLogger("git-xargs") 23 | 24 | // The regex for all common special characters to remove from the repo lines in the allowed repos file 25 | charRegex := regexp.MustCompile(`['",!]`) 26 | 27 | trimmedLine := strings.TrimSpace(repoInput) 28 | cleanedLine := charRegex.ReplaceAllString(trimmedLine, "") 29 | orgAndRepoSlice := strings.Split(cleanedLine, "/") 30 | // Guard against stray lines, extra dangling single quotes, etc 31 | if len(orgAndRepoSlice) < 2 { 32 | 33 | logger.WithFields(logrus.Fields{ 34 | "Repo input": repoInput, 35 | }).Debug("Malformed repo input detected - skipping") 36 | 37 | return nil 38 | } 39 | 40 | // Validate both the org and name are not empty 41 | parsedOrg := orgAndRepoSlice[0] 42 | parsedName := orgAndRepoSlice[1] 43 | 44 | // If both org name and repo name are present, create a new allowed repo and add it to the list 45 | if parsedOrg != "" && parsedName != "" { 46 | repo := &types.AllowedRepo{ 47 | Organization: parsedOrg, 48 | Name: parsedName, 49 | } 50 | return repo 51 | } 52 | 53 | logger.WithFields(logrus.Fields{ 54 | "Repo input": repoInput, 55 | }).Debug("Could not parse a valid repo from input. Repo must be specified in format /, e.g., gruntwork-io/cloud-nuke") 56 | 57 | return nil 58 | } 59 | 60 | func RandStringBytes(n int) string { 61 | b := make([]byte, n) 62 | for i := range b { 63 | b[i] = letterBytes[rand.Intn(len(letterBytes))] 64 | } 65 | return string(b) 66 | } 67 | 68 | func NewTestFileName() string { 69 | return fmt.Sprintf("test-file-%s", RandStringBytes(9)) 70 | } 71 | --------------------------------------------------------------------------------