├── .circleci └── config.yml ├── .gitignore ├── .jdk8 ├── .mvn ├── jvm.config ├── maven.config └── wrapper │ ├── MavenWrapperDownloader.java │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .settings.xml ├── LICENSE ├── LICENSE.txt ├── README.adoc ├── mvnw ├── mvnw.cmd ├── pom.xml └── src ├── main └── java │ └── io │ └── cloudpipelines │ └── projectcrawler │ ├── BitbucketRepositoryManagementBuilder.java │ ├── GithubRepositoryManagementBuilder.java │ ├── GitlabRepositoryManagementBuilder.java │ ├── Options.java │ ├── OptionsBuilder.java │ ├── ProjectAndBranch.java │ ├── ProjectCrawler.java │ ├── Repositories.java │ ├── Repository.java │ ├── RepositoryManagement.java │ └── RepositoryManagementBuilder.java └── test ├── java └── io │ └── cloudpipelines │ └── projectcrawler │ ├── BitbucketRepositoryManagementBuilderTests.java │ ├── GithubRepositoryManagementBuilderTests.java │ ├── GithubRepositoryManagementTests.java │ ├── GitlabRepositoryManagementBuilderTests.java │ ├── OptionsBuilderTest.java │ ├── OptionsTest.java │ ├── ProjectCrawlerTests.java │ ├── RepositoryManagementBuilderTests.java │ ├── TestRepositoryManagementBuilder.java │ └── util │ └── SocketUtils.java └── resources ├── META-INF └── services │ └── io.cloudpipelines.projectcrawler.RepositoryManagementBuilder ├── bitbucket └── projects.json ├── gitlab ├── curls ├── v4_file_from_path.json └── v4_projects_for_group.json ├── logback.xml └── spring_cloud_repos.json /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: springcloud/pipeline-base 6 | environment: 7 | _JAVA_OPTIONS: "-Xms1024m -Xmx2048m" 8 | branches: 9 | ignore: 10 | - gh-pages # list of branches to ignore 11 | steps: 12 | - checkout 13 | - run: 14 | name: "Running build" 15 | command: ./mvnw clean org.jacoco:jacoco-maven-plugin:prepare-agent install -U -P sonar -nsu --batch-mode -Dmaven.test.redirectTestOutputToFile=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn 16 | - run: 17 | name: "Aggregate test results" 18 | when: always 19 | command: | 20 | mkdir -p $CIRCLE_TEST_REPORTS/junit/ 21 | find . -type f -regex ".*/target/.*-reports/.*" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \; 22 | bash <(curl -s https://codecov.io/bash) 23 | - store_artifacts: 24 | path: /junit/ 25 | destination: artifacts 26 | - store_test_results: 27 | path: /junit/ 28 | destination: testartifacts 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | 12 | ### IntelliJ IDEA ### 13 | .idea 14 | *.iws 15 | *.iml 16 | *.ipr 17 | 18 | ### NetBeans ### 19 | nbproject/private/ 20 | build/ 21 | nbbuild/ 22 | dist/ 23 | nbdist/ 24 | .nb-gradle/ 25 | 26 | .DS_Store 27 | *.log 28 | _site 29 | effective.pom 30 | .vscode/ 31 | bin/ -------------------------------------------------------------------------------- /.jdk8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/cloudpipelines-project-crawler/e6693deb40ded0d75cc3a369162a8caec037f434/.jdk8 -------------------------------------------------------------------------------- /.mvn/jvm.config: -------------------------------------------------------------------------------- 1 | -Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom -------------------------------------------------------------------------------- /.mvn/maven.config: -------------------------------------------------------------------------------- 1 | -DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring 2 | -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | https://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | */ 19 | 20 | import java.net.*; 21 | import java.io.*; 22 | import java.nio.channels.*; 23 | import java.util.Properties; 24 | 25 | public class MavenWrapperDownloader { 26 | 27 | /** 28 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 29 | */ 30 | private static final String DEFAULT_DOWNLOAD_URL = 31 | "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar"; 32 | 33 | /** 34 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 35 | * use instead of the default one. 36 | */ 37 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 38 | ".mvn/wrapper/maven-wrapper.properties"; 39 | 40 | /** 41 | * Path where the maven-wrapper.jar will be saved to. 42 | */ 43 | private static final String MAVEN_WRAPPER_JAR_PATH = 44 | ".mvn/wrapper/maven-wrapper.jar"; 45 | 46 | /** 47 | * Name of the property which should be used to override the default download url for the wrapper. 48 | */ 49 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 50 | 51 | public static void main(String args[]) { 52 | System.out.println("- Downloader started"); 53 | File baseDirectory = new File(args[0]); 54 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 55 | 56 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 57 | // wrapperUrl parameter. 58 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 59 | String url = DEFAULT_DOWNLOAD_URL; 60 | if(mavenWrapperPropertyFile.exists()) { 61 | FileInputStream mavenWrapperPropertyFileInputStream = null; 62 | try { 63 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 64 | Properties mavenWrapperProperties = new Properties(); 65 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 66 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 67 | } catch (IOException e) { 68 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 69 | } finally { 70 | try { 71 | if(mavenWrapperPropertyFileInputStream != null) { 72 | mavenWrapperPropertyFileInputStream.close(); 73 | } 74 | } catch (IOException e) { 75 | // Ignore ... 76 | } 77 | } 78 | } 79 | System.out.println("- Downloading from: : " + url); 80 | 81 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 82 | if(!outputFile.getParentFile().exists()) { 83 | if(!outputFile.getParentFile().mkdirs()) { 84 | System.out.println( 85 | "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 86 | } 87 | } 88 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 89 | try { 90 | downloadFileFromURL(url, outputFile); 91 | System.out.println("Done"); 92 | System.exit(0); 93 | } catch (Throwable e) { 94 | System.out.println("- Error downloading"); 95 | e.printStackTrace(); 96 | System.exit(1); 97 | } 98 | } 99 | 100 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 101 | URL website = new URL(urlString); 102 | ReadableByteChannel rbc; 103 | rbc = Channels.newChannel(website.openStream()); 104 | FileOutputStream fos = new FileOutputStream(destination); 105 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 106 | fos.close(); 107 | rbc.close(); 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/cloudpipelines-project-crawler/e6693deb40ded0d75cc3a369162a8caec037f434/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip -------------------------------------------------------------------------------- /.settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | repo.spring.io 6 | ${env.CI_DEPLOY_USERNAME} 7 | ${env.CI_DEPLOY_PASSWORD} 8 | 9 | 10 | 11 | 12 | 18 | spring 19 | true 20 | 21 | 22 | spring-snapshots 23 | Spring Snapshots 24 | https://repo.spring.io/libs-snapshot-local 25 | 26 | true 27 | 28 | 29 | 30 | spring-milestones 31 | Spring Milestones 32 | https://repo.spring.io/libs-milestone-local 33 | 34 | false 35 | 36 | 37 | 38 | spring-releases 39 | Spring Releases 40 | https://repo.spring.io/release 41 | 42 | false 43 | 44 | 45 | 46 | 47 | 48 | spring-snapshots 49 | Spring Snapshots 50 | https://repo.spring.io/libs-snapshot-local 51 | 52 | true 53 | 54 | 55 | 56 | spring-milestones 57 | Spring Milestones 58 | https://repo.spring.io/libs-milestone-local 59 | 60 | false 61 | 62 | 63 | 64 | 65 | 66 | 69 | ide 70 | true 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2015-Present Pivotal Software Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | 205 | 206 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | https://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | https://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | # cloudpipelines-project-crawler is no longer actively maintained by VMware, Inc. 2 | 3 | :jdkversion: 1.8 4 | :org: CloudPipelines 5 | :repo: project-crawler 6 | :branch: master 7 | 8 | image::https://circleci.com/gh/{org}/{repo}/tree/{branch}.svg?style=svg["CircleCI", link="https://circleci.com/gh/{org}/{repo}/tree/{branch}"] 9 | image::https://codecov.io/gh/{org}/{repo}/branch/{branch}/graph/badge.svg["codecov", link="https://codecov.io/gh/{org}/{repo}"] 10 | 11 | :toc: left 12 | :toclevels: 8 13 | :nofooter: 14 | 15 | == Project Crawler 16 | 17 | Small project to iterate over and fetch files from different repository management tools like Github, Gitlab, BitBucket 18 | 19 | == Examples 20 | 21 | Fetch the repositories for an org and download a file from it 22 | 23 | ```groovy 24 | ProjectCrawler crawler = new ProjectCrawler(OptionsBuilder.builder() 25 | // basing the root URL we can resolve the type of repo (e.g. https://gitlab.com) 26 | .rootUrl(urlRoot) 27 | // username to access the API 28 | .username(username) 29 | // password to access the API 30 | .password(password) 31 | // token to access the API 32 | .token(token) 33 | // repository type (GITHUB, BITBUCKET, GITLAB, OTHER) 34 | .repository(repoType) 35 | .build()); 36 | // get the repos from the org 37 | List repositories = crawler.repositories(org); 38 | repositories.each { Repository repo -> 39 | // fetch a file from the repository 40 | String file = crawler.fileContent(org, repo.name, repo.requestedBranch, "path/to/file.txt") 41 | } 42 | ``` 43 | 44 | For BitBucket: 45 | 46 | * remember to pass the URL to the API (e.g. `https://api.bitbucket.org`). 47 | * we support only the 2.0 API. 48 | 49 | 50 | == Adding your own implementation 51 | 52 | If you're using some other tool than Github, Gitlab or Bitbucket you can 53 | write an implementation that integrates with that tool. 54 | 55 | We're using the standard, Java `ServiceLoader` mechanism to load any extensions 56 | and the interface to implement is `RepositoryManagementBuilder`. 57 | 58 | To do that just create a file called `META-INF/io.cloudpipelines.projectcrawler.RepositoryManagementBuilder` 59 | that is accessible on classpath. The file should contain a line with fully 60 | qualified name of your implementation class (e.g. `com.example.TestRepositoryManagementBuilder`) -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | ########################################################################################## 204 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 205 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 206 | ########################################################################################## 207 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 208 | if [ "$MVNW_VERBOSE" = true ]; then 209 | echo "Found .mvn/wrapper/maven-wrapper.jar" 210 | fi 211 | else 212 | if [ "$MVNW_VERBOSE" = true ]; then 213 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 214 | fi 215 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar" 216 | while IFS="=" read key value; do 217 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 218 | esac 219 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 220 | if [ "$MVNW_VERBOSE" = true ]; then 221 | echo "Downloading from: $jarUrl" 222 | fi 223 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 224 | 225 | if command -v wget > /dev/null; then 226 | if [ "$MVNW_VERBOSE" = true ]; then 227 | echo "Found wget ... using wget" 228 | fi 229 | wget "$jarUrl" -O "$wrapperJarPath" 230 | elif command -v curl > /dev/null; then 231 | if [ "$MVNW_VERBOSE" = true ]; then 232 | echo "Found curl ... using curl" 233 | fi 234 | curl -o "$wrapperJarPath" "$jarUrl" 235 | else 236 | if [ "$MVNW_VERBOSE" = true ]; then 237 | echo "Falling back to using Java to download" 238 | fi 239 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 240 | if [ -e "$javaClass" ]; then 241 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 242 | if [ "$MVNW_VERBOSE" = true ]; then 243 | echo " - Compiling MavenWrapperDownloader.java ..." 244 | fi 245 | # Compiling the Java class 246 | ("$JAVA_HOME/bin/javac" "$javaClass") 247 | fi 248 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 249 | # Running the downloader 250 | if [ "$MVNW_VERBOSE" = true ]; then 251 | echo " - Running MavenWrapperDownloader.java ..." 252 | fi 253 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 254 | fi 255 | fi 256 | fi 257 | fi 258 | ########################################################################################## 259 | # End of extension 260 | ########################################################################################## 261 | 262 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 263 | if [ "$MVNW_VERBOSE" = true ]; then 264 | echo $MAVEN_PROJECTBASEDIR 265 | fi 266 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 267 | 268 | # For Cygwin, switch paths to Windows format before running java 269 | if $cygwin; then 270 | [ -n "$M2_HOME" ] && 271 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 272 | [ -n "$JAVA_HOME" ] && 273 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 274 | [ -n "$CLASSPATH" ] && 275 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 276 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 277 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 278 | fi 279 | 280 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 281 | 282 | exec "$JAVACMD" \ 283 | $MAVEN_OPTS \ 284 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 285 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 286 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 287 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM https://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar" 124 | FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( 125 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 126 | ) 127 | 128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 130 | if exist %WRAPPER_JAR% ( 131 | echo Found %WRAPPER_JAR% 132 | ) else ( 133 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 134 | echo Downloading from: %DOWNLOAD_URL% 135 | powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" 136 | echo Finished downloading %WRAPPER_JAR% 137 | ) 138 | @REM End of extension 139 | 140 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 141 | if ERRORLEVEL 1 goto error 142 | goto end 143 | 144 | :error 145 | set ERROR_CODE=1 146 | 147 | :end 148 | @endlocal & set ERROR_CODE=%ERROR_CODE% 149 | 150 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 151 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 152 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 153 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 154 | :skipRcPost 155 | 156 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 157 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 158 | 159 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 160 | 161 | exit /B %ERROR_CODE% 162 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | io.cloudpipelines 8 | project-crawler 9 | 1.0.0.BUILD-SNAPSHOT 10 | jar 11 | 12 | 13 | org.springframework.cloud 14 | spring-cloud-build 15 | 2.1.7.RELEASE 16 | 17 | 18 | 19 | 20 | 21 | UTF-8 22 | 1.8 23 | 24 | Greenwich.SR3 25 | 1.5.2 26 | 1.2.3 27 | 1.2 28 | 2.9.10 29 | 2.9.10.7 30 | 1.0 31 | 1.1.4 32 | 4.1.0 33 | 4.2.0 34 | 2.7 35 | 3.13.2 36 | 3.0.0 37 | 38 | 39 | 40 | 41 | 42 | org.springframework.cloud 43 | spring-cloud-dependencies 44 | ${spring-cloud-bom.version} 45 | pom 46 | import 47 | 48 | 49 | 50 | 51 | 52 | 53 | ch.qos.logback 54 | logback-classic 55 | ${logback-classic.version} 56 | 57 | 58 | commons-logging 59 | commons-logging 60 | ${commons-logging.version} 61 | 62 | 63 | com.fasterxml.jackson.core 64 | jackson-core 65 | ${jackson-core.version} 66 | 67 | 68 | com.fasterxml.jackson.core 69 | jackson-databind 70 | ${jackson-databind.version} 71 | 72 | 73 | 74 | com.jcabi 75 | jcabi-github 76 | ${jcabi-github.version} 77 | 78 | 79 | org.glassfish 80 | javax.json 81 | ${javax.json.version} 82 | 83 | 84 | 85 | org.gitlab 86 | java-gitlab-api 87 | ${java-gitlab-api.version} 88 | 89 | 90 | 91 | com.squareup.okhttp3 92 | okhttp 93 | ${okhttp.version} 94 | 95 | 96 | 97 | org.junit.jupiter 98 | junit-jupiter-engine 99 | test 100 | 101 | 102 | commons-io 103 | commons-io 104 | ${commons-io.version} 105 | test 106 | 107 | 108 | org.assertj 109 | assertj-core 110 | ${assertj-core.version} 111 | test 112 | 113 | 114 | org.mockito 115 | mockito-core 116 | ${mockito-core.version} 117 | test 118 | 119 | 120 | 121 | 122 | 123 | 124 | maven-surefire-plugin 125 | 3.0.0-M3 126 | 127 | 128 | 129 | 130 | 131 | 132 | sonar 133 | 134 | 135 | 136 | org.jacoco 137 | jacoco-maven-plugin 138 | 139 | 140 | pre-unit-test 141 | 142 | prepare-agent 143 | 144 | 145 | surefireArgLine 146 | ${project.build.directory}/jacoco.exec 147 | 148 | 149 | 150 | post-unit-test 151 | test 152 | 153 | report 154 | 155 | 156 | 157 | ${project.build.directory}/jacoco.exec 158 | 159 | 160 | 161 | 162 | 163 | org.apache.maven.plugins 164 | maven-surefire-plugin 165 | 166 | 167 | ${surefireArgLine} 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | jcenter 178 | https://jcenter.bintray.com/ 179 | 180 | 181 | spring-snapshots 182 | Spring Snapshots 183 | https://repo.spring.io/libs-snapshot-local 184 | 185 | true 186 | 187 | 188 | false 189 | 190 | 191 | 192 | spring-milestones 193 | Spring Milestones 194 | https://repo.spring.io/libs-milestone-local 195 | 196 | false 197 | 198 | 199 | 200 | spring-releases 201 | Spring Releases 202 | https://repo.spring.io/release 203 | 204 | false 205 | 206 | 207 | 208 | 209 | 210 | spring-snapshots 211 | Spring Snapshots 212 | https://repo.spring.io/libs-snapshot-local 213 | 214 | true 215 | 216 | 217 | false 218 | 219 | 220 | 221 | spring-milestones 222 | Spring Milestones 223 | https://repo.spring.io/libs-milestone-local 224 | 225 | false 226 | 227 | 228 | 229 | spring-releases 230 | Spring Releases 231 | https://repo.spring.io/libs-release-local 232 | 233 | false 234 | 235 | 236 | 237 | 238 | 239 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/BitbucketRepositoryManagementBuilder.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.stream.Collectors; 9 | 10 | import com.fasterxml.jackson.databind.ObjectMapper; 11 | import okhttp3.Credentials; 12 | import okhttp3.OkHttpClient; 13 | import okhttp3.Request; 14 | import okhttp3.Response; 15 | import okhttp3.ResponseBody; 16 | import org.apache.commons.lang3.StringUtils; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | /** 21 | * @author Marcin Grzejszczak 22 | */ 23 | class BitbucketRepositoryManagementBuilder implements RepositoryManagementBuilder { 24 | 25 | private static final Logger log = LoggerFactory.getLogger(GitlabRepositoryManagementBuilder.class); 26 | 27 | @Override public RepositoryManagement build(Options options) { 28 | boolean applicable = isApplicable(options.rootUrl); 29 | if (applicable) { 30 | return createNewRepoManagement(options); 31 | } 32 | if (options.repository != Repositories.BITBUCKET) { 33 | return null; 34 | } 35 | return createNewRepoManagement(options); 36 | } 37 | 38 | RepositoryManagement createNewRepoManagement(Options options) { 39 | return new BitbucketRepositoryManagement(options); 40 | } 41 | 42 | private boolean isApplicable(String url) { 43 | boolean applicable = StringUtils.isNotBlank(url) && url.contains("bitbucket"); 44 | if (log.isDebugEnabled()) { 45 | log.debug("URL [{}] is applicable [{}]", url, applicable); 46 | } 47 | return applicable; 48 | } 49 | } 50 | 51 | class BitbucketRepositoryManagement implements RepositoryManagement { 52 | 53 | private static final Logger log = LoggerFactory.getLogger(BitbucketRepositoryManagement.class); 54 | 55 | private final OkHttpClient client; 56 | private final Options options; 57 | private final ObjectMapper objectMapper = new ObjectMapper(); 58 | 59 | BitbucketRepositoryManagement(Options options) { 60 | this.client = connect(options); 61 | this.options = options; 62 | } 63 | 64 | private OkHttpClient connect(Options options) { 65 | if (StringUtils.isNotBlank(options.token)) { 66 | return new OkHttpClient.Builder() 67 | .followRedirects(true) 68 | .followSslRedirects(true) 69 | .authenticator((route, response) -> { 70 | if (response.request().header("Authorization") != null) { 71 | return null; // Give up, we've already attempted to authenticate. 72 | } 73 | if (log.isDebugEnabled()) { 74 | log.debug("Authenticating for response: " + response); 75 | log.debug("Challenges: " + response.challenges()); 76 | } 77 | return response.request().newBuilder() 78 | .header("Authorization Bearer", options.token) 79 | .build(); 80 | }) 81 | .build(); 82 | } else if (StringUtils.isNotBlank(options.username)) { 83 | return new OkHttpClient.Builder() 84 | .followRedirects(true) 85 | .followSslRedirects(true) 86 | .authenticator((route, response) -> { 87 | if (response.request().header("Authorization") != null) { 88 | return null; // Give up, we've already attempted to authenticate. 89 | } 90 | if (log.isDebugEnabled()) { 91 | log.debug("Authenticating for response: " + response); 92 | log.debug("Challenges: " + response.challenges()); 93 | } 94 | String credential = Credentials.basic(options.username, options.password); 95 | return response.request().newBuilder() 96 | .header("Authorization", credential) 97 | .build(); 98 | }) 99 | .build(); 100 | } 101 | throw new IllegalStateException("Neither token, nor username and password passed"); 102 | } 103 | 104 | @Override public List repositories(String org) { 105 | try { 106 | List repositories = new ArrayList<>(); 107 | Map map = null; 108 | int page = 1; 109 | while (map == null || map.containsKey("next")) { 110 | log.info("Grabbing page [" + page + "]"); 111 | Response execute = callRepositories(org, page); 112 | ResponseBody body = execute.body(); 113 | if (execute.code() >= 400) { 114 | throw new IllegalStateException("Status code [" + execute.code() + "] and body [" + body 115 | + "]"); 116 | } 117 | String response = body != null ? body.string() : ""; 118 | map = this.objectMapper.readValue(response, Map.class); 119 | List nonFilteredOutProjects = allNonFilteredOutProjects((List) map.get("values")); 120 | repositories.addAll(addManuallySetProjects(org, nonFilteredOutProjects)); 121 | page = page + 1; 122 | } 123 | return repositories; 124 | } catch (IOException e) { 125 | throw new IllegalStateException(e); 126 | } 127 | } 128 | 129 | Response callRepositories(String org, int page) throws IOException { 130 | return this.client.newCall( 131 | new Request.Builder().get().url(rootUrl() + "repositories/" + org + "?page=" + page) 132 | .build()).execute(); 133 | } 134 | 135 | private String rootUrl() { 136 | String url = this.options.rootUrl.endsWith("/") ? this.options.rootUrl : this.options.rootUrl + "/"; 137 | // we support only version 2.0 of the API 138 | return url + "2.0/"; 139 | } 140 | 141 | private List allNonFilteredOutProjects(List map) { 142 | return map.stream() 143 | .map(entry -> new Repository( 144 | options.projectName((String) entry.get("name")), 145 | url(entry, "ssh"), 146 | url(entry, "https"), 147 | "master")) 148 | .filter(repo -> !options.isIgnored(repo.name)) 149 | .collect(Collectors.toList()); 150 | } 151 | 152 | private String url(Map project, String name) { 153 | Map links = (Map) project.get("links"); 154 | List> clone = (List>) links.get("clone"); 155 | List strings = clone.stream() 156 | .filter(map -> name.equals(map.get("name"))) 157 | .map(map -> map.get("href")) 158 | .collect(Collectors.toList()); 159 | return strings.get(0); 160 | } 161 | 162 | private List addManuallySetProjects(String org, List repositories) { 163 | repositories.addAll(this.options.projects 164 | .stream().map(pb -> new Repository(options.projectName(pb.projectName), 165 | sshKey(org, pb), cloneUrl(org, pb), pb.branch)) 166 | .collect(Collectors.toSet())); 167 | return repositories; 168 | } 169 | 170 | private String sshKey(String org, ProjectAndBranch pb) { 171 | return "git@" + host() + ":" + org + "/" + pb.project + ".git"; 172 | } 173 | 174 | private String host() { 175 | return URI.create(this.options.rootUrl).getHost(); 176 | } 177 | 178 | private String cloneUrl(String org, ProjectAndBranch pb) { 179 | return "https://" + host() + "/" + org + "/" + pb.project + ".git"; 180 | } 181 | 182 | @Override public String fileContent(String org, String repo, 183 | String branch, String filePath) { 184 | return getDescriptor(org, repo, branch, filePath); 185 | } 186 | 187 | String getDescriptor(String org, String repo, String branch, 188 | String filePath) { 189 | try { 190 | return this.client 191 | .newCall(new Request.Builder() 192 | .url(rootUrl() + "repositories/" + org + "/" + repo + "/src/" + branch + "/" + filePath) 193 | .get() 194 | .build()).execute().body().string(); 195 | } 196 | catch (IOException e) { 197 | throw new IllegalStateException(e); 198 | } 199 | } 200 | } 201 | 202 | 203 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/GithubRepositoryManagementBuilder.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.ArrayList; 6 | import java.util.Collection; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.stream.Collectors; 10 | 11 | import com.fasterxml.jackson.databind.ObjectMapper; 12 | import com.jcabi.github.Coordinates; 13 | import com.jcabi.github.Github; 14 | import com.jcabi.github.RtGithub; 15 | import com.jcabi.http.Response; 16 | import com.jcabi.http.wire.RetryWire; 17 | import org.apache.commons.lang3.StringUtils; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | /** 22 | * @author Marcin Grzejszczak 23 | */ 24 | class GithubRepositoryManagementBuilder implements RepositoryManagementBuilder { 25 | 26 | private static final Logger log = LoggerFactory.getLogger(GithubRepositoryManagementBuilder.class); 27 | 28 | @Override public RepositoryManagement build(Options options) { 29 | boolean applicable = isApplicable(options.rootUrl); 30 | if (applicable) { 31 | return createNewRepoManagement(options); 32 | } 33 | if (options.repository != Repositories.GITHUB) { 34 | return null; 35 | } 36 | return createNewRepoManagement(options); 37 | } 38 | 39 | RepositoryManagement createNewRepoManagement(Options options) { 40 | return new GithubRepositoryManagement(options); 41 | } 42 | 43 | private boolean isApplicable(String url) { 44 | boolean applicable = StringUtils.isNotBlank(url) && url.contains("github"); 45 | if (log.isDebugEnabled()) { 46 | log.debug("URL [{}] is applicable [{}]", url, applicable); 47 | } 48 | return applicable; 49 | } 50 | } 51 | 52 | class GithubRepositoryManagement implements RepositoryManagement { 53 | 54 | private static final Logger log = LoggerFactory.getLogger(GithubRepositoryManagement.class); 55 | 56 | private final Github github; 57 | private final ObjectMapper objectMapper = new ObjectMapper(); 58 | private final Options options; 59 | 60 | GithubRepositoryManagement(Options options) { 61 | this.github = new RtGithub(github(options) 62 | .entry().through(RetryWire.class)); 63 | this.options = options; 64 | } 65 | 66 | GithubRepositoryManagement(Github github, Options options) { 67 | this.github = github; 68 | this.options = options; 69 | } 70 | 71 | private Github github(Options options) { 72 | if (StringUtils.isNotBlank(options.token)) { 73 | log.info("Token passed to github client"); 74 | return new RtGithub(options.token); 75 | } 76 | if (StringUtils.isNotBlank(options.username)) { 77 | log.info("Username and password passed to github client"); 78 | return new RtGithub(options.username, options.password); 79 | } 80 | log.info("No security passed to github client"); 81 | return new RtGithub(); 82 | } 83 | 84 | @Override public List repositories(String org) { 85 | try { 86 | List responses = orgRepos(org); 87 | List map = responses.stream() 88 | .map(this::read) 89 | .flatMap(Collection::stream) 90 | .collect(Collectors.toList()); 91 | List repositories = allNonFilteredOutProjects(map); 92 | return addManuallySetProjects(org, repositories); 93 | } 94 | catch (IOException e) { 95 | throw new IllegalStateException(e); 96 | } 97 | } 98 | 99 | private List read(String response) { 100 | try { 101 | return this.objectMapper.readValue(response, List.class); 102 | } 103 | catch (IOException e) { 104 | throw new IllegalStateException(e); 105 | } 106 | } 107 | 108 | private List allNonFilteredOutProjects(List map) { 109 | return map.stream() 110 | .map(entry -> new Repository( 111 | options.projectName(entry.get("name").toString()), 112 | entry.get("ssh_url").toString(), 113 | entry.get("clone_url").toString(), 114 | "master")) 115 | .filter(repo -> !options.isIgnored(repo.name)) 116 | .collect(Collectors.toList()); 117 | } 118 | 119 | private List addManuallySetProjects(String org, List repositories) { 120 | repositories.addAll(this.options.projects 121 | .stream().map(pb -> new Repository(options.projectName(pb.projectName), 122 | sshKey(org, pb), cloneUrl(org, pb), pb.branch)) 123 | .collect(Collectors.toSet())); 124 | return repositories; 125 | } 126 | 127 | private String sshKey(String org, ProjectAndBranch pb) { 128 | return "git@github.com:" + org + "/" + pb.project + ".git"; 129 | } 130 | 131 | private String cloneUrl(String org, ProjectAndBranch pb) { 132 | return "https://github.com/" + org + "/" + pb.project + ".git"; 133 | } 134 | 135 | List orgRepos(String org) throws IOException { 136 | List repos = new ArrayList<>(); 137 | Response response = null; 138 | int page = 1; 139 | while (response == null || hasNextLink(response)) { 140 | log.info("Grabbing page [" + page + "]"); 141 | response = fetchOrgsRepo(org, page); 142 | if (response.status() == 404) { 143 | log.warn("Got 404, will assume that org is actually a user"); 144 | response = fetchUsersRepo(org, page); 145 | if (response.status() >= 400) { 146 | throw new IllegalStateException("Status [" + response.status() + "] was returned for orgs and users"); 147 | } 148 | } 149 | repos.add(response.body()); 150 | page = page + 1; 151 | } 152 | return repos; 153 | } 154 | 155 | private boolean hasNextLink(Response response) { 156 | List link = response.headers().get("Link"); 157 | if (link == null || link.isEmpty()) { 158 | return false; 159 | } 160 | return link.get(0).contains("rel=\"next\""); 161 | } 162 | 163 | private Response fetchUsersRepo(String org, int page) throws IOException { 164 | return this.github.entry().method("GET").uri() 165 | .path("users/" + org + "/repos") 166 | .queryParam("page", page) 167 | .back().fetch(); 168 | } 169 | 170 | private Response fetchOrgsRepo(String org, int page) throws IOException { 171 | return this.github.entry().method("GET").uri() 172 | .path("orgs/" + org + "/repos") 173 | .queryParam("page", page).back().fetch(); 174 | } 175 | 176 | @Override public String fileContent(String org, String repo, 177 | String branch, String filePath) { 178 | try { 179 | String content = new java.util.Scanner( 180 | getFileContent(org, repo, branch, filePath)) 181 | .useDelimiter("\\A").next(); 182 | if (log.isDebugEnabled()) { 183 | log.debug("File [{}] for branch [{}] org [{}] and repo [{}] exists", 184 | filePath, branch, org, repo); 185 | } 186 | return content; 187 | } catch (IOException e) { 188 | throw new IllegalStateException(e); 189 | } catch (AssertionError | Exception e) { 190 | log.warn("Exception [{}] occurred when retrieving file [{}] for branch [{}] org [{}] and repo [{}]", 191 | e, filePath, branch, org, repo); 192 | return ""; 193 | } 194 | } 195 | 196 | InputStream getFileContent(String org, String repo, String branch, 197 | String filePath) throws IOException { 198 | return this.github.repos().get( 199 | new Coordinates.Simple(org, repo)) 200 | .contents().get(filePath, branch).raw(); 201 | } 202 | 203 | boolean descriptorExists(String org, String repo, String branch, 204 | String filePath) throws IOException { 205 | return this.github.repos().get( 206 | new Coordinates.Simple(org, repo)) 207 | .contents() 208 | .exists(filePath, branch); 209 | } 210 | } 211 | 212 | 213 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/GitlabRepositoryManagementBuilder.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | import java.net.URLEncoder; 6 | import java.util.List; 7 | import java.util.stream.Collectors; 8 | 9 | import org.apache.commons.lang3.StringUtils; 10 | import org.gitlab.api.GitlabAPI; 11 | import org.gitlab.api.models.GitlabProject; 12 | import org.gitlab.api.models.GitlabSession; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | /** 17 | * @author Marcin Grzejszczak 18 | */ 19 | class GitlabRepositoryManagementBuilder implements RepositoryManagementBuilder { 20 | 21 | private static final Logger log = LoggerFactory.getLogger(GitlabRepositoryManagementBuilder.class); 22 | 23 | @Override public RepositoryManagement build(Options options) { 24 | boolean applicable = isApplicable(options.rootUrl); 25 | if (applicable) { 26 | return createNewRepoManagement(options); 27 | } 28 | if (options.repository != Repositories.GITLAB) { 29 | return null; 30 | } 31 | return createNewRepoManagement(options); 32 | } 33 | 34 | RepositoryManagement createNewRepoManagement(Options options) { 35 | return new GitlabRepositoryManagement(options); 36 | } 37 | 38 | private boolean isApplicable(String url) { 39 | boolean applicable = StringUtils.isNotBlank(url) && url.contains("gitlab"); 40 | if (log.isDebugEnabled()) { 41 | log.debug("URL [{}] is applicable [{}]", url, applicable); 42 | } 43 | return applicable; 44 | } 45 | } 46 | 47 | class GitlabRepositoryManagement implements RepositoryManagement { 48 | 49 | private final GitlabAPI gitlabApi; 50 | private final Options options; 51 | 52 | GitlabRepositoryManagement(Options options) { 53 | this.gitlabApi = connect(options); 54 | this.gitlabApi.setRequestTimeout(5000); 55 | this.options = options; 56 | } 57 | 58 | GitlabAPI connect(Options options) { 59 | if (StringUtils.isNotBlank(options.token)) { 60 | return GitlabAPI 61 | .connect(options.rootUrl, options.token); 62 | } else if (StringUtils.isNotBlank(options.username)) { 63 | try { 64 | GitlabSession session = GitlabAPI 65 | .connect(options.rootUrl, options.username, options.password); 66 | return GitlabAPI 67 | .connect(options.rootUrl, session.getPrivateToken()); 68 | } 69 | catch (IOException e) { 70 | throw new IllegalStateException(e); 71 | } 72 | } 73 | throw new IllegalStateException("Neither token, nor username and password passed"); 74 | } 75 | 76 | GitlabRepositoryManagement(GitlabAPI gitlabApi, Options options) { 77 | this.gitlabApi = gitlabApi; 78 | this.options = options; 79 | } 80 | 81 | @Override public List repositories(String org) { 82 | try { 83 | List gitlabProjects = groupRepos(org); 84 | if (gitlabProjects.isEmpty()) { 85 | throw new IllegalStateException("No projects found for group [" + org + "]"); 86 | } 87 | List repositories = allNonFilteredOutProjects(gitlabProjects); 88 | return addManuallySetProjects(org, repositories); 89 | } 90 | catch (IOException e) { 91 | throw new IllegalStateException(e); 92 | } 93 | } 94 | 95 | private List allNonFilteredOutProjects(List map) { 96 | return map.stream() 97 | .map(entry -> new Repository( 98 | options.projectName(entry.getName()), 99 | entry.getSshUrl(), 100 | entry.getHttpUrl(), 101 | "master")) 102 | .filter(repo -> !options.isIgnored(repo.name)) 103 | .collect(Collectors.toList()); 104 | } 105 | 106 | private List addManuallySetProjects(String org, List repositories) { 107 | repositories.addAll(this.options.projects 108 | .stream().map(pb -> new Repository(options.projectName(pb.projectName), 109 | sshKey(org, pb), cloneUrl(org, pb), pb.branch)) 110 | .collect(Collectors.toSet())); 111 | return repositories; 112 | } 113 | 114 | private String sshKey(String org, ProjectAndBranch pb) { 115 | return "git@" + host() + ":" + org + "/" + pb.project + ".git"; 116 | } 117 | 118 | private String host() { 119 | return URI.create(this.options.rootUrl).getHost(); 120 | } 121 | 122 | private String cloneUrl(String org, ProjectAndBranch pb) { 123 | return "https://" + host() + "/" + org + "/" + pb.project + ".git"; 124 | } 125 | 126 | List groupRepos(String org) throws IOException { 127 | return this.gitlabApi 128 | .getGroupProjects(this.gitlabApi.getGroup(org)); 129 | } 130 | 131 | @Override public String fileContent(String org, String repo, 132 | String branch, String filePath) { 133 | try { 134 | byte[] bytes = getDescriptor(org, repo, branch, filePath); 135 | return new String(bytes, "UTF-8"); 136 | } catch (IOException e) { 137 | throw new IllegalStateException(e); 138 | } 139 | } 140 | 141 | byte[] getDescriptor(String org, String repo, String branch, 142 | String filePath) throws IOException { 143 | GitlabProject project = this.gitlabApi.getProject(org, repo); 144 | return this.gitlabApi.getRawFileContent(project, branch, URLEncoder.encode(filePath, "UTF-8")); 145 | } 146 | } 147 | 148 | 149 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/Options.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.util.Set; 4 | 5 | /** 6 | * Contains options required to connect to a password 7 | * management 8 | * 9 | * @author Marcin Grzejszczak 10 | * @since 1.0.0 11 | */ 12 | public class Options { 13 | public final String username, password, token, rootUrl; 14 | public final Repositories repository; 15 | public final Set projects; 16 | public final Set renamedProjects; 17 | public final Set excludedProjectsRegex; 18 | 19 | Options(String username, String password, String token, String rootUrl, 20 | Repositories repository, Set projects, 21 | Set renamedProjects, 22 | Set excludedProjectsRegex) { 23 | this.username = username; 24 | this.password = password; 25 | this.token = token; 26 | this.rootUrl = rootUrl; 27 | this.repository = repository; 28 | this.projects = projects; 29 | this.renamedProjects = renamedProjects; 30 | this.excludedProjectsRegex = excludedProjectsRegex; 31 | } 32 | 33 | public boolean isIgnored(String project) { 34 | return this.excludedProjectsRegex 35 | .stream() 36 | .anyMatch(project::matches); 37 | } 38 | 39 | public String projectName(String projectName) { 40 | return this.renamedProjects.stream() 41 | .filter(renamedPb -> renamedPb.project.equals(projectName)) 42 | .findFirst() 43 | .map(optionalPb -> optionalPb.projectName) 44 | .orElse(projectName); 45 | } 46 | } -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/OptionsBuilder.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * @author Marcin Grzejszczak 8 | * @since 1.0.0 9 | */ 10 | public class OptionsBuilder { 11 | private String username; 12 | private String password; 13 | private String token; 14 | private String rootUrl; 15 | private Repositories repository = Repositories.OTHER; 16 | private final Set projects = new HashSet<>(); 17 | private final Set renamedProjects = new HashSet<>(); 18 | private final Set excludedProjectsRegex = new HashSet<>(); 19 | 20 | public static OptionsBuilder builder() { 21 | return new OptionsBuilder(); 22 | } 23 | 24 | public OptionsBuilder username(String username) { 25 | this.username = username; 26 | return this; 27 | } 28 | 29 | public OptionsBuilder password(String password) { 30 | this.password = password; 31 | return this; 32 | } 33 | 34 | public OptionsBuilder token(String token) { 35 | this.token = token; 36 | return this; 37 | } 38 | 39 | public OptionsBuilder repository(Repositories repository) { 40 | this.repository = repository; 41 | return this; 42 | } 43 | 44 | public OptionsBuilder repository(String repository) { 45 | this.repository = Repositories.valueOf(repository.toUpperCase()); 46 | return this; 47 | } 48 | 49 | public OptionsBuilder project(String projectName, String branch) { 50 | this.projects.add(new ProjectAndBranch(projectName, projectName, branch)); 51 | return this; 52 | } 53 | 54 | public OptionsBuilder project(String project, String projectName, String branch) { 55 | this.projects.add(new ProjectAndBranch(project, projectName, branch)); 56 | return this; 57 | } 58 | 59 | public OptionsBuilder projectName(String projectName, String newProjectName) { 60 | this.renamedProjects.add(new ProjectAndBranch(projectName, newProjectName)); 61 | return this; 62 | } 63 | 64 | public OptionsBuilder project(String projectName) { 65 | this.projects.add(new ProjectAndBranch(projectName)); 66 | return this; 67 | } 68 | 69 | public OptionsBuilder exclude(String regex) { 70 | this.excludedProjectsRegex.add(regex); 71 | return this; 72 | } 73 | 74 | public OptionsBuilder rootUrl(String rootUrl) { 75 | this.rootUrl = rootUrl; 76 | return this; 77 | } 78 | 79 | public Options build() { 80 | return new Options(this.username, this.password, 81 | this.token, this.rootUrl, this.repository, this.projects, 82 | this.renamedProjects, 83 | this.excludedProjectsRegex); 84 | } 85 | } -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/ProjectAndBranch.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.util.Objects; 4 | 5 | /** 6 | * @author Marcin Grzejszczak 7 | * @since 1.0.0 8 | */ 9 | public class ProjectAndBranch { 10 | public final String project, projectName, branch; 11 | 12 | public ProjectAndBranch(String project, String projectName, String branch) { 13 | this.project = project; 14 | this.projectName = projectName; 15 | this.branch = branch; 16 | } 17 | 18 | public ProjectAndBranch(String project) { 19 | this(project, project, "master"); 20 | } 21 | 22 | public ProjectAndBranch(String project, String newProjectName) { 23 | this(project, newProjectName, "master"); 24 | } 25 | 26 | @Override public boolean equals(Object o) { 27 | if (this == o) 28 | return true; 29 | if (o == null || getClass() != o.getClass()) 30 | return false; 31 | ProjectAndBranch that = (ProjectAndBranch) o; 32 | return Objects.equals(project, that.project) && Objects 33 | .equals(projectName, that.projectName) && Objects 34 | .equals(branch, that.branch); 35 | } 36 | 37 | @Override public int hashCode() { 38 | return Objects.hash(project, projectName, branch); 39 | } 40 | } -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/ProjectCrawler.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | import java.util.ServiceLoader; 6 | 7 | /** 8 | * Entry class that wraps around all available implementations 9 | * of repository managers. 10 | * 11 | * You can extend the available list by using the {@link ServiceLoader} 12 | * and putting the {@link RepositoryManagementBuilder} implementation in the 13 | * {@code /META-INF/services/io.cloudpipelines.projectcrawler.RepositoryManagementBuilder} file 14 | * 15 | * @author Marcin Grzejszczak 16 | * @since 1.0.0 17 | */ 18 | public final class ProjectCrawler implements RepositoryManagement { 19 | 20 | private final Options options; 21 | private static final ServiceLoader LOADED = ServiceLoader 22 | .load(RepositoryManagementBuilder.class); 23 | private static final List DEFAULT_BUILDERS 24 | = Arrays.asList( 25 | new GithubRepositoryManagementBuilder(), 26 | new GitlabRepositoryManagementBuilder() 27 | ); 28 | 29 | public ProjectCrawler(Options options) { 30 | this.options = options; 31 | } 32 | 33 | @Override public List repositories(String org) { 34 | return firstMatching().repositories(org); 35 | } 36 | 37 | @Override public String fileContent(String org, String repo, String branch, 38 | String filePath) { 39 | return firstMatching().fileContent(org, repo, branch, filePath); 40 | } 41 | 42 | private RepositoryManagement firstMatching() { 43 | RepositoryManagement management = firstMatching(LOADED); 44 | if (management != null) { 45 | return management; 46 | } 47 | management = firstMatching(DEFAULT_BUILDERS); 48 | if (management == null) { 49 | throw new IllegalStateException("Nothing is matching the root url [" + this.options.rootUrl + "]"); 50 | } 51 | return management; 52 | } 53 | private RepositoryManagement firstMatching(Iterable builders) { 54 | for (RepositoryManagementBuilder builder : builders) { 55 | RepositoryManagement management = builder.build(this.options); 56 | if (management != null) { 57 | return management; 58 | } 59 | } 60 | return null; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/Repositories.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | /** 4 | * Supported repository 5 | * 6 | * @author Marcin Grzejszczak 7 | * @since 1.0.0 8 | */ 9 | public enum Repositories { 10 | GITHUB, GITLAB, BITBUCKET, OTHER 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/Repository.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | /** 4 | * @author Marcin Grzejszczak 5 | */ 6 | public class Repository { 7 | public String name, ssh_url, clone_url, requestedBranch; 8 | 9 | public Repository(String name, String ssh_url, String clone_url, String requestedBranch) { 10 | this.name = name; 11 | this.ssh_url = ssh_url; 12 | this.clone_url = clone_url; 13 | this.requestedBranch = requestedBranch; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/RepositoryManagement.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.util.Collections; 4 | import java.util.List; 5 | 6 | /** 7 | * Informs whether the given password management is applicable. 8 | * Also capable of fetching contents of a file. 9 | * 10 | * @author Marcin Grzejszczak 11 | * @since 1.0.0 12 | */ 13 | public interface RepositoryManagement { 14 | 15 | /** 16 | * 17 | * @param org - for the given organization 18 | * @return list of corresponding repository 19 | */ 20 | default List repositories(String org) { 21 | return Collections.emptyList(); 22 | } 23 | 24 | /** 25 | * Fetches the contents of the file 26 | * @param org - organization 27 | * @param repo - repository 28 | * @param branch - branch of the repository 29 | * @param filePath - path to the file 30 | * @return contents of the file or empty string if file not found 31 | */ 32 | default String fileContent(String org, String repo, String branch, String filePath) { 33 | return ""; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/cloudpipelines/projectcrawler/RepositoryManagementBuilder.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | /** 4 | * Builder for {@link RepositoryManagement}. Might be 5 | * required to pass the credentials, tokens etc. 6 | * 7 | * @author Marcin Grzejszczak 8 | * @since 1.0.0 9 | */ 10 | public interface RepositoryManagementBuilder { 11 | /** 12 | * @param options - repository options 13 | * @return {@code null} if {@link RepositoryManagement} can't be built for the given options 14 | */ 15 | default RepositoryManagement build(Options options) { 16 | return null; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/BitbucketRepositoryManagementBuilderTests.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.nio.file.Files; 6 | 7 | import okhttp3.MediaType; 8 | import okhttp3.Protocol; 9 | import okhttp3.Request; 10 | import okhttp3.Response; 11 | import okhttp3.ResponseBody; 12 | import org.junit.jupiter.api.Disabled; 13 | import org.junit.jupiter.api.Test; 14 | 15 | import static org.assertj.core.api.BDDAssertions.then; 16 | 17 | /** 18 | * @author Marcin Grzejszczak 19 | */ 20 | class BitbucketRepositoryManagementBuilderTests { 21 | 22 | BitbucketRepositoryManagementBuilder sut = new BitbucketRepositoryManagementBuilder(); 23 | 24 | @Test 25 | void should_return_false_when_url_is_empty() { 26 | then(sut.build(OptionsBuilder.builder().build())).isNull(); 27 | } 28 | 29 | @Test 30 | void should_return_false_when_url_does_not_contain_bitbucket() { 31 | then(sut.build(OptionsBuilder.builder() 32 | .token("foo") 33 | .rootUrl("foo").build())).isNull(); 34 | } 35 | 36 | @Test 37 | void should_return_true_when_repositories_is_bitbucket_as_enum() { 38 | then(builder().build(OptionsBuilder.builder() 39 | .token("foo") 40 | .rootUrl("foo") 41 | .repository(Repositories.BITBUCKET).build())).isNotNull(); 42 | } 43 | 44 | @Test 45 | void should_return_true_when_repositories_is_bitbucket() { 46 | then(builder().build(OptionsBuilder.builder() 47 | .username("foo").password("bar") 48 | .rootUrl("foo") 49 | .repository("bitbucket").build())).isNotNull(); 50 | } 51 | 52 | @Test 53 | void should_return_true_when_url_contains_gitlab() { 54 | then(builder().build(OptionsBuilder.builder() 55 | .token("foo") 56 | .rootUrl("https://bitbucket").build())).isNotNull(); 57 | } 58 | 59 | @Test 60 | void should_fetch_the_repos() { 61 | then(builder().build(OptionsBuilder.builder() 62 | .token("foo") 63 | .rootUrl("https://bitbucket").build()) 64 | .repositories("scpipelines")).isNotNull(); 65 | } 66 | 67 | @Test 68 | void should_fetch_the_file() { 69 | then(builder().build(OptionsBuilder.builder() 70 | .token("foo") 71 | .rootUrl("https://bitbucket").build()) 72 | .fileContent("scpipelines", 73 | "github-webhook", "master", "sc-pipelines.yml")).isNotEmpty(); 74 | } 75 | 76 | @Test 77 | @Disabled 78 | void should_call_the_real_thing_via_org() { 79 | then(new BitbucketRepositoryManagementBuilder().build( 80 | OptionsBuilder.builder() 81 | .username("foo") 82 | .password("bar") 83 | .exclude(".*") 84 | .project("github-webhook") 85 | .rootUrl("https://api.bitbucket.org") 86 | .build()) 87 | .repositories("scpipelines")).isNotNull(); 88 | } 89 | 90 | @Test 91 | @Disabled 92 | void should_call_the_real_thing_to_get_a_file() { 93 | then(new BitbucketRepositoryManagementBuilder().build( 94 | OptionsBuilder.builder() 95 | .token("foo") 96 | .rootUrl("https://api.bitbucket.org") 97 | .build()) 98 | .fileContent("scpipelines", 99 | "github-webhook", "master", "sc-pipelines.yml")).isNotEmpty(); 100 | } 101 | 102 | private BitbucketRepositoryManagementBuilder builder() { 103 | return new BitbucketRepositoryManagementBuilder() { 104 | @Override RepositoryManagement createNewRepoManagement(Options options) { 105 | return new BitbucketRepositoryManagement(options) { 106 | @Override Response callRepositories(String org, int page) throws IOException { 107 | File file = new File(BitbucketRepositoryManagementBuilderTests.class.getResource("/bitbucket/projects.json").getFile()); 108 | String body = new String(Files.readAllBytes(file.toPath())); 109 | return new Response.Builder() 110 | .request(new Request.Builder() 111 | .url("http://www.foo.com/") 112 | .get() 113 | .build()) 114 | .protocol(Protocol.HTTP_1_1) 115 | .code(200) 116 | .message(body) 117 | .body(ResponseBody.create(MediaType.get("application/json"), body)) 118 | .build(); 119 | } 120 | 121 | @Override String getDescriptor(String org, String repo, String branch, 122 | String filePath) { 123 | return "hello"; 124 | } 125 | }; 126 | } 127 | }; 128 | } 129 | 130 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/GithubRepositoryManagementBuilderTests.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.util.List; 4 | 5 | import org.junit.jupiter.api.Disabled; 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static org.assertj.core.api.BDDAssertions.then; 9 | 10 | /** 11 | * @author Marcin Grzejszczak 12 | */ 13 | 14 | class GithubRepositoryManagementBuilderTests { 15 | 16 | GithubRepositoryManagementBuilder sut = new GithubRepositoryManagementBuilder(); 17 | 18 | @Test 19 | void should_return_false_when_url_is_empty() { 20 | then(sut.build(OptionsBuilder.builder().build())).isNull(); 21 | } 22 | 23 | @Test 24 | void should_return_false_when_url_does_not_contain_github() { 25 | then(sut.build(OptionsBuilder.builder().rootUrl("foo").build())).isNull(); 26 | } 27 | 28 | @Test 29 | void should_return_true_when_repositories_is_github_as_enum() { 30 | then(githubBuilder().build(OptionsBuilder.builder().rootUrl("foo") 31 | .repository(Repositories.GITHUB).build())).isNotNull(); 32 | } 33 | 34 | @Test 35 | void should_return_true_when_repositories_is_github() { 36 | then(githubBuilder().build(OptionsBuilder.builder().rootUrl("foo") 37 | .repository("github").build())).isNotNull(); 38 | } 39 | 40 | @Test 41 | void should_return_true_when_url_contains_github() { 42 | then(githubBuilder().build(OptionsBuilder.builder() 43 | .rootUrl("https://github").build())).isNotNull(); 44 | } 45 | 46 | @Test 47 | @Disabled 48 | void should_call_the_real_thing_via_org() { 49 | then(new GithubRepositoryManagementBuilder().build( 50 | OptionsBuilder.builder() 51 | .exclude(".*") 52 | .project("github-webook") 53 | .rootUrl("https://github") 54 | .build()) 55 | .repositories("spring-cloud")).isNotNull(); 56 | } 57 | 58 | @Test 59 | @Disabled 60 | void should_call_the_real_thing_via_username() { 61 | then(new GithubRepositoryManagementBuilder().build( 62 | OptionsBuilder.builder() 63 | .exclude(".*") 64 | .project("github-webook") 65 | .rootUrl("https://github") 66 | .build()) 67 | .repositories("marcingrzejszczak")).isNotNull(); 68 | } 69 | 70 | @Test 71 | @Disabled 72 | void should_call_the_real_thing_to_get_a_file() { 73 | then(new GithubRepositoryManagementBuilder().build( 74 | OptionsBuilder.builder() 75 | .rootUrl("https://github") 76 | .build()) 77 | .fileContent("marcingrzejszczak", 78 | "github-webhook", "master", "sc-pipelines.yml")).isNotNull(); 79 | } 80 | 81 | private GithubRepositoryManagementBuilder githubBuilder() { 82 | return new GithubRepositoryManagementBuilder() { 83 | @Override RepositoryManagement createNewRepoManagement(Options options) { 84 | return new RepositoryManagement() { 85 | @Override public List repositories(String org) { 86 | return null; 87 | } 88 | 89 | @Override public String fileContent(String org, String repo, 90 | String branch, String filePath) { 91 | return null; 92 | } 93 | }; 94 | } 95 | }; 96 | } 97 | 98 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/GithubRepositoryManagementTests.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.net.URL; 8 | import java.nio.file.Files; 9 | import java.util.Collections; 10 | import java.util.List; 11 | 12 | import com.jcabi.github.Repo; 13 | import com.jcabi.github.Repos; 14 | import com.jcabi.github.mock.MkGithub; 15 | import com.jcabi.github.mock.MkStorage; 16 | import org.apache.commons.io.FileUtils; 17 | import org.junit.jupiter.api.AfterEach; 18 | import org.junit.jupiter.api.BeforeEach; 19 | import org.junit.jupiter.api.Test; 20 | 21 | import static java.nio.file.Files.createTempDirectory; 22 | import static org.assertj.core.api.Assertions.tuple; 23 | import static org.assertj.core.api.BDDAssertions.then; 24 | 25 | /** 26 | * @author Marcin Grzejszczak 27 | */ 28 | 29 | class GithubRepositoryManagementTests { 30 | 31 | MkGithub github; 32 | Repo repo; 33 | File folder; 34 | File repoXml; 35 | GithubRepositoryManagement sut; 36 | 37 | @BeforeEach 38 | void setup() throws IOException { 39 | this.folder = createTempDirectory("foo").toFile(); 40 | this.repoXml = new File(this.folder, "foo.xml"); 41 | this.github = new MkGithub(new MkStorage.InFile(this.repoXml), "jeff"); 42 | this.repo = createSleuthRepo(this.github); 43 | this.sut = new GithubRepositoryManagement(this.github, OptionsBuilder.builder().build()); 44 | } 45 | 46 | @AfterEach 47 | void cleanup() throws IOException { 48 | FileUtils.deleteDirectory(this.folder); 49 | } 50 | 51 | @Test 52 | void should_return_a_list_of_names_of_repos_for_an_org() { 53 | then(new GithubRepositoryManagement(this.github, 54 | OptionsBuilder.builder().exclude("^.*github\\.io$").build()) { 55 | @Override List orgRepos(String org) throws IOException { 56 | URL resource = GithubRepositoryManagementTests.class 57 | .getResource("/spring_cloud_repos.json"); 58 | return Collections.singletonList(new String( 59 | Files.readAllBytes(new File(resource.getFile()).toPath()))); 60 | } 61 | }.repositories("jeff")).hasSize(29) 62 | .extracting("name").doesNotContain("spring-cloud.github.io"); 63 | } 64 | 65 | @Test 66 | void should_return_a_list_of_names_of_repos_for_an_org_with_manual() { 67 | then(new GithubRepositoryManagement(this.github, 68 | OptionsBuilder.builder() 69 | .project("spring-cloud-gdpr") 70 | .projectName("spring-cloud-kubernetes-connector", "foo") 71 | .exclude("^.*github\\.io$").build()) { 72 | @Override List orgRepos(String org) throws IOException { 73 | URL resource = GithubRepositoryManagementTests.class 74 | .getResource("/spring_cloud_repos.json"); 75 | return Collections.singletonList(new String( 76 | Files.readAllBytes(new File(resource.getFile()).toPath()))); 77 | } 78 | }.repositories("jeff")).hasSize(30) 79 | .extracting("name", "ssh_url", "clone_url") 80 | .contains(tuple("spring-cloud-gdpr", 81 | "git@github.com:jeff/spring-cloud-gdpr.git", 82 | "https://github.com/jeff/spring-cloud-gdpr.git")) 83 | .contains(tuple("foo", 84 | "git@github.com:spring-cloud/spring-cloud-kubernetes-connector.git", 85 | "https://github.com/spring-cloud/spring-cloud-kubernetes-connector.git")) 86 | .doesNotContain((tuple("spring-cloud-kubernetes-connector", 87 | "git@github.com:spring-cloud/spring-cloud-kubernetes-connector.git", 88 | "https://github.com/spring-cloud/spring-cloud-kubernetes-connector.git"))); 89 | } 90 | 91 | @Test 92 | void should_return_a_list_of_only_manually_added_projects() { 93 | then(new GithubRepositoryManagement(this.github, 94 | OptionsBuilder.builder() 95 | .project("spring-cloud-gdpr") 96 | .exclude("^.*$").build()) { 97 | @Override List orgRepos(String org) throws IOException { 98 | URL resource = GithubRepositoryManagementTests.class 99 | .getResource("/spring_cloud_repos.json"); 100 | return Collections.singletonList(new String( 101 | Files.readAllBytes(new File(resource.getFile()).toPath()))); 102 | } 103 | }.repositories("jeff")).hasSize(1) 104 | .extracting("name").contains("spring-cloud-gdpr"); 105 | } 106 | 107 | @Test 108 | void should_return_empty_when_file_does_not_exist() { 109 | then(sut.fileContent("jeff", 110 | "spring-cloud-sleuth", "master", "sc-pipelines")).isEmpty(); 111 | } 112 | 113 | @Test 114 | void should_return_file_contents_when_file_exists() throws IOException { 115 | File file = new File(this.folder, "sc-pipelines.yml"); 116 | file.createNewFile(); 117 | Files.write(file.toPath(), "hello: world".getBytes()); 118 | 119 | then(new GithubRepositoryManagement(this.github, OptionsBuilder.builder().build()) { 120 | @Override InputStream getFileContent(String org, String repo, String branch, 121 | String filePath) throws IOException { 122 | return new FileInputStream(file); 123 | } 124 | 125 | @Override boolean descriptorExists(String org, String repo, String branch, 126 | String filePath) throws IOException { 127 | return true; 128 | } 129 | }.fileContent("jeff", 130 | "spring-cloud-sleuth", "master", "sc-pipelines.yml")).isEqualTo("hello: world"); 131 | } 132 | 133 | private Repo createSleuthRepo(MkGithub github) throws IOException { 134 | return github.repos().create(new Repos.RepoCreate("spring-cloud-sleuth", false)); 135 | } 136 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/GitlabRepositoryManagementBuilderTests.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import java.io.IOException; 4 | import java.util.Collections; 5 | import java.util.List; 6 | 7 | import org.gitlab.api.GitlabAPI; 8 | import org.gitlab.api.models.GitlabProject; 9 | import org.junit.jupiter.api.Disabled; 10 | import org.junit.jupiter.api.Test; 11 | import org.mockito.Mockito; 12 | 13 | import static org.assertj.core.api.BDDAssertions.then; 14 | 15 | /** 16 | * @author Marcin Grzejszczak 17 | */ 18 | class GitlabRepositoryManagementBuilderTests { 19 | 20 | GitlabRepositoryManagementBuilder sut = new GitlabRepositoryManagementBuilder(); 21 | 22 | @Test 23 | void should_return_false_when_url_is_empty() { 24 | then(sut.build(OptionsBuilder.builder() 25 | .username("foo").password("bar") 26 | .build())).isNull(); 27 | } 28 | 29 | @Test 30 | void should_return_false_when_url_does_not_contain_gitlab() { 31 | then(sut.build(OptionsBuilder.builder() 32 | .username("foo").password("bar") 33 | .rootUrl("foo").build())).isNull(); 34 | } 35 | 36 | @Test 37 | void should_return_true_when_repositories_is_gitlab_as_enum() { 38 | then(builder().build(OptionsBuilder.builder().rootUrl("http://www.foo.com/") 39 | .username("foo").password("bar") 40 | .repository(Repositories.GITLAB).build())).isNotNull(); 41 | } 42 | 43 | @Test 44 | void should_return_true_when_repositories_is_gitlab() { 45 | then(builder().build(OptionsBuilder.builder().rootUrl("foo") 46 | .token("foo") 47 | .repository("gitlab").build())).isNotNull(); 48 | } 49 | 50 | @Test 51 | void should_return_true_when_url_contains_gitlab() { 52 | then(builder().build(OptionsBuilder.builder() 53 | .token("foo") 54 | .rootUrl("https://gitlab").build())).isNotNull(); 55 | } 56 | 57 | @Test 58 | @Disabled 59 | void should_call_the_real_thing_via_org() { 60 | then(new GitlabRepositoryManagementBuilder().build( 61 | OptionsBuilder.builder() 62 | .token("foo") 63 | .exclude(".*") 64 | .project("github-webook") 65 | .rootUrl("https://gitlab.com") 66 | .build()) 67 | .repositories("sc-pipelines")).isNotNull(); 68 | } 69 | 70 | @Test 71 | @Disabled 72 | void should_call_the_real_thing_to_get_a_file() { 73 | then(new GitlabRepositoryManagementBuilder().build( 74 | OptionsBuilder.builder() 75 | .token("foo") 76 | .rootUrl("https://gitlab.com") 77 | .build()) 78 | .fileContent("sc-pipelines", 79 | "github-webhook", "master", "sc-pipelines.yml")).isNotEmpty(); 80 | } 81 | 82 | private GitlabRepositoryManagementBuilder builder() { 83 | return new GitlabRepositoryManagementBuilder() { 84 | @Override RepositoryManagement createNewRepoManagement(Options options) { 85 | return new GitlabRepositoryManagement(options) { 86 | 87 | @Override GitlabAPI connect(Options options) { 88 | return Mockito.mock(GitlabAPI.class); 89 | } 90 | 91 | @Override List groupRepos(String org) 92 | throws IOException { 93 | return Collections.singletonList(new GitlabProject()); 94 | } 95 | 96 | @Override byte[] getDescriptor(String org, String repo, String branch, 97 | String filePath) throws IOException { 98 | return "".getBytes(); 99 | } 100 | }; 101 | } 102 | }; 103 | } 104 | 105 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/OptionsBuilderTest.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import org.assertj.core.api.BDDAssertions; 4 | import org.junit.jupiter.api.Test; 5 | 6 | /** 7 | * @author Marcin Grzejszczak 8 | */ 9 | class OptionsBuilderTest { 10 | 11 | @Test void should_contain_project_names() { 12 | Options options = OptionsBuilder.builder() 13 | .project("foo", "foo", "foo") 14 | .project("bar", "bar", "bar") 15 | .build(); 16 | 17 | BDDAssertions.then(options.projects) 18 | .extracting("projectName") 19 | .contains("foo", "bar"); 20 | } 21 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/OptionsTest.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import org.assertj.core.api.BDDAssertions; 4 | import org.junit.jupiter.api.Test; 5 | 6 | /** 7 | * @author Marcin Grzejszczak 8 | */ 9 | class OptionsTest { 10 | 11 | @Test 12 | void should_return_true_when_project_ignored() { 13 | Options build = OptionsBuilder.builder() 14 | .exclude("^.*github\\.io$").build(); 15 | 16 | BDDAssertions.then(build.isIgnored("foo.github.io")).isTrue(); 17 | BDDAssertions.then(build.isIgnored("foo")).isFalse(); 18 | } 19 | 20 | @Test void should_override_the_project_name() { 21 | Options build = OptionsBuilder.builder() 22 | .projectName("foo", "bar") 23 | .build(); 24 | 25 | BDDAssertions.then(build.projectName("foo")) 26 | .isEqualTo("bar"); 27 | } 28 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/ProjectCrawlerTests.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import org.assertj.core.api.BDDAssertions; 4 | import org.junit.jupiter.api.AfterEach; 5 | import org.junit.jupiter.api.BeforeEach; 6 | import org.junit.jupiter.api.Test; 7 | 8 | /** 9 | * @author Marcin Grzejszczak 10 | */ 11 | class ProjectCrawlerTests { 12 | 13 | @BeforeEach 14 | void before() { 15 | TestRepositoryManagementBuilder.EXECUTED = false; 16 | } 17 | 18 | @AfterEach 19 | void after() { 20 | TestRepositoryManagementBuilder.EXECUTED = false; 21 | } 22 | 23 | @Test 24 | void should_call_the_test_repo_manager_for_repositories() { 25 | new ProjectCrawler(OptionsBuilder.builder() 26 | .repository(Repositories.OTHER) 27 | .build()) 28 | .repositories("foo"); 29 | 30 | BDDAssertions.then(TestRepositoryManagementBuilder.EXECUTED).isTrue(); 31 | } 32 | 33 | @Test 34 | void should_call_the_test_repo_manager_for_path() { 35 | new ProjectCrawler(OptionsBuilder.builder() 36 | .repository(Repositories.OTHER) 37 | .build()) 38 | .fileContent("org", "repo", "branch", "path"); 39 | 40 | BDDAssertions.then(TestRepositoryManagementBuilder.EXECUTED).isTrue(); 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/RepositoryManagementBuilderTests.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | import org.assertj.core.api.BDDAssertions; 4 | import org.junit.jupiter.api.Test; 5 | 6 | /** 7 | * @author Marcin Grzejszczak 8 | */ 9 | class RepositoryManagementBuilderTests { 10 | 11 | @Test 12 | void should_do_nothing_by_default() { 13 | BDDAssertions.then(new RepositoryManagementBuilder() { 14 | 15 | }.build(OptionsBuilder.builder().build())).isNull(); 16 | } 17 | } -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/TestRepositoryManagementBuilder.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler; 2 | 3 | /** 4 | * @author Marcin Grzejszczak 5 | */ 6 | public class TestRepositoryManagementBuilder implements RepositoryManagementBuilder { 7 | 8 | static boolean EXECUTED = false; 9 | 10 | @Override public RepositoryManagement build(Options options) { 11 | EXECUTED = true; 12 | return new RepositoryManagement() { 13 | }; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/test/java/io/cloudpipelines/projectcrawler/util/SocketUtils.java: -------------------------------------------------------------------------------- 1 | package io.cloudpipelines.projectcrawler.util; 2 | 3 | import java.net.DatagramSocket; 4 | import java.net.InetAddress; 5 | import java.net.ServerSocket; 6 | import java.util.Random; 7 | import java.util.SortedSet; 8 | import java.util.TreeSet; 9 | import javax.net.ServerSocketFactory; 10 | 11 | // TAKEN FROM SPRING 12 | 13 | /** 14 | * Simple utility methods for working with network sockets — for example, 15 | * for finding available ports on {@code localhost}. 16 | * 17 | *

Within this class, a TCP port refers to a port for a {@link ServerSocket}; 18 | * whereas, a UDP port refers to a port for a {@link DatagramSocket}. 19 | * 20 | * @author Sam Brannen 21 | * @author Ben Hale 22 | * @author Arjen Poutsma 23 | * @author Gunnar Hillert 24 | * @author Gary Russell 25 | * @since 4.0 26 | */ 27 | public class SocketUtils { 28 | 29 | /** 30 | * The default minimum value for port ranges used when finding an available 31 | * socket port. 32 | */ 33 | public static final int PORT_RANGE_MIN = 1024; 34 | 35 | /** 36 | * The default maximum value for port ranges used when finding an available 37 | * socket port. 38 | */ 39 | public static final int PORT_RANGE_MAX = 65535; 40 | 41 | 42 | private static final Random random = new Random(System.currentTimeMillis()); 43 | 44 | 45 | /** 46 | * Although {@code SocketUtils} consists solely of static utility methods, 47 | * this constructor is intentionally {@code public}. 48 | *

Rationale

49 | *

Static methods from this class may be invoked from within XML 50 | * configuration files using the Spring Expression Language (SpEL) and the 51 | * following syntax. 52 | *

<bean id="bean1" ... p:port="#{T(org.springframework.util.SocketUtils).findAvailableTcpPort(12000)}" />
53 | * If this constructor were {@code private}, you would be required to supply 54 | * the fully qualified class name to SpEL's {@code T()} function for each usage. 55 | * Thus, the fact that this constructor is {@code public} allows you to reduce 56 | * boilerplate configuration with SpEL as can be seen in the following example. 57 | *
<bean id="socketUtils" class="org.springframework.util.SocketUtils" />
 58 | 	 * <bean id="bean1" ... p:port="#{socketUtils.findAvailableTcpPort(12000)}" />
 59 | 	 * <bean id="bean2" ... p:port="#{socketUtils.findAvailableTcpPort(30000)}" />
60 | */ 61 | public SocketUtils() { 62 | /* no-op */ 63 | } 64 | 65 | 66 | /** 67 | * Find an available TCP port randomly selected from the range 68 | * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. 69 | * @return an available TCP port number 70 | * @throws IllegalStateException if no available port could be found 71 | */ 72 | public static int findAvailableTcpPort() { 73 | return findAvailableTcpPort(PORT_RANGE_MIN); 74 | } 75 | 76 | /** 77 | * Find an available TCP port randomly selected from the range 78 | * [{@code minPort}, {@value #PORT_RANGE_MAX}]. 79 | * @param minPort the minimum port number 80 | * @return an available TCP port number 81 | * @throws IllegalStateException if no available port could be found 82 | */ 83 | public static int findAvailableTcpPort(int minPort) { 84 | return findAvailableTcpPort(minPort, PORT_RANGE_MAX); 85 | } 86 | 87 | /** 88 | * Find an available TCP port randomly selected from the range 89 | * [{@code minPort}, {@code maxPort}]. 90 | * @param minPort the minimum port number 91 | * @param maxPort the maximum port number 92 | * @return an available TCP port number 93 | * @throws IllegalStateException if no available port could be found 94 | */ 95 | public static int findAvailableTcpPort(int minPort, int maxPort) { 96 | return SocketType.TCP.findAvailablePort(minPort, maxPort); 97 | } 98 | 99 | /** 100 | * Find the requested number of available TCP ports, each randomly selected 101 | * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. 102 | * @param numRequested the number of available ports to find 103 | * @return a sorted set of available TCP port numbers 104 | * @throws IllegalStateException if the requested number of available ports could not be found 105 | */ 106 | public static SortedSet findAvailableTcpPorts(int numRequested) { 107 | return findAvailableTcpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); 108 | } 109 | 110 | /** 111 | * Find the requested number of available TCP ports, each randomly selected 112 | * from the range [{@code minPort}, {@code maxPort}]. 113 | * @param numRequested the number of available ports to find 114 | * @param minPort the minimum port number 115 | * @param maxPort the maximum port number 116 | * @return a sorted set of available TCP port numbers 117 | * @throws IllegalStateException if the requested number of available ports could not be found 118 | */ 119 | public static SortedSet findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { 120 | return SocketType.TCP.findAvailablePorts(numRequested, minPort, maxPort); 121 | } 122 | 123 | /** 124 | * Find an available UDP port randomly selected from the range 125 | * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. 126 | * @return an available UDP port number 127 | * @throws IllegalStateException if no available port could be found 128 | */ 129 | public static int findAvailableUdpPort() { 130 | return findAvailableUdpPort(PORT_RANGE_MIN); 131 | } 132 | 133 | /** 134 | * Find an available UDP port randomly selected from the range 135 | * [{@code minPort}, {@value #PORT_RANGE_MAX}]. 136 | * @param minPort the minimum port number 137 | * @return an available UDP port number 138 | * @throws IllegalStateException if no available port could be found 139 | */ 140 | public static int findAvailableUdpPort(int minPort) { 141 | return findAvailableUdpPort(minPort, PORT_RANGE_MAX); 142 | } 143 | 144 | /** 145 | * Find an available UDP port randomly selected from the range 146 | * [{@code minPort}, {@code maxPort}]. 147 | * @param minPort the minimum port number 148 | * @param maxPort the maximum port number 149 | * @return an available UDP port number 150 | * @throws IllegalStateException if no available port could be found 151 | */ 152 | public static int findAvailableUdpPort(int minPort, int maxPort) { 153 | return SocketType.UDP.findAvailablePort(minPort, maxPort); 154 | } 155 | 156 | /** 157 | * Find the requested number of available UDP ports, each randomly selected 158 | * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. 159 | * @param numRequested the number of available ports to find 160 | * @return a sorted set of available UDP port numbers 161 | * @throws IllegalStateException if the requested number of available ports could not be found 162 | */ 163 | public static SortedSet findAvailableUdpPorts(int numRequested) { 164 | return findAvailableUdpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); 165 | } 166 | 167 | /** 168 | * Find the requested number of available UDP ports, each randomly selected 169 | * from the range [{@code minPort}, {@code maxPort}]. 170 | * @param numRequested the number of available ports to find 171 | * @param minPort the minimum port number 172 | * @param maxPort the maximum port number 173 | * @return a sorted set of available UDP port numbers 174 | * @throws IllegalStateException if the requested number of available ports could not be found 175 | */ 176 | public static SortedSet findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { 177 | return SocketType.UDP.findAvailablePorts(numRequested, minPort, maxPort); 178 | } 179 | 180 | 181 | private enum SocketType { 182 | 183 | TCP { 184 | @Override 185 | protected boolean isPortAvailable(int port) { 186 | try { 187 | ServerSocket serverSocket = ServerSocketFactory.getDefault().createServerSocket( 188 | port, 1, InetAddress.getByName("localhost")); 189 | serverSocket.close(); 190 | return true; 191 | } 192 | catch (Exception ex) { 193 | return false; 194 | } 195 | } 196 | }, 197 | 198 | UDP { 199 | @Override 200 | protected boolean isPortAvailable(int port) { 201 | try { 202 | DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost")); 203 | socket.close(); 204 | return true; 205 | } 206 | catch (Exception ex) { 207 | return false; 208 | } 209 | } 210 | }; 211 | 212 | /** 213 | * Determine if the specified port for this {@code SocketType} is 214 | * currently available on {@code localhost}. 215 | */ 216 | protected abstract boolean isPortAvailable(int port); 217 | 218 | /** 219 | * Find a pseudo-random port number within the range 220 | * [{@code minPort}, {@code maxPort}]. 221 | * @param minPort the minimum port number 222 | * @param maxPort the maximum port number 223 | * @return a random port number within the specified range 224 | */ 225 | private int findRandomPort(int minPort, int maxPort) { 226 | int portRange = maxPort - minPort; 227 | return minPort + random.nextInt(portRange + 1); 228 | } 229 | 230 | /** 231 | * Find an available port for this {@code SocketType}, randomly selected 232 | * from the range [{@code minPort}, {@code maxPort}]. 233 | * @param minPort the minimum port number 234 | * @param maxPort the maximum port number 235 | * @return an available port number for this socket type 236 | * @throws IllegalStateException if no available port could be found 237 | */ 238 | int findAvailablePort(int minPort, int maxPort) { 239 | int portRange = maxPort - minPort; 240 | int candidatePort; 241 | int searchCounter = 0; 242 | do { 243 | if (++searchCounter > portRange) { 244 | throw new IllegalStateException(String.format( 245 | "Could not find an available %s port in the range [%d, %d] after %d attempts", 246 | name(), minPort, maxPort, searchCounter)); 247 | } 248 | candidatePort = findRandomPort(minPort, maxPort); 249 | } 250 | while (!isPortAvailable(candidatePort)); 251 | 252 | return candidatePort; 253 | } 254 | 255 | /** 256 | * Find the requested number of available ports for this {@code SocketType}, 257 | * each randomly selected from the range [{@code minPort}, {@code maxPort}]. 258 | * @param numRequested the number of available ports to find 259 | * @param minPort the minimum port number 260 | * @param maxPort the maximum port number 261 | * @return a sorted set of available port numbers for this socket type 262 | * @throws IllegalStateException if the requested number of available ports could not be found 263 | */ 264 | SortedSet findAvailablePorts(int numRequested, int minPort, int maxPort) { 265 | SortedSet availablePorts = new TreeSet(); 266 | int attemptCount = 0; 267 | while ((++attemptCount <= numRequested + 100) && availablePorts.size() < numRequested) { 268 | availablePorts.add(findAvailablePort(minPort, maxPort)); 269 | } 270 | 271 | if (availablePorts.size() != numRequested) { 272 | throw new IllegalStateException(String.format( 273 | "Could not find %d available %s ports in the range [%d, %d]", 274 | numRequested, name(), minPort, maxPort)); 275 | } 276 | 277 | return availablePorts; 278 | } 279 | } 280 | 281 | } 282 | -------------------------------------------------------------------------------- /src/test/resources/META-INF/services/io.cloudpipelines.projectcrawler.RepositoryManagementBuilder: -------------------------------------------------------------------------------- 1 | io.cloudpipelines.projectcrawler.TestRepositoryManagementBuilder -------------------------------------------------------------------------------- /src/test/resources/bitbucket/projects.json: -------------------------------------------------------------------------------- 1 | { 2 | "pagelen": 10, 3 | "values": [ 4 | { 5 | "scm": "git", 6 | "website": "", 7 | "has_wiki": false, 8 | "uuid": "{2969f380-836c-46e7-beaf-0c74d7ca9a24}", 9 | "links": { 10 | "watchers": { 11 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/watchers" 12 | }, 13 | "branches": { 14 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/refs/branches" 15 | }, 16 | "tags": { 17 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/refs/tags" 18 | }, 19 | "commits": { 20 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/commits" 21 | }, 22 | "clone": [ 23 | { 24 | "href": "https://bitbucket.org/scpipelines/github-webhook.git", 25 | "name": "https" 26 | }, 27 | { 28 | "href": "git@bitbucket.org:scpipelines/github-webhook.git", 29 | "name": "ssh" 30 | } 31 | ], 32 | "self": { 33 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook" 34 | }, 35 | "source": { 36 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/src" 37 | }, 38 | "html": { 39 | "href": "https://bitbucket.org/scpipelines/github-webhook" 40 | }, 41 | "avatar": { 42 | "href": "https://bytebucket.org/ravatar/%7B2969f380-836c-46e7-beaf-0c74d7ca9a24%7D?ts=default" 43 | }, 44 | "hooks": { 45 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/hooks" 46 | }, 47 | "forks": { 48 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/forks" 49 | }, 50 | "downloads": { 51 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/downloads" 52 | }, 53 | "pullrequests": { 54 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-webhook/pullrequests" 55 | } 56 | }, 57 | "fork_policy": "allow_forks", 58 | "name": "github-webhook", 59 | "project": { 60 | "key": "GIT", 61 | "type": "project", 62 | "uuid": "{1c1015cc-bf6b-4857-b0ae-f548c1c2e625}", 63 | "links": { 64 | "self": { 65 | "href": "https://api.bitbucket.org/2.0/teams/scpipelines/projects/GIT" 66 | }, 67 | "html": { 68 | "href": "https://bitbucket.org/account/user/scpipelines/projects/GIT" 69 | }, 70 | "avatar": { 71 | "href": "https://bitbucket.org/account/user/scpipelines/projects/GIT/avatar/32" 72 | } 73 | }, 74 | "name": "github-webhook" 75 | }, 76 | "language": "", 77 | "created_on": "2018-07-25T07:00:28.296081+00:00", 78 | "mainbranch": { 79 | "type": "branch", 80 | "name": "master" 81 | }, 82 | "full_name": "scpipelines/github-webhook", 83 | "has_issues": false, 84 | "owner": { 85 | "username": "scpipelines", 86 | "display_name": "sc-pipelines", 87 | "type": "team", 88 | "uuid": "{7d9675e0-6e92-41b8-aec4-dff4487d8bf5}", 89 | "links": { 90 | "self": { 91 | "href": "https://api.bitbucket.org/2.0/teams/scpipelines" 92 | }, 93 | "html": { 94 | "href": "https://bitbucket.org/scpipelines/" 95 | }, 96 | "avatar": { 97 | "href": "https://bitbucket.org/account/scpipelines/avatar/" 98 | } 99 | } 100 | }, 101 | "updated_on": "2018-07-25T07:00:28.695833+00:00", 102 | "size": 279046, 103 | "type": "repository", 104 | "slug": "github-webhook", 105 | "is_private": false, 106 | "description": "" 107 | }, 108 | { 109 | "scm": "git", 110 | "website": "", 111 | "has_wiki": false, 112 | "uuid": "{33176226-263c-43a6-8709-7b5e8411df11}", 113 | "links": { 114 | "watchers": { 115 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/watchers" 116 | }, 117 | "branches": { 118 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/refs/branches" 119 | }, 120 | "tags": { 121 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/refs/tags" 122 | }, 123 | "commits": { 124 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/commits" 125 | }, 126 | "clone": [ 127 | { 128 | "href": "https://bitbucket.org/scpipelines/github-analytics.git", 129 | "name": "https" 130 | }, 131 | { 132 | "href": "git@bitbucket.org:scpipelines/github-analytics.git", 133 | "name": "ssh" 134 | } 135 | ], 136 | "self": { 137 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics" 138 | }, 139 | "source": { 140 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/src" 141 | }, 142 | "html": { 143 | "href": "https://bitbucket.org/scpipelines/github-analytics" 144 | }, 145 | "avatar": { 146 | "href": "https://bytebucket.org/ravatar/%7B33176226-263c-43a6-8709-7b5e8411df11%7D?ts=default" 147 | }, 148 | "hooks": { 149 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/hooks" 150 | }, 151 | "forks": { 152 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/forks" 153 | }, 154 | "downloads": { 155 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/downloads" 156 | }, 157 | "pullrequests": { 158 | "href": "https://api.bitbucket.org/2.0/repositories/scpipelines/github-analytics/pullrequests" 159 | } 160 | }, 161 | "fork_policy": "allow_forks", 162 | "name": "github-analytics", 163 | "project": { 164 | "key": "GIT-2", 165 | "type": "project", 166 | "uuid": "{e62c6dff-ca9d-4067-ae65-77e8498f5f00}", 167 | "links": { 168 | "self": { 169 | "href": "https://api.bitbucket.org/2.0/teams/scpipelines/projects/GIT-2" 170 | }, 171 | "html": { 172 | "href": "https://bitbucket.org/account/user/scpipelines/projects/GIT-2" 173 | }, 174 | "avatar": { 175 | "href": "https://bitbucket.org/account/user/scpipelines/projects/GIT-2/avatar/32" 176 | } 177 | }, 178 | "name": "github-analytics" 179 | }, 180 | "language": "", 181 | "created_on": "2018-07-25T08:39:20.103485+00:00", 182 | "mainbranch": { 183 | "type": "branch", 184 | "name": "master" 185 | }, 186 | "full_name": "scpipelines/github-analytics", 187 | "has_issues": false, 188 | "owner": { 189 | "username": "scpipelines", 190 | "display_name": "sc-pipelines", 191 | "type": "team", 192 | "uuid": "{7d9675e0-6e92-41b8-aec4-dff4487d8bf5}", 193 | "links": { 194 | "self": { 195 | "href": "https://api.bitbucket.org/2.0/teams/scpipelines" 196 | }, 197 | "html": { 198 | "href": "https://bitbucket.org/scpipelines/" 199 | }, 200 | "avatar": { 201 | "href": "https://bitbucket.org/account/scpipelines/avatar/" 202 | } 203 | } 204 | }, 205 | "updated_on": "2018-07-25T08:39:20.515725+00:00", 206 | "size": 354313, 207 | "type": "repository", 208 | "slug": "github-analytics", 209 | "is_private": false, 210 | "description": "" 211 | } 212 | ], 213 | "page": 1, 214 | "size": 2 215 | } -------------------------------------------------------------------------------- /src/test/resources/gitlab/curls: -------------------------------------------------------------------------------- 1 | $ echo "Getting projects for group" 2 | $ curl --header "Private-Token: foo-bar" https://gitlab.com/api/v4/groups/sc-pipelines/projects 3 | $ echo "Getting a file" 4 | $ curl --header "Private-Token: foo-bar" 'https://gitlab.com/api/v4/projects/7620377/repository/files/sc-pipelines%2Eyml?ref=master' | jq -------------------------------------------------------------------------------- /src/test/resources/gitlab/v4_file_from_path.json: -------------------------------------------------------------------------------- 1 | { 2 | "file_name": "sc-pipelines.yml", 3 | "file_path": "sc-pipelines.yml", 4 | "size": 1293, 5 | "encoding": "base64", 6 | "content_sha256": "fa6bf9d208dc13439b4122ec43df507990359f2929c1bb7102dad1964b49ad8c", 7 | "ref": "master", 8 | "blob_id": "8bae677dbc61a5bf8c1dc0cb6164b4468b376487", 9 | "commit_id": "adba557aa5dc80a47708d6a9887a7a0d1a70c990", 10 | "last_commit_id": "0489ec15cae3b8acdd313c4fc28256532c2ac8c9", 11 | "content": "IyBUaGlzIGZpbGUgZGVzY3JpYmVzIHdoaWNoIHNlcnZpY2VzIGFyZSByZXF1aXJlZCBieSB0aGlzIGFwcGxpY2F0aW9uCiMgaW4gb3JkZXIgZm9yIHRoZSBzbW9rZSB0ZXN0cyBvbiB0aGUgVEVTVCBlbnZpcm9ubWVudCBhbmQgZW5kIHRvIGVuZCB0ZXN0cwojIG9uIHRoZSBTVEFHRSBlbnZpcm9ubWVudCB0byBwYXNzCgojIGxvd2VyY2FzZSBuYW1lIG9mIHRoZSBlbnZpcm9ubWVudAp0ZXN0OgogICMgbGlzdCBvZiByZXF1aXJlZCBzZXJ2aWNlcwogIHNlcnZpY2VzOgogICAgIyBQcmVwYXJlZCBmb3IgUENGIERFVgogICAgIyB0eXBlIGFuZCBuYW1lIG9mIHRoZSBzZXJ2aWNlCiAgICAtIG5hbWU6IG15c3FsLWdpdGh1Yi1hbmFseXRpY3MKICAgICAgdHlwZTogYnJva2VyCiAgICAgIGJyb2tlcjogcC1teXNxbAogICAgICBwbGFuOiA1MTJtYgogICAgLSBuYW1lOiBnaXRodWItcmFiYml0bXEKICAgICAgdHlwZTogYnJva2VyCiAgICAgIGJyb2tlcjogY2xvdWRhbXFwCiAgICAgIHBsYW46IGxlbXVyCiAgICAtIG5hbWU6IGdpdGh1Yi1ldXJla2EKICAgICAgdHlwZTogYXBwCiAgICAgIGNvb3JkaW5hdGVzOiBjb20uZXhhbXBsZS5ldXJla2E6Z2l0aHViLWV1cmVrYTowLjAuMS5NMQogICAgICBwYXRoVG9NYW5pZmVzdDogc2MtcGlwZWxpbmVzL21hbmlmZXN0LWV1cmVrYS55bWwKICAgIC0gdHlwZTogc3R1YnJ1bm5lcgogICAgICBuYW1lOiBzdHVicnVubmVyCiAgICAgIGNvb3JkaW5hdGVzOiBjb20uZXhhbXBsZS5naXRodWI6Z2l0aHViLWFuYWx5dGljcy1zdHViLXJ1bm5lci1ib290LWNsYXNzcGF0aC1zdHViczowLjAuMS5NMQogICAgICBwYXRoVG9NYW5pZmVzdDogc2MtcGlwZWxpbmVzL21hbmlmZXN0LXN0dWJydW5uZXIueW1sCnN0YWdlOgogIHNlcnZpY2VzOgogICAgIyBQcmVwYXJlZCBmb3IgUENGIERFVgogICAgIyB0eXBlIGFuZCBuYW1lIG9mIHRoZSBzZXJ2aWNlCiAgICAtIG5hbWU6IG15c3FsLWdpdGh1Yi1hbmFseXRpY3MKICAgICAgdHlwZTogYnJva2VyCiAgICAgIGJyb2tlcjogcC1teXNxbAogICAgICBwbGFuOiA1MTJtYgogICAgLSBuYW1lOiBnaXRodWItcmFiYml0bXEKICAgICAgdHlwZTogYnJva2VyCiAgICAgIGJyb2tlcjogY2xvdWRhbXFwCiAgICAgIHBsYW46IGxlbXVyCiAgICAtIG5hbWU6IGdpdGh1Yi1ldXJla2EKICAgICAgdHlwZTogYXBwCiAgICAgIGNvb3JkaW5hdGVzOiBjb20uZXhhbXBsZS5ldXJla2E6Z2l0aHViLWV1cmVrYTowLjAuMS5NMQogICAgICBwYXRoVG9NYW5pZmVzdDogc2MtcGlwZWxpbmVzL21hbmlmZXN0LWV1cmVrYS55bWwK" 12 | } -------------------------------------------------------------------------------- /src/test/resources/gitlab/v4_projects_for_group.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 7620377, 4 | "description": "", 5 | "name": "github-analytics", 6 | "name_with_namespace": "sc-pipelines / github-analytics", 7 | "path": "github-analytics", 8 | "path_with_namespace": "sc-pipelines/github-analytics", 9 | "created_at": "2018-07-24T22:37:47.677Z", 10 | "default_branch": "master", 11 | "tag_list": [], 12 | "ssh_url_to_repo": "git@gitlab.com:sc-pipelines/github-analytics.git", 13 | "http_url_to_repo": "https://gitlab.com/sc-pipelines/github-analytics.git", 14 | "web_url": "https://gitlab.com/sc-pipelines/github-analytics", 15 | "readme_url": "https://gitlab.com/sc-pipelines/github-analytics/blob/master/README.adoc", 16 | "avatar_url": null, 17 | "star_count": 0, 18 | "forks_count": 0, 19 | "last_activity_at": "2018-07-24T22:37:47.677Z", 20 | "_links": { 21 | "self": "https://gitlab.com/api/v4/projects/7620377", 22 | "issues": "https://gitlab.com/api/v4/projects/7620377/issues", 23 | "merge_requests": "https://gitlab.com/api/v4/projects/7620377/merge_requests", 24 | "repo_branches": "https://gitlab.com/api/v4/projects/7620377/repository/branches", 25 | "labels": "https://gitlab.com/api/v4/projects/7620377/labels", 26 | "events": "https://gitlab.com/api/v4/projects/7620377/events", 27 | "members": "https://gitlab.com/api/v4/projects/7620377/members" 28 | }, 29 | "archived": false, 30 | "visibility": "internal", 31 | "resolve_outdated_diff_discussions": false, 32 | "container_registry_enabled": true, 33 | "issues_enabled": true, 34 | "merge_requests_enabled": true, 35 | "wiki_enabled": true, 36 | "jobs_enabled": true, 37 | "snippets_enabled": true, 38 | "shared_runners_enabled": true, 39 | "lfs_enabled": true, 40 | "creator_id": 829256, 41 | "namespace": { 42 | "id": 3282590, 43 | "name": "sc-pipelines", 44 | "path": "sc-pipelines", 45 | "kind": "group", 46 | "full_path": "sc-pipelines", 47 | "parent_id": null 48 | }, 49 | "import_status": "none", 50 | "open_issues_count": 0, 51 | "public_jobs": true, 52 | "ci_config_path": null, 53 | "shared_with_groups": [], 54 | "only_allow_merge_if_pipeline_succeeds": false, 55 | "request_access_enabled": false, 56 | "only_allow_merge_if_all_discussions_are_resolved": false, 57 | "printing_merge_request_link_enabled": true, 58 | "merge_method": "merge" 59 | }, 60 | { 61 | "id": 7620371, 62 | "description": "", 63 | "name": "github-webhook", 64 | "name_with_namespace": "sc-pipelines / github-webhook", 65 | "path": "github-webhook", 66 | "path_with_namespace": "sc-pipelines/github-webhook", 67 | "created_at": "2018-07-24T22:36:45.175Z", 68 | "default_branch": "master", 69 | "tag_list": [], 70 | "ssh_url_to_repo": "git@gitlab.com:sc-pipelines/github-webhook.git", 71 | "http_url_to_repo": "https://gitlab.com/sc-pipelines/github-webhook.git", 72 | "web_url": "https://gitlab.com/sc-pipelines/github-webhook", 73 | "readme_url": "https://gitlab.com/sc-pipelines/github-webhook/blob/master/README.adoc", 74 | "avatar_url": null, 75 | "star_count": 0, 76 | "forks_count": 0, 77 | "last_activity_at": "2018-07-24T22:36:45.175Z", 78 | "_links": { 79 | "self": "https://gitlab.com/api/v4/projects/7620371", 80 | "issues": "https://gitlab.com/api/v4/projects/7620371/issues", 81 | "merge_requests": "https://gitlab.com/api/v4/projects/7620371/merge_requests", 82 | "repo_branches": "https://gitlab.com/api/v4/projects/7620371/repository/branches", 83 | "labels": "https://gitlab.com/api/v4/projects/7620371/labels", 84 | "events": "https://gitlab.com/api/v4/projects/7620371/events", 85 | "members": "https://gitlab.com/api/v4/projects/7620371/members" 86 | }, 87 | "archived": false, 88 | "visibility": "internal", 89 | "resolve_outdated_diff_discussions": false, 90 | "container_registry_enabled": true, 91 | "issues_enabled": true, 92 | "merge_requests_enabled": true, 93 | "wiki_enabled": true, 94 | "jobs_enabled": true, 95 | "snippets_enabled": true, 96 | "shared_runners_enabled": true, 97 | "lfs_enabled": true, 98 | "creator_id": 829256, 99 | "namespace": { 100 | "id": 3282590, 101 | "name": "sc-pipelines", 102 | "path": "sc-pipelines", 103 | "kind": "group", 104 | "full_path": "sc-pipelines", 105 | "parent_id": null 106 | }, 107 | "import_status": "none", 108 | "open_issues_count": 0, 109 | "public_jobs": true, 110 | "ci_config_path": null, 111 | "shared_with_groups": [], 112 | "only_allow_merge_if_pipeline_succeeds": false, 113 | "request_access_enabled": false, 114 | "only_allow_merge_if_all_discussions_are_resolved": false, 115 | "printing_merge_request_link_enabled": true, 116 | "merge_method": "merge" 117 | } 118 | ] -------------------------------------------------------------------------------- /src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | --------------------------------------------------------------------------------