├── .gitignore ├── LICENSE ├── README.md ├── connect-jenkins-source.properties ├── connect-standalone.properties ├── gmavenplus-plugin-1.4.pom ├── pom.xml └── src ├── assembly ├── development.xml ├── package.xml └── standalone.xml ├── main └── java │ └── org │ └── aravind │ └── oss │ ├── jenkins │ ├── JenkinsClient.java │ ├── JenkinsException.java │ └── domain │ │ ├── Build.java │ │ ├── BuildCollection.java │ │ ├── Jenkins.java │ │ ├── JenkinsItem.java │ │ └── Job.java │ └── kafka │ └── connect │ ├── jenkins │ ├── JenkinsSourceConfig.java │ ├── JenkinsSourceConnector.java │ ├── JenkinsSourceTask.java │ ├── JobTaskConfigExtractor.java │ ├── ReadYourWritesOffsetStorageAdapter.java │ ├── Util.java │ └── Version.java │ └── lib │ ├── Partitions.java │ ├── SourceOffset.java │ ├── SourcePartition.java │ ├── TaskConfigBuilder.java │ └── TaskConfigExtractor.java ├── site └── resources │ └── images │ └── jenkins-resource-relationships.png └── test ├── groovy └── org │ └── aravind │ └── oss │ ├── jenkins │ ├── JenkinsClientTest.groovy │ └── domain │ │ └── BuildTest.groovy │ └── kafka │ └── connect │ └── jenkins │ ├── JenkinsSourceConfigTest.groovy │ ├── JenkinsSourceConnectorTest.groovy │ ├── JenkinsSourceTaskTest.groovy │ └── ReadYourWritesOffsetStorageAdapterTest.groovy └── resources ├── BuildTest-mock-server-cfg.json ├── JenkinsSourceTaskTest-mock-server-cfg.json ├── abdera-trunk-2546-build-details.json ├── abdera-trunk-builds.json ├── accumulo-trunk-18-build-details.json ├── accumulo-trunk-builds.json ├── all-jobs.json ├── cert.jks ├── jenkins-mock-server-cfg.json ├── jenkins-mock-server-single-job-cfg.json ├── jenkins-mock-server-three-job-cfg.json ├── jenkins-mock-server-with-authuentication-cfg.json ├── keystore.jks ├── log4j.properties ├── moco-cert.cer ├── new-job-with-no-builds.json ├── single-job.json └── three-jobs.json /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Maven 4 | target 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 12 | hs_err_pid* 13 | 14 | # IntelliJ IDEA 15 | .idea 16 | *.iml 17 | /connect-jenkins-source-local.properties 18 | /log.txt 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka Connect Jenkins 2 | 3 | [![Build Status](https://snap-ci.com/yaravind/kafka-connect-jenkins/branch/master/build_image)](https://snap-ci.com/yaravind/kafka-connect-jenkins/branch/master) [![Coverage Status](https://coveralls.io/repos/github/yaravind/kafka-connect-jenkins/badge.svg?branch=master)](https://coveralls.io/github/yaravind/kafka-connect-jenkins?branch=master) [![Codacy Badge](https://api.codacy.com/project/badge/grade/c6faadd0154740aeb202710fcdea3dfc)](https://www.codacy.com/app/yaravind/kafka-connect-jenkins/dashboard) [![Stories in Ready](https://badge.waffle.io/yaravind/kafka-connect-jenkins.png?label=ready&title=Ready)](https://waffle.io/yaravind/kafka-connect-jenkins) [![Join the chat at https://gitter.im/yaravind/kafka-connect-jenkins](https://badges.gitter.im/yaravind/kafka-connect-jenkins.svg)](https://gitter.im/yaravind/kafka-connect-jenkins?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | 5 | Kafka Connect Connector for [Jenkins](https://jenkins.io/) Open Source Continuous Integration Tool. 6 | 7 | ## What is it? 8 | 9 | kafka-connect-jenkins is a [Kafka Connector](http://kafka.apache.org/0100/documentation.html#connect) for loading data from [Jenkins](https://jenkins.io/) Open Source [Continuous Integration](https://en.wikipedia.org/wiki/Continuous_integration) Tool. 10 | 11 | ### Details 12 | 13 | +--------+ 0..* +-------+ 0..* +-------+ 1 +--------------+ 14 | | Server +--------> | Job +-------> | Build +--------> | BuildDetails | 15 | +--------+ +-------+ +-------+ +--------------+ 16 | 17 | The following comparison might give you more insight. 18 | 19 | | Example | Input Partitions (Logical) | Records | Offsets/Position | 20 | | ------- | -------------------------- | ------- | ---------------- | 21 | | Set of files | each File | each Line | line Number within a file | 22 | | Database instance | each Table | each Row | primary id + timestamp | 23 | | Jenkins Server | each Job | BuildDetails of each Build (**lastBuild**) | build number of **lastBuild**| 24 | 25 | ### Example 26 | 27 | We will use Apache Jenkins REST API to demonstrate an example. 28 | 29 | | Resource | Details | 30 | |-----------|---------| 31 | | Get list of all Jobs:
https://builds.apache.org/api/json

GET **lastBuild** of `Abdera-trunk` Job:
https://builds.apache.org/job/Abdera-trunk/api/json

GET BuildDetails for build `2546` of job `Abdera-trunk`:
https://builds.apache.org/job/Abdera-trunk/2546/api/json | ![](https://github.com/yaravind/kafka-connect-jenkins/blob/master/src/site/resources/images/jenkins-resource-relationships.png) | 32 | 33 | **What gets published to the topic?** 34 | 35 | The BuildDetails is treated as an event and is persisted to the topic in JSON format. Below is the content of a sample event 36 | 37 | ``` 38 | { 39 | "actions": [ 40 | { 41 | "causes": [ 42 | { 43 | "shortDescription": "Started by an SCM change" 44 | } 45 | ] 46 | }, 47 | {}, 48 | {}, 49 | {}, 50 | {}, 51 | {}, 52 | {}, 53 | { 54 | "failCount": 0, 55 | "skipCount": 1, 56 | "totalCount": 491, 57 | "urlName": "testReport" 58 | }, 59 | {}, 60 | {} 61 | ], 62 | "artifacts": [], 63 | "building": false, 64 | "description": null, 65 | "displayName": "#2546", 66 | "duration": 480582, 67 | "estimatedDuration": 457794, 68 | "executor": null, 69 | "fullDisplayName": "Abdera-trunk #2546", 70 | "id": "2015-08-25_22-08-43", 71 | "keepLog": false, 72 | "number": 2546, 73 | "queueId": -1, 74 | "result": "SUCCESS", 75 | "timestamp": 1440540523000, 76 | "url": "https://builds.apache.org/job/Abdera-trunk/2546/", 77 | "builtOn": "jenkins-ubuntu-1404-4gb-c51", 78 | "changeSet": { 79 | "items": [ 80 | { 81 | "affectedPaths": [ 82 | "client/src/test/java/org/apache/abdera/test/client/util/MultipartRelatedRequestEntityTest.java" 83 | ], 84 | "author": { 85 | "absoluteUrl": "https://builds.apache.org/user/veithen", 86 | "fullName": "veithen" 87 | }, 88 | "commitId": "1697770", 89 | "timestamp": 1440537458260, 90 | "date": "2015-08-25T21:17:38.260515Z", 91 | "msg": "Use factory to create FOMEntry instance.", 92 | "paths": [ 93 | { 94 | "editType": "edit", 95 | "file": "/abdera/java/trunk/client/src/test/java/org/apache/abdera/test/client/util/MultipartRelatedRequestEntityTest.java" 96 | } 97 | ], 98 | "revision": 1697770, 99 | "user": "veithen" 100 | } 101 | ], 102 | "kind": "svn", 103 | "revisions": [ 104 | { 105 | "module": "https://svn.apache.org/repos/asf/abdera/java/trunk", 106 | "revision": 1697770 107 | } 108 | ] 109 | }, 110 | "culprits": [ 111 | { 112 | "absoluteUrl": "https://builds.apache.org/user/veithen", 113 | "fullName": "veithen" 114 | } 115 | ], 116 | "mavenArtifacts": {}, 117 | "mavenVersionUsed": "3.0.4" 118 | } 119 | ``` 120 | 121 | ## What can I do with it? 122 | 123 | In an organization, there can be one or more CI/CD servers (usually Jenkins) managing the day to day builds 124 | and deployments of various components or services. We usually see each business unit (or vertical) managing 125 | their own Jenkins instance. Any such organization can benefit from treating **build events** same like any other 126 | transactional or master data. [Here are some use-cases](https://github.com/yaravind/kafka-connect-jenkins/wiki/Use-cases) that these **build events** can enable. 127 | 128 | ## Configurations 129 | 130 | | Property | Description | Required? | Default value | 131 | |----------|-------------|------------|---------------| 132 | |`connector.class`|Class implementing source connector for Jenkins.|Yes|org.aravind.oss.kafka.connect.
jenkins.JenkinsSourceConnector| 133 | |`tasks.max`| |Yes|1| 134 | |`jenkins.base.url`|The URL where jenkins server is running|Yes|None| 135 | |`jenkins.pollIntervalInMillis`|Frequency (in milliseconds) to poll for new build events in Jenkins|Yes|1 minute| 136 | |`jenkins.username`|If your Jenkins is secured, you can provide the username with this property|No|None| 137 | |`jenkins.password.or.api.token`|If your Jenkins is secured, you can provide the password or api token with this property|No|None| 138 | |`jenkins.connection.timeoutInMillis`|Connection timeout in milliseconds. This denotes the time elapsed before the connection established or Server responded to connection request.|Yes|500| 139 | |`jenkins.read.timeoutInMillis`|Response read timeout in milliseconds. After establishing the connection, the client socket waits for response after sending the request. This is the elapsed time since the client has sent request to the server before server responds.|Yes|3000| 140 | |`jenkins.jobs.resource.path`|Relative path to REST API|Yes|`/api/json`| 141 | |`topic`|Name of the topic where the Build status records are written to. **Make sure you explicitly create this topic using tools provided by Kafka. Do not rely on the default topic creation functionality in PRODUCTION.**|Yes|jenkins.connector.topic| 142 | 143 | ## How to use it? 144 | 145 | ### Standalone mode 146 | 147 | While testing, you might want to run the connector in standalone mode. Follow these steps 148 | 149 | 1. Download Confluent platform 3.0.0 ZIP file from [here](http://www.confluent.io/download) 150 | 2. Unzip it `unzip confluent-3.0.0-2.11.zip` 151 | 3. Move to Confluent home directory `cd confluent-3.0.0` 152 | 4. Start Zookeeper `./bin/zookeeper-server-start etc/kafka/zookeeper.properties` 153 | 5. Start Kafka Broker `./bin/kafka-server-start ./etc/kafka/server.properties` 154 | 6. Clone https://github.com/yaravind/kafka-connect-jenkins.git 155 | 7. Run `mvn clean install` 156 | 8. Update the `jenkins.base.url` property in `connect-jenkins-source.properties` file with your jenkins base url 157 | 9. Add Jenkins connector to classpath and start the connector in standalone mode `CLASSPATH=./target/kafka-connect-jenkins-0.5.0-SNAPSHOT-package/share/java/kafka-connect-jenkins/* <>/bin/connect-standalone connect-standalone.properties connect-jenkins-source.properties` 158 | 159 | > If you need to proxy to connect to Jenkins then append these to the above command `-Dhttp.proxyHost=... -Dhttp.proxyPort=... -Dhttps.proxyHost=... -Dhttps.proxyPort=...` 160 | 161 | ### Logging 162 | 163 | You can enable the logging for the connector by adding `log4j.logger.org.aravind.oss=DEBUG` (TRACE) to `connect-log4j.properties`. Following table can help you with finer level of logging. 164 | 165 | | Intent | Configuration (`DEBUG` or `TRACE`) | 166 | |--------|------------------------------------| 167 | | Enable logging for Jenkins **Source Connector** only | `log4j.logger.org.aravind.oss.kafka.connect.jenkins.JenkinsSourceConnector=DEBUG`
`log4j.logger.org.aravind.oss.kafka.connect.lib.TaskConfigBuilder=DEBUG` | 168 | | Enable logging for Jenkins **Source Task** only | `log4j.logger.org.aravind.oss.kafka.connect.jenkins.JenkinsSourceTask=DEBUG`
`log4j.logger.org.aravind.oss.kafka.connect.jenkins.ReadYourWritesOffsetStorageAdapter=ERROR`
`log4j.logger.org.aravind.oss.kafka.connect.lib.Partitions=DEBUG`| 169 | | Enable logging for communication with Jenkins API only | `log4j.logger.org.aravind.oss.jenkins.JenkinsClient=TRACE` | 170 | 171 | ### Monitoring 172 | 173 | > Replace `localhost` with your server and `8083` with `rest.port` configuration value. 174 | 175 | | REST API | Details | 176 | |----------|---------| 177 | | `http://localhost:8083/` | Kafka Connector | 178 | | `http://localhost:8083/connectors/` | All deployed connectors | 179 | | `http://localhost:8083/connectors/kafka-jenkins-source-connector/` | JenkinsSourceConnector | 180 | | `http://localhost:8083/connectors/kafka-jenkins-source-connector/config` | JenkinsSourceConnector config | 181 | | `http://localhost:8083/connectors/kafka-jenkins-source-connector/tasks` | JenkinsSourceConnector tasks | 182 | 183 | ## Limitations 184 | 185 | - Saves only the most recent build (**lastBuild**) know after configured `jenkins.pollIntervalInMillis`. i.e. if a Job has been built multiple times within the poll intervals, it isn't accounted for. 186 | - Requires JDK 8 to run the connector. Making JDK 7 compatible version isn't a big deal. Raise an issue if you need one. 187 | 188 | ## Dependencies 189 | 190 | - Maven 3.x 191 | - JDK 1.8 or newer 192 | - [Spock Framework](https://spockframework.github.io/spock/docs/1.0/index.html) 193 | - [Snap CI](https://snap-ci.com/yaravind/kafka-connect-jenkins/branch/master) for Continuous Integration 194 | - [JaCoCo](https://github.com/jacoco/jacoco) and [Coveralls](https://coveralls.io/github/yaravind/kafka-connect-jenkins) for code coverage 195 | - [Codacy](https://www.codacy.com/app/yaravind/kafka-connect-jenkins/dashboard) to measure code quality 196 | 197 | ## Contribute 198 | 199 | - Source code: https://github.com/yaravind/kafka-connect-jenkins 200 | - Issue tracker: https://github.com/yaravind/kafka-connect-jenkins/issues 201 | 202 | ## License 203 | 204 | The project is licensed under the Apache 2 license. 205 | -------------------------------------------------------------------------------- /connect-jenkins-source.properties: -------------------------------------------------------------------------------- 1 | name=kafka-jenkins-source-connector 2 | connector.class=org.aravind.oss.kafka.connect.jenkins.JenkinsSourceConnector 3 | tasks.max=1 4 | topic=jenkins.connector.topic 5 | jenkins.base.url=<> 6 | jenkins.connection.timeoutInMillis=500 7 | jenkins.read.timeoutInMillis=3000 8 | jenkins.pollIntervalInMillis=<> -------------------------------------------------------------------------------- /connect-standalone.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # These are defaults. This file just demonstrates how to override some settings. 17 | bootstrap.servers=localhost:9092 18 | 19 | # The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will 20 | # need to configure these based on the format they want their data in when loaded from or stored into Kafka 21 | key.converter=org.apache.kafka.connect.storage.StringConverter 22 | value.converter=org.apache.kafka.connect.json.JsonConverter 23 | # Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply 24 | # it to 25 | key.converter.schemas.enable=true 26 | value.converter.schemas.enable=true 27 | 28 | # The internal converter used for offsets and config data is configurable and must be specified, but most users will 29 | # always want to use the built-in default. Offset and config data is never visible outside of Copcyat in this format. 30 | internal.key.converter=org.apache.kafka.connect.json.JsonConverter 31 | internal.value.converter=org.apache.kafka.connect.json.JsonConverter 32 | internal.key.converter.schemas.enable=false 33 | internal.value.converter.schemas.enable=false 34 | 35 | offset.storage.file.filename=/tmp/connect.offsets 36 | # Flush much faster than normal, which is useful for testing/debugging 37 | offset.flush.interval.ms=1000 38 | -------------------------------------------------------------------------------- /gmavenplus-plugin-1.4.pom: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.codehaus 7 | codehaus-parent 8 | 4 9 | 10 | 11 | org.codehaus.gmavenplus 12 | gmavenplus-plugin 13 | maven-plugin 14 | 1.4 15 | 16 | 17 | UTF-8 18 | UTF-8 19 | 1.5 20 | 2.2.1 21 | 22 | 1.7 23 | 3.2.1 24 | 2.6 25 | 2.9.1 26 | 3.1 27 | 2.17 28 | 3.2 29 | 4.12 30 | 31 | 2.4.1 32 | 33 | 34 | 35 | 36 | org.apache.maven 37 | maven-plugin-api 38 | ${mavenVersion} 39 | 40 | 41 | org.apache.maven 42 | maven-project 43 | ${mavenVersion} 44 | 45 | 46 | org.apache.maven 47 | maven-core 48 | ${mavenVersion} 49 | 50 | 51 | org.apache.maven.shared 52 | file-management 53 | 1.2.1 54 | 55 | 56 | org.codehaus.plexus 57 | plexus-container-default 58 | 1.6 59 | 60 | 61 | org.codehaus.plexus 62 | plexus-classworlds 63 | 2.5.2 64 | 65 | 66 | org.apache.maven 67 | maven-plugin-registry 68 | ${mavenVersion} 69 | runtime 70 | 71 | 72 | 73 | org.fusesource.jansi 74 | jansi 75 | 1.11 76 | runtime 77 | 78 | 79 | 80 | 81 | jline 82 | jline 83 | 2.12 84 | runtime 85 | 86 | 87 | 88 | org.apache.ant 89 | ant 90 | 1.9.4 91 | runtime 92 | 93 | 94 | 95 | org.apache.ivy 96 | ivy 97 | 2.4.0 98 | runtime 99 | 100 | 101 | junit 102 | junit 103 | ${junitVersion} 104 | test 105 | 106 | 107 | org.mockito 108 | mockito-all 109 | 1.10.19 110 | test 111 | 112 | 113 | 114 | 115 | 116 | disable-doclint 117 | 118 | [1.8,) 119 | 120 | 121 | 122 | 123 | org.apache.maven.plugins 124 | maven-javadoc-plugin 125 | 126 | -Xdoclint:none 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | org.apache.maven.plugins 135 | maven-javadoc-plugin 136 | 137 | -Xdoclint:none 138 | 139 | 140 | 141 | 142 | 143 | 144 | indy 145 | 146 | 147 | org.codehaus.groovy 148 | groovy-all 149 | ${groovyVersion} 150 | indy 151 | test 152 | 153 | 154 | org.codehaus.groovy 155 | groovy-ant 156 | ${groovyVersion} 157 | indy 158 | test 159 | 160 | 161 | 162 | 163 | 164 | org.apache.maven.plugins 165 | maven-invoker-plugin 166 | 167 | 168 | indy 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | nonindy 177 | 178 | 179 | org.codehaus.groovy 180 | groovy-all 181 | ${groovyVersion} 182 | test 183 | 184 | 185 | org.codehaus.groovy 186 | groovy-ant 187 | ${groovyVersion} 188 | test 189 | 190 | 191 | 192 | 193 | 194 | org.apache.maven.plugins 195 | maven-invoker-plugin 196 | 197 | 198 | nonindy 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | pre2.3-indy 207 | 208 | 209 | org.codehaus.groovy 210 | groovy-all 211 | ${groovyVersion} 212 | indy 213 | test 214 | 215 | 216 | 217 | 218 | 219 | org.apache.maven.plugins 220 | maven-invoker-plugin 221 | 222 | 223 | pre2.3-indy 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | pre2.3-nonindy 232 | 233 | 234 | org.codehaus.groovy 235 | groovy-all 236 | ${groovyVersion} 237 | test 238 | 239 | 240 | 241 | 242 | 243 | org.apache.maven.plugins 244 | maven-invoker-plugin 245 | 246 | 247 | pre2.3-nonindy 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | release-sign-artifacts 256 | 257 | 258 | performRelease 259 | true 260 | 261 | 262 | 263 | 264 | 265 | org.apache.maven.plugins 266 | maven-gpg-plugin 267 | 1.5 268 | 269 | 270 | sign-artifacts 271 | 272 | sign 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | ${mavenVersion} 284 | 285 | 286 | 287 | 288 | 289 | org.apache.maven.plugins 290 | maven-enforcer-plugin 291 | 1.3.1 292 | 293 | 294 | enforce-versions 295 | 296 | enforce 297 | 298 | 299 | 300 | 301 | [${testMavenVersion},) 302 | 303 | 304 | [${testJavaVersion},) 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | org.apache.maven.plugins 313 | maven-clean-plugin 314 | 2.5 315 | 316 | 317 | org.apache.maven.plugins 318 | maven-dependency-plugin 319 | 2.8 320 | 321 | 322 | org.apache.maven.plugins 323 | maven-resources-plugin 324 | 2.6 325 | 326 | 327 | org.apache.maven.plugins 328 | maven-compiler-plugin 329 | ${compilerPluginVersion} 330 | 331 | ${javaVersion} 332 | ${javaVersion} 333 | 334 | 335 | 336 | org.codehaus.plexus 337 | plexus-component-metadata 338 | 1.5.5 339 | 340 | 341 | 342 | generate-metadata 343 | 344 | 345 | 346 | 347 | 348 | org.apache.maven.plugins 349 | maven-help-plugin 350 | 2.2 351 | 352 | 353 | org.apache.maven.plugins 354 | maven-plugin-plugin 355 | ${pluginPluginVersion} 356 | 357 | 358 | generate-helpmojo 359 | 360 | helpmojo 361 | 362 | 363 | 364 | generate-descriptor 365 | 366 | descriptor 367 | 368 | 369 | 370 | 371 | 372 | org.codehaus.mojo 373 | animal-sniffer-maven-plugin 374 | 1.11 375 | 376 | 377 | sniffClasses 378 | 379 | check 380 | 381 | 382 | 383 | 384 | 385 | org.codehaus.mojo.signature 386 | java15 387 | 1.0 388 | 389 | false 390 | 391 | 392 | org/apache/maven/* 393 | org/codehaus/plexus/* 394 | org/codehaus/classworlds/* 395 | 396 | 397 | 398 | 399 | org.apache.maven.plugins 400 | maven-surefire-plugin 401 | ${surefirePluginVersion} 402 | 403 | 404 | org.codehaus.mojo 405 | cobertura-maven-plugin 406 | ${coberturaPluginVersion} 407 | 408 | 409 | 410 | 411 | 412 | org/codehaus/gmavenplus/mojo/HelpMojo.class 413 | 414 | org/codehaus/gmavenplus/util/JDK5Utils.class 415 | 416 | 417 | 418 | 419 | 420 | org.apache.maven.plugins 421 | maven-jar-plugin 422 | 2.4 423 | 424 | 425 | org.apache.maven.plugins 426 | maven-invoker-plugin 427 | 1.8 428 | 429 | src/it 430 | ${project.build.directory}/it 431 | 432 | */pom.xml 433 | 434 | src/it/settings.xml 435 | ${project.build.directory}/local-repo 436 | 437 | 438 | 439 | org.apache.maven.plugins 440 | maven-source-plugin 441 | 2.2.1 442 | 443 | 444 | attach-sources 445 | 446 | jar 447 | 448 | 449 | 450 | 451 | 452 | org.apache.maven.plugins 453 | maven-javadoc-plugin 454 | ${javadocPluginVersion} 455 | 456 | 457 | attach-javadocs 458 | 459 | jar 460 | 461 | 462 | 463 | 464 | 465 | org.apache.maven.plugins 466 | maven-install-plugin 467 | 2.5.1 468 | 469 | 470 | org.apache.maven.plugins 471 | maven-deploy-plugin 472 | 2.8.1 473 | 474 | 475 | org.apache.maven.plugins 476 | maven-site-plugin 477 | 3.3 478 | 479 | 480 | org.apache.maven.plugins 481 | maven-release-plugin 482 | 2.5 483 | 484 | 485 | 486 | 487 | 488 | org.apache.maven.scm 489 | maven-scm-provider-gitexe 490 | 1.8.1 491 | 492 | 493 | org.apache.maven.scm 494 | maven-scm-manager-plexus 495 | 1.8.1 496 | 497 | 498 | 499 | org.kathrynhuxtable.maven.wagon 500 | wagon-gitsite 501 | 0.3.1 502 | 503 | 504 | 505 | 506 | 507 | 508 | 509 | org.apache.maven.plugins 510 | maven-plugin-plugin 511 | ${pluginPluginVersion} 512 | 513 | 514 | ${javaVersion} 515 | ${mavenVersion} 516 | 517 | 518 | 519 | 520 | org.apache.maven.plugins 521 | maven-jxr-plugin 522 | 2.4 523 | 524 | 525 | org.apache.maven.plugins 526 | maven-javadoc-plugin 527 | ${javadocPluginVersion} 528 | 529 | 530 | org.apache.maven.plugins 531 | maven-project-info-reports-plugin 532 | 2.7 533 | 534 | 535 | org.apache.maven.plugins 536 | maven-surefire-report-plugin 537 | ${surefirePluginVersion} 538 | 539 | 540 | org.codehaus.mojo 541 | cobertura-maven-plugin 542 | ${coberturaPluginVersion} 543 | 544 | 545 | org.codehaus.mojo 546 | taglist-maven-plugin 547 | 2.4 548 | 549 | 550 | 551 | 552 | Todo 553 | 554 | 555 | todo 556 | ignoreCase 557 | 558 | 559 | 560 | 561 | 562 | 563 | 564 | 565 | org.apache.maven.plugins 566 | maven-pmd-plugin 567 | 3.0.1 568 | 569 | 570 | org.codehaus.mojo 571 | findbugs-maven-plugin 572 | 3.0.0 573 | 574 | 575 | org.apache.maven.plugins 576 | maven-checkstyle-plugin 577 | 2.12 578 | 579 | 580 | org.codehaus.mojo 581 | versions-maven-plugin 582 | 2.1 583 | 584 | 585 | 586 | dependency-updates-report 587 | plugin-updates-report 588 | property-updates-report 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | github-project-site 599 | gitsite:git@github.com/groovy/GMavenPlus.git 600 | 601 | 602 | 603 | GMavenPlus Plugin 604 | Integrates Groovy into Maven projects. 605 | 2011 606 | http://groovy.github.io/GMavenPlus/ 607 | 608 | 609 | keeganwitt 610 | Keegan Witt 611 | 612 | developer 613 | 614 | 615 | 616 | 617 | 618 | Apache 2 619 | http://www.apache.org/licenses/LICENSE-2.0.txt 620 | 621 | 622 | 623 | Jira 624 | http://jira.codehaus.org/browse/GMAVENPLUS 625 | 626 | 627 | 628 | User List 629 | http://xircles.codehaus.org/manage_email/user@gmavenplus.codehaus.org 630 | http://xircles.codehaus.org/manage_email/user@gmavenplus.codehaus.org 631 | user@gmavenplus.codehaus.org 632 | http://gmavenplus.56682.x6.nabble.com/user-gmavenplus-codehaus-org-f5.html 633 | 634 | http://markmail.org/search/list:org.codehaus.gmavenplus#query:%20list%3Aorg.codehaus.gmavenplus.user 635 | 636 | 637 | 638 | Dev List 639 | http://xircles.codehaus.org/manage_email/dev@gmavenplus.codehaus.org 640 | http://xircles.codehaus.org/manage_email/dev@gmavenplus.codehaus.org 641 | dev@gmavenplus.codehaus.org 642 | http://gmavenplus.56682.x6.nabble.com/dev-gmavenplus-codehaus-org-f1.html 643 | 644 | http://markmail.org/search/list:org.codehaus.gmavenplus#query:%20list%3Aorg.codehaus.gmavenplus.dev 645 | 646 | 647 | 648 | Announcement List 649 | http://xircles.codehaus.org/manage_email/announce@gmavenplus.codehaus.org 650 | http://xircles.codehaus.org/manage_email/announce@gmavenplus.codehaus.org 651 | announce@gmavenplus.codehaus.org 652 | http://gmavenplus.56682.x6.nabble.com/announce-gmavenplus-codehaus-org-f4.html 653 | 654 | 655 | SCM List 656 | http://xircles.codehaus.org/manage_email/scm@gmavenplus.codehaus.org 657 | http://xircles.codehaus.org/manage_email/scm@gmavenplus.codehaus.org 658 | http://gmavenplus.56682.x6.nabble.com/scm-gmavenplus-codehaus-org-f9.html 659 | 660 | http://markmail.org/search/list:org.codehaus.gmavenplus#query:%20list%3Aorg.codehaus.gmavenplus.scm 661 | 662 | 663 | 664 | 665 | http://github.com/groovy/GMavenPlus 666 | scm:git:git@github.com:groovy/GMavenPlus.git 667 | scm:git:git@github.com:groovy/GMavenPlus.git 668 | 1.4 669 | 670 | 671 | 672 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.aravind.oss 8 | kafka-connect-jenkins 9 | 0.5.0-SNAPSHOT 10 | 11 | 12 | UTF-8 13 | 0.10.0.0 14 | 2.10.0 15 | 1.7.21 16 | 1.0-groovy-2.4 17 | 0.10.2 18 | 2.4.1 19 | 2.5 20 | 1.8 21 | 22 | 23 | 24 | 25 | 26 | org.apache.maven.plugins 27 | maven-compiler-plugin 28 | 3.5.1 29 | 30 | ${java.version} 31 | ${java.version} 32 | 33 | 34 | 35 | 36 | 38 | org.codehaus.gmavenplus 39 | gmavenplus-plugin 40 | 1.4 41 | 42 | 43 | 44 | compile 45 | testCompile 46 | 47 | 48 | 49 | 50 | 51 | 52 | org.eluder.coveralls 53 | coveralls-maven-plugin 54 | 4.1.0 55 | 56 | 57 | 58 | org.jacoco 59 | jacoco-maven-plugin 60 | 0.7.6.201602180812 61 | 62 | 63 | prepare-agent 64 | 65 | prepare-agent 66 | 67 | 68 | 69 | 70 | 71 | maven-assembly-plugin 72 | 2.5.3 73 | 74 | 75 | src/assembly/development.xml 76 | src/assembly/package.xml 77 | 78 | 79 | 80 | 81 | make-assembly 82 | package 83 | 84 | single 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | org.apache.kafka 95 | connect-api 96 | ${kafka.version} 97 | 98 | 99 | org.apache.kafka 100 | kafka-clients 101 | ${kafka.version} 102 | 103 | 104 | com.fasterxml.jackson.core 105 | jackson-databind 106 | ${jackson.version} 107 | 108 | 109 | com.fasterxml.jackson.core 110 | jackson-annotations 111 | ${jackson.version} 112 | 113 | 114 | org.slf4j 115 | slf4j-api 116 | ${slf4j.version} 117 | provided 118 | 119 | 120 | org.slf4j 121 | slf4j-log4j12 122 | ${slf4j.version} 123 | provided 124 | 125 | 126 | commons-io 127 | commons-io 128 | ${commons.io.version} 129 | 130 | 131 | org.spockframework 132 | spock-core 133 | ${spock.version} 134 | test 135 | 136 | 137 | org.codehaus.groovy 138 | groovy-all 139 | ${groovy.version} 140 | test 141 | 142 | 143 | com.github.dreamhead 144 | moco-core 145 | ${moco.version} 146 | test 147 | 148 | 149 | com.github.dreamhead 150 | moco-runner 151 | ${moco.version} 152 | test 153 | 154 | 155 | ch.qos.logback 156 | logback-classic 157 | 158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /src/assembly/development.xml: -------------------------------------------------------------------------------- 1 | 5 | 7 | development 8 | 9 | dir 10 | 11 | false 12 | 13 | 14 | share/java/kafka-connect-jenkins/ 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/assembly/package.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | package 7 | 8 | dir 9 | 10 | false 11 | 12 | 13 | ${project.basedir} 14 | share/doc/kafka-connect-jenkins/ 15 | 16 | README* 17 | LICENSE* 18 | NOTICE* 19 | licenses/ 20 | 21 | 22 | 23 | ${project.basedir}/config 24 | etc/kafka-connect-jdbc 25 | 26 | * 27 | 28 | 29 | 30 | 31 | 32 | share/java/kafka-connect-jenkins 33 | true 34 | true 35 | 36 | org.apache.kafka:connect-api 37 | org.apache.kafka:kafka-clients 38 | org.apache.kafka:connect-api 39 | org.apache.kafka:connect-api 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /src/assembly/standalone.xml: -------------------------------------------------------------------------------- 1 | 5 | 7 | standalone 8 | 9 | jar 10 | 11 | false 12 | 13 | 14 | ${project.basedir} 15 | / 16 | 17 | README* 18 | LICENSE* 19 | NOTICE* 20 | 21 | 22 | 23 | 24 | 25 | / 26 | true 27 | true 28 | runtime 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/jenkins/JenkinsClient.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.apache.commons.io.IOUtils; 5 | import org.aravind.oss.jenkins.domain.Jenkins; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.*; 10 | import java.net.HttpURLConnection; 11 | import java.net.URL; 12 | import java.nio.charset.Charset; 13 | import java.util.Base64; 14 | import java.util.Optional; 15 | 16 | /** 17 | * Represents one single running instance of Jenkins. You will create one object per running Jenkins instance. 18 | * 19 | * @author Aravind R Yarram 20 | * @since 0.5.0 21 | */ 22 | public class JenkinsClient { 23 | private final int connTimeoutInMillis; 24 | private final int readTimeoutInMillis; 25 | private final URL resourceUrl; 26 | private Optional userName = Optional.empty(); 27 | private Optional passwordOrApiToken = Optional.empty(); 28 | private ObjectMapper mapper = new ObjectMapper(); 29 | private static final Logger logger = LoggerFactory.getLogger(JenkinsClient.class); 30 | 31 | /** 32 | * @param url resource url of the jenkins item 33 | * @param connTimeout Connection timeout in milliseconds. This denotes the time elapsed before the connection established or Server responded to connection request. 34 | * @param readTimeout Response read timeout in milliseconds. After establishing the connection, the client socket waits for response after sending the request. This is the elapsed time since the client has sent request to the server before server responds. 35 | */ 36 | public JenkinsClient(URL url, int connTimeout, int readTimeout) { 37 | resourceUrl = url; 38 | connTimeoutInMillis = connTimeout; 39 | readTimeoutInMillis = readTimeout; 40 | } 41 | 42 | /** 43 | * @param url url of the jenkins instance 44 | * @param uname username if authentication is enabled in jenkins instance 45 | * @param password password or API token if authentication is enabled in jenkins instance 46 | * @param connTimeout Connection timeout in milliseconds. This denotes the time elapsed before the connection established or Server responded to connection request. 47 | * @param readTimeout Response read timeout in milliseconds. After establishing the connection, the client socket waits for response after sending the request. This is the elapsed time since the client has sent request to the server before server responds. 48 | */ 49 | public JenkinsClient(URL url, String uname, String password, int connTimeout, int readTimeout) throws JenkinsException { 50 | if (uname.isEmpty()) { 51 | throw new JenkinsException("Missing Jenkins username for authentication"); 52 | } 53 | if (password.isEmpty()) { 54 | throw new JenkinsException("Missing Jenkins password (or API token) for authentication"); 55 | } 56 | resourceUrl = url; 57 | userName = Optional.of(uname); 58 | passwordOrApiToken = Optional.of(password); 59 | connTimeoutInMillis = connTimeout; 60 | readTimeoutInMillis = readTimeout; 61 | } 62 | 63 | public HttpURLConnection connect() throws JenkinsException { 64 | logger.trace("Connecting to {} with conn timeout {} ms and read timeout {} ms", resourceUrl, connTimeoutInMillis, readTimeoutInMillis); 65 | HttpURLConnection conn = null; 66 | try { 67 | logger.trace("Using username {} for connection.", userName); 68 | conn = (HttpURLConnection) resourceUrl.openConnection(); 69 | conn.setConnectTimeout(connTimeoutInMillis); 70 | conn.setReadTimeout(readTimeoutInMillis); 71 | if (userName.isPresent()) { 72 | conn.setRequestProperty("Authorization", "Basic " + getAuthenticationString()); 73 | logger.trace("Using Basic Authentication with username {}", userName); 74 | } 75 | conn.connect(); 76 | 77 | return conn; 78 | } catch (IOException e) { 79 | logger.error("Error while connecting to {}", resourceUrl, e); 80 | throw new JenkinsException("Error while opening a connection to " + resourceUrl, e); 81 | } 82 | } 83 | 84 | private String getAuthenticationString() { 85 | String authString = userName.get() + ":" + passwordOrApiToken.get(); 86 | return Base64.getEncoder().encodeToString(authString.getBytes()); 87 | } 88 | 89 | public Optional get() throws JenkinsException { 90 | logger.trace("GET to {}", resourceUrl); 91 | HttpURLConnection conn = connect(); 92 | 93 | try { 94 | InputStream is = conn.getInputStream(); 95 | String resp = IOUtils.toString(is, Charset.forName("UTF-8")); 96 | 97 | // close the input stream so that the connection can be reused 98 | is.close(); 99 | 100 | return Optional.of(resp); 101 | } catch (IOException e) { 102 | logger.warn("IGNORING this exception. Just a WARNING to debug this issue. Error while HTTP GET to {}", resourceUrl, e); 103 | //Need to read even the error stream so that we can take advantage of socket reuse in Keep-Alive 104 | //http://docs.oracle.com/javase/7/docs/technotes/guides/net/http-keepalive.html 105 | try { 106 | int respCode = ((HttpURLConnection) conn).getResponseCode(); 107 | logger.warn("HTTP response code {}", respCode); 108 | InputStream es = ((HttpURLConnection) conn).getErrorStream(); 109 | 110 | if (es != null) { 111 | // read the error response body so that the connection can be reused 112 | IOUtils.toString(es, Charset.forName("UTF-8")); 113 | 114 | // close the error stream so that the connection can be reused 115 | es.close(); 116 | } 117 | } catch (IOException ex) { 118 | // ignore the exception 119 | } 120 | } 121 | return Optional.empty(); 122 | } 123 | 124 | public Optional getJenkins() throws JenkinsException { 125 | Optional resp = get(); 126 | 127 | if (resp.isPresent()) { 128 | try { 129 | Jenkins j = mapper.readValue(resp.get(), Jenkins.class); 130 | return Optional.of(j); 131 | } catch (IOException e) { 132 | logger.error("Error while parsing the JSON {}", resp.get(), e); 133 | return Optional.empty(); 134 | } 135 | } 136 | return Optional.empty(); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/jenkins/JenkinsException.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins; 2 | 3 | /** 4 | * Wrapper exception for everything from Jenkins sub-system 5 | * 6 | * @author Aravind R Yarram 7 | * @since 0.5.0 8 | */ 9 | public class JenkinsException extends Exception { 10 | public JenkinsException(String msg, Exception e) { 11 | super(msg, e); 12 | } 13 | 14 | public JenkinsException(String msg) { 15 | super(msg); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/jenkins/domain/Build.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins.domain; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | import org.aravind.oss.jenkins.JenkinsClient; 5 | import org.aravind.oss.jenkins.JenkinsException; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.net.MalformedURLException; 10 | import java.net.URL; 11 | import java.util.Optional; 12 | 13 | /** 14 | * @author Aravind R Yarram 15 | * @since 0.5.0 16 | */ 17 | @JsonIgnoreProperties(ignoreUnknown = true) 18 | public class Build extends JenkinsItem { 19 | private Long number; 20 | private String url; 21 | private static final Logger logger = LoggerFactory.getLogger(Build.class); 22 | 23 | public Long getNumber() { 24 | return number; 25 | } 26 | 27 | public void setNumber(Long number) { 28 | this.number = number; 29 | } 30 | 31 | public String getUrl() { 32 | return url; 33 | } 34 | 35 | public void setUrl(String url) { 36 | this.url = url; 37 | } 38 | 39 | public String getBuildDetailsResource() { 40 | return getUrl() != null ? getUrl() + "api/json" : null; 41 | } 42 | 43 | public Optional getDetails() { 44 | logger.debug("GET build details for {}", getBuildDetailsResource()); 45 | String username = getUsername(); 46 | String password = getPassword(); 47 | try { 48 | if (username != null && !username.isEmpty()) { 49 | setClient(new JenkinsClient(new URL(getBuildDetailsResource()), username, password, getConnTimeoutInMillis(), getReadTimeoutInMillis())); 50 | } 51 | else { 52 | setClient(new JenkinsClient(new URL(getBuildDetailsResource()), getConnTimeoutInMillis(), getReadTimeoutInMillis())); 53 | } 54 | return getClient().get(); 55 | } catch (MalformedURLException | JenkinsException e) { 56 | logger.error("WARNING only. Unable to get the build details from {}", getUrl(), e); 57 | return Optional.empty(); 58 | } 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "Build{" + 64 | "number=" + number + 65 | ", url='" + url + '\'' + 66 | '}'; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/jenkins/domain/BuildCollection.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins.domain; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @author Aravind R Yarram 9 | * @since 0.5.0 10 | */ 11 | @JsonIgnoreProperties(ignoreUnknown = true) 12 | public class BuildCollection { 13 | private String name; 14 | private List builds; 15 | private Build lastBuild; 16 | 17 | public String getName() { 18 | return name; 19 | } 20 | 21 | public void setName(String name) { 22 | this.name = name; 23 | } 24 | 25 | public List getBuilds() { 26 | return builds; 27 | } 28 | 29 | public void setBuilds(List builds) { 30 | this.builds = builds; 31 | } 32 | 33 | public Build getLastBuild() { 34 | return lastBuild; 35 | } 36 | 37 | public void setLastBuild(Build lastBuild) { 38 | this.lastBuild = lastBuild; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "BuildCollection{" + 44 | "name='" + name + '\'' + 45 | ", builds=" + builds + 46 | ", lastBuild=" + lastBuild + 47 | '}'; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/jenkins/domain/Jenkins.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins.domain; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | 5 | import java.util.Collections; 6 | import java.util.List; 7 | 8 | /** 9 | * Domain object representing the all jobs resource: /api/json 10 | * 11 | * @author Aravind R Yarram 12 | * @since 0.5.0 13 | */ 14 | @JsonIgnoreProperties(ignoreUnknown = true) 15 | public class Jenkins { 16 | private List jobs; 17 | 18 | public List getJobs() { 19 | return jobs; 20 | } 21 | 22 | public void setJobs(List jobs) { 23 | this.jobs = Collections.unmodifiableList(jobs); 24 | } 25 | 26 | public int getJobCount() { 27 | return getJobs() == null || getJobs().isEmpty() ? 0 : getJobs().size(); 28 | } 29 | 30 | @Override 31 | public String toString() { 32 | return "Jenkins{" + 33 | "jobs=" + jobs + 34 | '}'; 35 | } 36 | } -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/jenkins/domain/JenkinsItem.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins.domain; 2 | 3 | import org.aravind.oss.jenkins.JenkinsClient; 4 | 5 | /** 6 | * Base abstraction that all Jenkin's domain classes extend from. 7 | * This encapsulates the properties required to fetch dependent parts of the whole. 8 | * For e.g. A {@link Build} need to obtain {@link Build#getDetails()}. 9 | * 10 | * @author Aravind R Yarram 11 | * @since 0.5.0 12 | */ 13 | public class JenkinsItem { 14 | protected JenkinsClient client; 15 | protected int connTimeoutInMillis; 16 | protected int readTimeoutInMillis; 17 | protected String username; 18 | protected String password; 19 | 20 | public int getReadTimeoutInMillis() { 21 | return readTimeoutInMillis; 22 | } 23 | 24 | public void setReadTimeoutInMillis(int readTimeoutInMillis) { 25 | this.readTimeoutInMillis = readTimeoutInMillis; 26 | } 27 | 28 | public int getConnTimeoutInMillis() { 29 | return connTimeoutInMillis; 30 | } 31 | 32 | public void setConnTimeoutInMillis(int connTimeoutInMillis) { 33 | this.connTimeoutInMillis = connTimeoutInMillis; 34 | } 35 | 36 | public JenkinsClient getClient() { 37 | return client; 38 | } 39 | 40 | public void setClient(JenkinsClient client) { 41 | this.client = client; 42 | } 43 | 44 | public String getUsername() { 45 | return username; 46 | } 47 | 48 | public void setUsername(String username) { 49 | this.username = username; 50 | } 51 | 52 | public String getPassword() { 53 | return password; 54 | } 55 | 56 | public void setPassword(String password) { 57 | this.password = password; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/jenkins/domain/Job.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins.domain; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | 5 | /** 6 | * @author Aravind R Yarram 7 | * @since 0.5.0 8 | */ 9 | @JsonIgnoreProperties(ignoreUnknown = true) 10 | public class Job extends JenkinsItem { 11 | private String name; 12 | private String url; 13 | private String color; 14 | 15 | public String getColor() { 16 | return color; 17 | } 18 | 19 | public void setColor(String color) { 20 | this.color = color; 21 | } 22 | 23 | public String getName() { 24 | return name; 25 | } 26 | 27 | public void setName(String name) { 28 | this.name = name; 29 | } 30 | 31 | public String getUrl() { 32 | return url; 33 | } 34 | 35 | public void setUrl(String url) { 36 | this.url = url; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return "Job{" + 42 | "name='" + name + '\'' + 43 | ", url='" + url + '\'' + 44 | ", color='" + color + '\'' + 45 | '}'; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/jenkins/JenkinsSourceConfig.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigException; 6 | 7 | import java.net.MalformedURLException; 8 | import java.net.URL; 9 | import java.util.Map; 10 | 11 | /** 12 | * Convenient class to hold configuration properties of the JenkinsSourceConnector 13 | * 14 | * @author Aravind R Yarram 15 | * @since 0.5 16 | */ 17 | public class JenkinsSourceConfig extends AbstractConfig { 18 | 19 | public static final String JENKINS_BASE_URL_CONFIG = "jenkins.base.url"; 20 | public static final String JENKINS_BASE_URL_DISPLAY = "Jenkins server url."; 21 | private static final String JENKINS_BASE_URL_DOC = "This is the URL of the home page of your Jenkins installation. " 22 | + "For e.g. https://builds.apache.org/ is the base url of the Apache Jenkins public instance. " + 23 | "In some installations, a context/prefix might have been specified (using the --prefix; For e.g. --prefix=/jenkins). " + 24 | "If so then the url should include the prefix as well. "; 25 | 26 | public static final String JOBS_RESOURCE_PATH_CONFIG = "jenkins.jobs.resource.path"; 27 | private static final String JOBS_RESOURCE_PATH_DEFAULT = "/api/json"; 28 | private static final String JOBS_RESOURCE_PATH_DISPLAY = "Relative path to REST API. Default (/api/json)"; 29 | private static final String JOBS_RESOURCE_PATH_DOC = "This is the REST resource path to retrieve all jobs defined in the Jenkins instance. " + 30 | "This is an optional configuration property. If not specified the default \"/api/json\" will be used."; 31 | 32 | public static final String JENKINS_USERNAME_CONFIG = "jenkins.username"; 33 | public static final String JENKINS_USERNAME_DISPLAY = "Jenkins Username."; 34 | private static final String JENKINS_USERNAME_CONFIG_DOC = "Username to use when connecting to protected Jenkins."; 35 | 36 | public static final String JENKINS_PASSWORD_OR_API_TOKEN_CONFIG = "jenkins.password.or.api.token"; 37 | public static final String JENKINS_PASSWORD_OR_API_TOKEN_DISPLAY = "Password (or API Token)"; 38 | private static final String JENKINS_PASSWORD_OR_API_TOKEN_DOC = "Password (or API Token) to use when connecting to protected Jenkins."; 39 | 40 | public static final String JENKINS_CONN_TIMEOUT_CONFIG = "jenkins.connection.timeoutInMillis"; 41 | public static final String JENKINS_CONN_TIMEOUT_DISPLAY = "Connection timeout in milliseconds."; 42 | public static final int JENKINS_CONN_TIMEOUT_DEFAULT = 500; 43 | public static final String JENKINS_CONN_TIMEOUT_DOC = "Connection timeout in milliseconds. " + 44 | "This denotes the time elapsed before the connection established or Server responded to connection request."; 45 | 46 | public static final String JENKINS_READ_TIMEOUT_CONFIG = "jenkins.read.timeoutInMillis"; 47 | public static final String JENKINS_READ_TIMEOUT_DISPLAY = "Response read timeout in milliseconds."; 48 | public static final int JENKINS_READ_TIMEOUT_DEFAULT = 3000; 49 | public static final String JENKINS_READ_TIMEOUT_DOC = "Response read timeout in milliseconds. After establishing the connection, " + 50 | "the client socket waits for response after sending the request. " + 51 | "This is the elapsed time since the client has sent request to the server before server responds."; 52 | 53 | public static final String TOPIC_CONFIG = "topic"; 54 | public static final String TOPIC_DISPLAY = "Topic to persist build events."; 55 | public static final String TOPIC_CONFIG_DOC = "This is the name of the Kafka Topic to which the source records containing Jenkins Build details are written to."; 56 | public static final String TOPIC_CONFIG_DEFAULT = "jenkins.connector.topic"; 57 | 58 | public static final String JENKINS_POLL_INTERVAL_MS_CONFIG = "jenkins.pollIntervalInMillis"; 59 | private static final String JENKINS_POLL_INTERVAL_MS_DOC = "Frequency in ms to poll for new build events in Jenkins."; 60 | public static final int JENKINS_POLL_INTERVAL_MS_DEFAULT = 60000;//every minute 61 | private static final String JENKINS_POLL_INTERVAL_MS_DISPLAY = "Poll Interval in milliseconds"; 62 | 63 | public static final String JENKINS_GROUP = "Jenkins"; 64 | public static final String CONNECTOR_GROUP = "Connector"; 65 | 66 | public static final ConfigDef DEFS = new ConfigDef(); 67 | 68 | static { 69 | DEFS 70 | .define(JENKINS_BASE_URL_CONFIG, ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, JENKINS_BASE_URL_DOC, JENKINS_GROUP, 1, ConfigDef.Width.LONG, JENKINS_BASE_URL_DISPLAY) 71 | .define(JENKINS_USERNAME_CONFIG, ConfigDef.Type.STRING, "", ConfigDef.Importance.LOW, JENKINS_USERNAME_CONFIG_DOC, JENKINS_GROUP, 5, ConfigDef.Width.MEDIUM, JENKINS_USERNAME_DISPLAY) 72 | .define(JENKINS_PASSWORD_OR_API_TOKEN_CONFIG, ConfigDef.Type.STRING, "", ConfigDef.Importance.LOW, JENKINS_PASSWORD_OR_API_TOKEN_DOC, JENKINS_GROUP, 6, ConfigDef.Width.MEDIUM, JENKINS_PASSWORD_OR_API_TOKEN_DISPLAY) 73 | .define(JENKINS_CONN_TIMEOUT_CONFIG, ConfigDef.Type.INT, JENKINS_CONN_TIMEOUT_DEFAULT, ConfigDef.Importance.LOW, JENKINS_CONN_TIMEOUT_DOC, JENKINS_GROUP, 3, ConfigDef.Width.SHORT, JENKINS_CONN_TIMEOUT_DISPLAY) 74 | .define(JENKINS_READ_TIMEOUT_CONFIG, ConfigDef.Type.INT, JENKINS_READ_TIMEOUT_DEFAULT, ConfigDef.Importance.LOW, JENKINS_READ_TIMEOUT_DOC, JENKINS_GROUP, 4, ConfigDef.Width.SHORT, JENKINS_READ_TIMEOUT_DISPLAY) 75 | .define(JOBS_RESOURCE_PATH_CONFIG, ConfigDef.Type.STRING, JOBS_RESOURCE_PATH_DEFAULT, ConfigDef.Importance.LOW, JOBS_RESOURCE_PATH_DOC, JENKINS_GROUP, 7, ConfigDef.Width.MEDIUM, JOBS_RESOURCE_PATH_DISPLAY) 76 | .define(JENKINS_POLL_INTERVAL_MS_CONFIG, ConfigDef.Type.LONG, JENKINS_POLL_INTERVAL_MS_DEFAULT, ConfigDef.Importance.LOW, JENKINS_POLL_INTERVAL_MS_DOC, JENKINS_GROUP, 2, ConfigDef.Width.SHORT, JENKINS_POLL_INTERVAL_MS_DISPLAY) 77 | .define(TOPIC_CONFIG, ConfigDef.Type.STRING, TOPIC_CONFIG_DEFAULT, ConfigDef.Importance.LOW, TOPIC_CONFIG_DOC, CONNECTOR_GROUP, 1, ConfigDef.Width.LONG, TOPIC_DISPLAY); 78 | } 79 | 80 | public JenkinsSourceConfig(Map originals) { 81 | super(DEFS, originals); 82 | } 83 | 84 | public URL getJenkinsUrl() { 85 | try { 86 | return new URL(getString(JENKINS_BASE_URL_CONFIG)); 87 | } catch (MalformedURLException e) { 88 | throw new ConfigException("Couldn't create the URL from " + getString(JENKINS_BASE_URL_CONFIG), e); 89 | } 90 | } 91 | 92 | public String getUsername() { 93 | return getString(JENKINS_USERNAME_CONFIG); 94 | } 95 | 96 | public String getPasswordOrApiToken() { 97 | return getString(JENKINS_PASSWORD_OR_API_TOKEN_CONFIG); 98 | } 99 | 100 | public boolean isProtected() { 101 | return getString(JENKINS_USERNAME_CONFIG) != null && !getString(JENKINS_USERNAME_CONFIG).isEmpty(); 102 | } 103 | 104 | public URL getJobsResource() { 105 | try { 106 | return new URL(getString(JENKINS_BASE_URL_CONFIG) + JOBS_RESOURCE_PATH_DEFAULT); 107 | } catch (MalformedURLException e) { 108 | throw new ConfigException("Couldn't create the URL from " + getString(JENKINS_BASE_URL_CONFIG), e); 109 | } 110 | } 111 | 112 | public int getJenkinsConnTimeout() { 113 | return getInt(JENKINS_CONN_TIMEOUT_CONFIG); 114 | } 115 | 116 | public int getJenkinsReadTimeout() { 117 | return getInt(JENKINS_READ_TIMEOUT_CONFIG); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/jenkins/JenkinsSourceConnector.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.connect.connector.Task; 5 | import org.apache.kafka.connect.errors.ConnectException; 6 | import org.apache.kafka.connect.source.SourceConnector; 7 | import org.aravind.oss.jenkins.JenkinsException; 8 | import org.aravind.oss.jenkins.JenkinsClient; 9 | import org.aravind.oss.jenkins.domain.Jenkins; 10 | import org.aravind.oss.jenkins.domain.Job; 11 | import org.aravind.oss.kafka.connect.lib.TaskConfigBuilder; 12 | import org.aravind.oss.kafka.connect.lib.TaskConfigExtractor; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.net.HttpURLConnection; 17 | import java.util.*; 18 | 19 | /** 20 | * Is responsible for breaking the job into a set make tasks that can be distributed to workers. 21 | * 22 | * @author Aravind R Yarram 23 | * @since 0.5.0 24 | */ 25 | public class JenkinsSourceConnector extends SourceConnector { 26 | private JenkinsSourceConfig jenkinsCfg; 27 | private JenkinsClient client; 28 | private static Logger logger = LoggerFactory.getLogger(JenkinsSourceConnector.class); 29 | 30 | @Override 31 | public String version() { 32 | return Version.get(); 33 | } 34 | 35 | /** 36 | * Start this Connector. This method will only be called on a clean Connector, i.e. it has either just been 37 | * instantiated and initialized or stop() has been invoked. 38 | * 39 | * @param props 40 | */ 41 | @Override 42 | public void start(Map props) { 43 | logger.info("JenkinsSourceConnector starting"); 44 | jenkinsCfg = new JenkinsSourceConfig(props); 45 | 46 | //Do a test connection to Fail Fast 47 | try { 48 | logger.trace("Doing a test connection to {}", jenkinsCfg.getJobsResource()); 49 | if (jenkinsCfg.isProtected()) { 50 | client = new JenkinsClient(jenkinsCfg.getJobsResource(), jenkinsCfg.getUsername(), jenkinsCfg.getPasswordOrApiToken(), jenkinsCfg.getJenkinsConnTimeout(), jenkinsCfg.getJenkinsReadTimeout()); 51 | } 52 | else { 53 | client = new JenkinsClient(jenkinsCfg.getJobsResource(), jenkinsCfg.getJenkinsConnTimeout(), jenkinsCfg.getJenkinsReadTimeout()); 54 | } 55 | HttpURLConnection connection = client.connect(); 56 | connection.disconnect(); 57 | } catch (JenkinsException e) { 58 | throw new ConnectException("Unable to open connection to " + jenkinsCfg.getJenkinsUrl(), e); 59 | } 60 | } 61 | 62 | @Override 63 | public Class taskClass() { 64 | return JenkinsSourceTask.class; 65 | } 66 | 67 | /** 68 | * Returns a set make configurations for {@link JenkinsSourceTask} based on the current configuration, producing at most {@code numTasks} configurations. 69 | * 70 | * @param maxTasks maximum number make configurations to generate 71 | * @return configurations for Tasks 72 | */ 73 | @Override 74 | public List> taskConfigs(int maxTasks) { 75 | logger.debug("Calculating taskConfigs"); 76 | Optional resp = null; 77 | try { 78 | resp = client.getJenkins(); 79 | } catch (JenkinsException e) { 80 | //TODO sometimes the client might have been brought down. Let us handle it silently for now. 81 | logger.warn("Error while GET to " + jenkinsCfg.getJobsResource() + ". Ignoring it.", e); 82 | } 83 | 84 | TaskConfigExtractor taskConfigExtractor = new JobTaskConfigExtractor(); 85 | 86 | if (resp.isPresent()) { 87 | Jenkins jenkins = resp.get(); 88 | 89 | TaskConfigBuilder taskCfgBuilder = new TaskConfigBuilder(maxTasks, JenkinsSourceTask.JOB_URLS, jenkinsCfg, taskConfigExtractor); 90 | 91 | return taskCfgBuilder.build(jenkins.getJobs()); 92 | } 93 | return Collections.emptyList(); 94 | } 95 | 96 | @Override 97 | public void stop() { 98 | logger.info("JenkinsSourceConnector stopping"); 99 | //Not used at this moment 100 | } 101 | 102 | @Override 103 | public ConfigDef config() { 104 | return JenkinsSourceConfig.DEFS; 105 | } 106 | 107 | public JenkinsSourceConfig getJenkinsCfg() { 108 | return jenkinsCfg; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/jenkins/JenkinsSourceTask.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.apache.kafka.connect.data.Schema; 5 | import org.apache.kafka.connect.source.SourceRecord; 6 | import org.apache.kafka.connect.source.SourceTask; 7 | import org.apache.kafka.connect.source.SourceTaskContext; 8 | import org.aravind.oss.jenkins.JenkinsClient; 9 | import org.aravind.oss.jenkins.JenkinsException; 10 | import org.aravind.oss.jenkins.domain.Build; 11 | import org.aravind.oss.jenkins.domain.BuildCollection; 12 | import org.aravind.oss.kafka.connect.lib.SourceOffset; 13 | import org.aravind.oss.kafka.connect.lib.Partitions; 14 | import org.aravind.oss.kafka.connect.lib.SourcePartition; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import java.io.IOException; 19 | import java.net.MalformedURLException; 20 | import java.net.URL; 21 | import java.text.SimpleDateFormat; 22 | import java.util.*; 23 | import java.util.concurrent.atomic.AtomicBoolean; 24 | 25 | import org.apache.kafka.common.utils.SystemTime; 26 | import org.apache.kafka.common.utils.Time; 27 | 28 | import static java.lang.String.valueOf; 29 | import static org.aravind.oss.kafka.connect.jenkins.JenkinsSourceConfig.*; 30 | import static org.aravind.oss.kafka.connect.jenkins.JenkinsSourceConfig.JENKINS_CONN_TIMEOUT_CONFIG; 31 | import static org.aravind.oss.kafka.connect.jenkins.Util.extractJobName; 32 | import static org.aravind.oss.kafka.connect.jenkins.Util.urlDecode; 33 | 34 | /** 35 | * @author Aravind R Yarram 36 | * @since 0.5.0 37 | */ 38 | public class JenkinsSourceTask extends SourceTask { 39 | public static final String JOB_URLS = "job.urls"; 40 | public static final String JOB_NAME = "jobName"; 41 | public static final String BUILD_NUMBER = "buildNumber"; 42 | private static final Logger logger = LoggerFactory.getLogger(JenkinsSourceTask.class); 43 | 44 | private Time time; 45 | private long lastUpdate; 46 | private long pollIntervalInMillis; 47 | private static int totalJenkinsPulls = 1; 48 | private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); 49 | private final static Partitions partitions = new Partitions(JenkinsSourceTask.JOB_NAME); 50 | 51 | private Map taskProps; 52 | private ObjectMapper mapper = new ObjectMapper(); 53 | private AtomicBoolean stop; 54 | private ReadYourWritesOffsetStorageAdapter storageAdapter; 55 | 56 | public JenkinsSourceTask() { 57 | this.time = new SystemTime(); 58 | } 59 | 60 | @Override 61 | public String version() { 62 | return Version.get(); 63 | } 64 | 65 | @Override 66 | public void start(Map props) { 67 | logger.info("JenkinsSourceTask starting"); 68 | lastUpdate = 0; 69 | taskProps = props; 70 | pollIntervalInMillis = Long.parseLong(taskProps.get(JENKINS_POLL_INTERVAL_MS_CONFIG)); 71 | stop = new AtomicBoolean(false); 72 | } 73 | 74 | public Optional createSourceRecord(String jobUrl, Long lastSavedBuildNumber) { 75 | JenkinsClient client = null; 76 | 77 | try { 78 | if (getJenkinsUsername() != null && !getJenkinsUsername().isEmpty()) { 79 | client = new JenkinsClient(new URL(jobUrl + "api/json"), getJenkinsUsername(), getJenkinsPassword(), getJenkinsConnTimeout(), getJenkinsReadTimeout()); 80 | } 81 | else { 82 | client = new JenkinsClient(new URL(jobUrl + "api/json"), getJenkinsConnTimeout(), getJenkinsReadTimeout()); 83 | } 84 | } 85 | catch (JenkinsException je) { 86 | logger.error("Can't create URL object for Jenkins server at {}.", jobUrl, je); 87 | //TODO Silently log the error and ignore? What should we do? 88 | } 89 | catch (MalformedURLException e) { 90 | logger.error("Can't create URL object for Jenkins server at {}.", jobUrl, e); 91 | //TODO Silently log the error and ignore? What should we do? 92 | } 93 | Optional resp = Optional.empty(); 94 | if (client != null) { 95 | try { 96 | logger.debug("GET job details for {}", jobUrl + "api/json"); 97 | resp = client.get(); 98 | } catch (JenkinsException e) { 99 | logger.warn("Can't do a GET to resource {}", jobUrl + "api/json", e); 100 | //TODO Silently log the error and ignore? What should we do? 101 | } 102 | if (resp.isPresent()) { 103 | logger.trace("Resp: {}", resp.get()); 104 | BuildCollection builds = null; 105 | 106 | //build SourceRecords 107 | try { 108 | builds = mapper.readValue(resp.get(), BuildCollection.class); 109 | } catch (IOException e) { 110 | logger.error("Error while parsing the Build JSON {} for {}", resp.get(), jobUrl + "api/json", e); 111 | } 112 | 113 | logger.trace("Builds are: {}", builds); 114 | 115 | if (builds != null) { 116 | SourcePartition partition = partitions.make(builds.getName()); 117 | 118 | Build lastBuild = builds.getLastBuild(); 119 | 120 | //Some jobs might not have any builds. TODO need to figure out how to represent these 121 | //And the lastBuild might have already been stored 122 | 123 | if (lastBuild != null && !lastBuild.getNumber().equals(lastSavedBuildNumber)) { 124 | Long offsetValue = lastBuild.getNumber(); 125 | logger.debug("Partition: {}, lastBuild: {}, lastSavedBuild: {}", partition.value, offsetValue, lastSavedBuildNumber); 126 | SourceOffset sourceOffset = SourceOffset.make(BUILD_NUMBER, offsetValue); 127 | 128 | //get Build details 129 | lastBuild.setConnTimeoutInMillis(getJenkinsConnTimeout()); 130 | lastBuild.setReadTimeoutInMillis(getJenkinsReadTimeout()); 131 | lastBuild.setUsername(getJenkinsUsername()); 132 | lastBuild.setPassword(getJenkinsPassword()); 133 | Optional lastBuildDetails = lastBuild.getDetails(); 134 | 135 | if (lastBuildDetails.isPresent()) { 136 | //add build details JSON string as the value 137 | logger.debug("Create SourceRecord for {}", partition.value); 138 | SourceRecord record = new SourceRecord(partition.encoded, sourceOffset.encoded, taskProps.get(TOPIC_CONFIG), Schema.STRING_SCHEMA, partition.value, Schema.STRING_SCHEMA, lastBuildDetails.get()); 139 | storageAdapter.cache(partition, sourceOffset); 140 | 141 | return Optional.of(record); 142 | } else { 143 | logger.debug("Ignoring job details for {} as there are no builds for this Job. Not creating SourceRecord.", lastBuild.getBuildDetailsResource()); 144 | } 145 | } else { 146 | logger.debug("Not creating SourceRecord for {} because either the lastBuild details aren't available or it was already saved earlier", jobUrl + "api/json"); 147 | } 148 | } 149 | } else { 150 | //If no builds were found 151 | logger.debug("No builds were found for {}", jobUrl); 152 | } 153 | 154 | } 155 | return Optional.empty(); 156 | } 157 | 158 | @Override 159 | public List poll() throws InterruptedException { 160 | logger.debug("In poll()"); 161 | 162 | //Keep trying in a loop until stop() is called on this instance. 163 | //TODO use RxJava for this in future 164 | while (!stop.get()) { 165 | long now = time.milliseconds(); 166 | logger.trace("Now: {}", sdf.format(new Date(now))); 167 | long nextUpdate = 0; 168 | 169 | //Check if poll time had elapsed 170 | if (lastUpdate == 0) { 171 | logger.trace("First call after starting the connector. So Pulling the Jobs from Jenkins now."); 172 | nextUpdate = now; 173 | } else { 174 | nextUpdate = lastUpdate + pollIntervalInMillis; 175 | logger.trace("Next pull from Jenkins should happen at {} (approx).", nextUpdate); 176 | } 177 | long untilNext = nextUpdate - now; 178 | logger.debug("now: {}, nextUpdate: {}, untilNext: {}", sdf.format(new Date(now)), sdf.format(new Date(nextUpdate)), untilNext); 179 | 180 | if (untilNext > 0) { 181 | logger.info("Waiting {} ms before next pull", untilNext); 182 | time.sleep(untilNext); 183 | continue; 184 | } 185 | 186 | logger.debug("Total pulls from Jenkins so far: {}", totalJenkinsPulls); 187 | String jobUrls = taskProps.get(JOB_URLS); 188 | storageAdapter = new ReadYourWritesOffsetStorageAdapter(context.offsetStorageReader(), jobUrls, partitions); 189 | 190 | String[] jobUrlArray = jobUrls.split(","); 191 | 192 | List records = new ArrayList<>(); 193 | for (String jobUrl : jobUrlArray) { 194 | 195 | SourcePartition partition = partitions.make(urlDecode(extractJobName(jobUrl))); 196 | 197 | logger.trace("Get lastSavedOffset for: '{}' with partitionValue: {}", jobUrl, partition.value); 198 | Optional offset = storageAdapter.getOffset(partition); 199 | 200 | Long lastSavedBuildNumber = null; 201 | if (offset.isPresent()) { 202 | logger.debug("lastSavedOffset for '{}' is: {}", partition.value, offset.get()); 203 | lastSavedBuildNumber = (Long) offset.get().value; 204 | } else { 205 | logger.debug("lastSavedOffset not available for: {}", partition.value); 206 | } 207 | Optional sourceRecord = createSourceRecord(jobUrl, lastSavedBuildNumber); 208 | if (sourceRecord.isPresent()) records.add(sourceRecord.get()); 209 | } 210 | 211 | logger.info("Total SourceRecords created: {}. Returning these from poll()", records.size()); 212 | 213 | //Update the last updated time to now just before returning the call 214 | lastUpdate = time.milliseconds(); 215 | logger.debug("Setting the lastUpdate time to : {}", sdf.format(new Date(lastUpdate))); 216 | 217 | totalJenkinsPulls++; 218 | return records; 219 | } 220 | 221 | logger.debug("Returning null from poll(). This is because the runtime called shutdown."); 222 | //Only in case make shutdown. null indicates no data 223 | return null; 224 | } 225 | 226 | @Override 227 | public synchronized void stop() { 228 | logger.info("JenkinsSourceTask stopping"); 229 | if (stop != null) stop.set(true); 230 | } 231 | 232 | @Override 233 | public void initialize(SourceTaskContext context) { 234 | super.initialize(context); 235 | } 236 | 237 | private int getJenkinsReadTimeout() { 238 | return Integer.valueOf(taskProps.getOrDefault(JENKINS_READ_TIMEOUT_CONFIG, valueOf(JENKINS_READ_TIMEOUT_DEFAULT))); 239 | } 240 | 241 | private int getJenkinsConnTimeout() { 242 | return Integer.valueOf(taskProps.getOrDefault(JENKINS_CONN_TIMEOUT_CONFIG, valueOf(JENKINS_CONN_TIMEOUT_DEFAULT))); 243 | } 244 | 245 | private String getJenkinsUsername() { 246 | return String.valueOf(taskProps.getOrDefault(JENKINS_USERNAME_CONFIG, "")); 247 | } 248 | 249 | private String getJenkinsPassword() { 250 | return String.valueOf(taskProps.getOrDefault(JENKINS_PASSWORD_OR_API_TOKEN_CONFIG, "")); 251 | } 252 | 253 | } 254 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/jenkins/JobTaskConfigExtractor.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins; 2 | 3 | import org.aravind.oss.jenkins.domain.Job; 4 | import org.aravind.oss.kafka.connect.lib.TaskConfigExtractor; 5 | 6 | /** 7 | * @author Aravind R Yarram 8 | * @since 0.5.0 9 | */ 10 | public class JobTaskConfigExtractor implements TaskConfigExtractor { 11 | 12 | @Override 13 | public String extract(Job input) { 14 | return input.getUrl(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/jenkins/ReadYourWritesOffsetStorageAdapter.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins; 2 | 3 | import org.apache.kafka.connect.storage.OffsetStorageReader; 4 | import org.aravind.oss.kafka.connect.lib.SourceOffset; 5 | import org.aravind.oss.kafka.connect.lib.Partitions; 6 | import org.aravind.oss.kafka.connect.lib.SourcePartition; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.*; 11 | 12 | import static org.aravind.oss.kafka.connect.jenkins.Util.*; 13 | 14 | /** 15 | * Sources cache the offsets and periodically flush(or commit) them asynchronously. The following two properties 16 | * are used to control this behavior. 17 | *

18 | *

    19 | *
  • offset.flush.timeout.ms = 5000
    • 20 | *
  • offset.flush.interval.ms = 1000
    • 21 | *
22 | *

23 | * Currently the {@link org.apache.kafka.connect.storage.OffsetStorageReader} only reads the offsets that 24 | * have been flushed. The cached offsets are not returned. However, the read-your-writes consistency 25 | * model is needed to simplify source connector development. 26 | *

27 | *

28 | * Read-your-writes consistency: A value written by a process on a data item X will be always available to a 29 | * successive read operation performed by the same process on data item X. 30 | *

31 | *

32 | * This class provides the Read-your-Writes semantics within a single {@link org.apache.kafka.connect.source.SourceTask} 33 | * 34 | * @author Aravind R Yarram 35 | * @since 0.5.0 36 | */ 37 | public class ReadYourWritesOffsetStorageAdapter { 38 | 39 | private OffsetStorageReader storageReader; 40 | 41 | private final Partitions partitions; 42 | 43 | //task level cache 44 | private Map, Map> cache = new HashMap<>(); 45 | 46 | //task level offsets from StorageReader 47 | private Map, Map> offsets; 48 | 49 | private static final Logger logger = LoggerFactory.getLogger(ReadYourWritesOffsetStorageAdapter.class); 50 | 51 | public ReadYourWritesOffsetStorageAdapter(OffsetStorageReader reader, String jobUrls, Partitions ps) { 52 | storageReader = reader; 53 | partitions = ps; 54 | offsets = loadAndGetOffsets(storageReader, jobUrls); 55 | logger.debug("Loaded offsets: {}", offsets); 56 | } 57 | 58 | public void cache(SourcePartition p, SourceOffset o) { 59 | Map key = p.encoded; 60 | Map value = o.encoded; 61 | 62 | cache.put(key, value); 63 | } 64 | 65 | public boolean containsPartition(SourcePartition partition) { 66 | if (offsets.keySet().contains(partition.encoded)) { 67 | return true; 68 | } else { 69 | logger.trace("Didn't find the key {} in offset storage", partition.encoded); 70 | return cache.containsKey(partition.encoded); 71 | } 72 | } 73 | 74 | public Optional getOffset(SourcePartition partition) { 75 | if (offsets.get(partition.encoded) != null) { 76 | return Optional.of(SourceOffset.decode(offsets.get(partition.encoded))); 77 | } else { 78 | logger.trace("Didn't find the key {} in offset storage so trying from cache", partition.encoded); 79 | return Optional.ofNullable(SourceOffset.decode(cache.get(partition.encoded))); 80 | } 81 | } 82 | 83 | private Map, Map> loadAndGetOffsets(OffsetStorageReader reader, String jobUrls) { 84 | String[] jobUrlArray = jobUrls.split(","); 85 | 86 | logger.debug("Total jobs: {}. Loading offsets from Connect.", jobUrlArray.length); 87 | Collection> partitions = new ArrayList<>(jobUrlArray.length); 88 | for (String jobUrl : jobUrlArray) { 89 | partitions.add(Collections.singletonMap(JenkinsSourceTask.JOB_NAME, urlDecode(extractJobName(jobUrl)))); 90 | } 91 | return reader.offsets(partitions); 92 | } 93 | } -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/jenkins/Util.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.UnsupportedEncodingException; 7 | import java.net.URLDecoder; 8 | import java.nio.charset.Charset; 9 | 10 | /** 11 | * @author Aravind R Yarram 12 | * @since 0.5.0 13 | */ 14 | public class Util { 15 | private static final Logger logger = LoggerFactory.getLogger(Util.class); 16 | 17 | public static String extractJobName(String jobUrl) { 18 | //For input - https://builds.apache.org/job/Accumulo-Master/ 19 | //This method should return - Accumulo-Master 20 | String[] tokens = jobUrl.split("/"); 21 | return tokens[tokens.length - 1]; 22 | } 23 | 24 | public static String urlDecode(String jobName) { 25 | try { 26 | return URLDecoder.decode(jobName, String.valueOf(Charset.forName("UTF-8"))); 27 | } catch (UnsupportedEncodingException e) { 28 | logger.warn("Ignoring the error and returning the original job name. Unable to URLDecode {}", jobName); 29 | return jobName; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/jenkins/Version.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins; 2 | 3 | /** 4 | * @author Aravind R Yarram 5 | * @since 0.5.0 6 | */ 7 | public class Version { 8 | public static String get() { 9 | return "0.5.0"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/lib/Partitions.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.lib; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | /** 10 | * @author Aravind R Yarram 11 | * @since 0.5.0 12 | */ 13 | public class Partitions { 14 | private static final Map cache = new HashMap<>(); 15 | private final String partitionKey; 16 | private static final Logger logger = LoggerFactory.getLogger(Partitions.class); 17 | 18 | public Partitions(String key) { 19 | partitionKey = key; 20 | } 21 | 22 | /** 23 | * Has a side-effect make caching the partition for future if it doesn't already exist in the cache. 24 | * 25 | * @param partition Name (or value) make the partition 26 | * @return Cached {@link SourcePartition} 27 | */ 28 | public SourcePartition make(String partition) { 29 | if (!cache.containsKey(partition)) { 30 | logger.trace("Adding {} to cache.", partition); 31 | cache.put(partition, SourcePartition.make(partitionKey, partition)); 32 | } 33 | return cache.get(partition); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/lib/SourceOffset.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.lib; 2 | 3 | import java.util.Collections; 4 | import java.util.Map; 5 | import java.util.Set; 6 | 7 | /** 8 | * @author Aravind R Yarram 9 | * @since 0.5.0 10 | */ 11 | public class SourceOffset { 12 | public final String key; 13 | public final Object value; 14 | public final Map encoded; 15 | 16 | public static SourceOffset make(String k, Object v) { 17 | return new SourceOffset(k, v); 18 | } 19 | 20 | public static SourceOffset decode(Map in) { 21 | if (in != null) { 22 | Set> entries = in.entrySet(); 23 | assert entries.size() == 1; 24 | Map.Entry entry = entries.iterator().next(); 25 | return new SourceOffset(entry.getKey(), entry.getValue()); 26 | } 27 | return null; 28 | } 29 | 30 | private SourceOffset(String k, Object v) { 31 | key = k.intern(); 32 | value = v; 33 | encoded = Collections.singletonMap(key, value); 34 | } 35 | 36 | @Override 37 | public boolean equals(Object o) { 38 | if (this == o) return true; 39 | if (o == null || getClass() != o.getClass()) return false; 40 | 41 | SourceOffset offset = (SourceOffset) o; 42 | 43 | if (!key.equals(offset.key)) return false; 44 | return value.equals(offset.value); 45 | 46 | } 47 | 48 | @Override 49 | public int hashCode() { 50 | int result = key.hashCode(); 51 | result = 31 * result + value.hashCode(); 52 | return result; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return "SourceOffset{" + 58 | "key='" + key + '\'' + 59 | ", value=" + value + 60 | ", encoded=" + encoded + 61 | '}'; 62 | } 63 | } -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/lib/SourcePartition.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.lib; 2 | 3 | import java.util.Collections; 4 | import java.util.Map; 5 | 6 | /** 7 | * @author Aravind R Yarram 8 | * @since 0.5.0 9 | */ 10 | public class SourcePartition { 11 | public final String key; 12 | public final String value; 13 | public final Map encoded; 14 | 15 | public static SourcePartition make(String k, String v) { 16 | return new SourcePartition(k, v); 17 | } 18 | 19 | private SourcePartition(String k, String v) { 20 | key = k.intern(); 21 | value = v.intern(); 22 | encoded = Collections.singletonMap(key, value); 23 | } 24 | 25 | @Override 26 | public boolean equals(Object o) { 27 | if (this == o) return true; 28 | if (o == null || getClass() != o.getClass()) return false; 29 | 30 | SourcePartition that = (SourcePartition) o; 31 | 32 | if (!key.equals(that.key)) return false; 33 | return value.equals(that.value); 34 | 35 | } 36 | 37 | @Override 38 | public int hashCode() { 39 | int result = key.hashCode(); 40 | result = 31 * result + value.hashCode(); 41 | return result; 42 | } 43 | } -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/lib/TaskConfigBuilder.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.lib; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.connect.source.SourceTask; 5 | import org.apache.kafka.connect.util.ConnectorUtils; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.ArrayList; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.stream.Collectors; 14 | 15 | /** 16 | * This class encapsulates the following common behavior of building task configurations 17 | *

18 | *

    19 | *
  • Group all the work into multiple smaller work groups.
    • 20 | *
  • For each smaller work group, creates a task config that will be handed over to a single {@link SourceTask}
    • 21 | *
  • Each task config is encoded as a comma separated string. This is the simpler convention followed by few other connectors as well. The {@link SourceTask} is expected to split this String with comma delimiter.
    • 22 | *
  • Adds the connector specific configuration to task config so that it can be forwarded to {@link SourceTask}
    • 23 | *
24 | * 25 | * @author Aravind R Yarram 26 | * @since 0.5.0 27 | */ 28 | public class TaskConfigBuilder { 29 | private final int maxTasks; 30 | private final AbstractConfig connectorCfg; 31 | private final TaskConfigExtractor taskCfgExtractor; 32 | private final String taskCfgKey; 33 | 34 | private static Logger logger = LoggerFactory.getLogger(TaskConfigBuilder.class); 35 | 36 | /** 37 | * @param maxTasks The maximum simultaneous tasks configured to be run by connector. 38 | * @param taskConfigKey The key to be used while adding the task config. 39 | * @param connectorCfg The reference to the custom config class you wrote extending from {@link AbstractConfig}. Null if not applicable. 40 | * @param taskConfigExtractor The class used to extract the task configuration. This is used as the value for key taskConfigKey. 41 | */ 42 | public TaskConfigBuilder(int maxTasks, String taskConfigKey, AbstractConfig connectorCfg, TaskConfigExtractor taskConfigExtractor) { 43 | this.maxTasks = maxTasks; 44 | this.connectorCfg = connectorCfg; 45 | this.taskCfgExtractor = taskConfigExtractor; 46 | this.taskCfgKey = taskConfigKey; 47 | } 48 | 49 | /** 50 | * Builds task config for each task. 51 | * 52 | * @param work The entire work expected to be fulfilled by this connector 53 | * @return The task configuration for each of the tasks 54 | */ 55 | public List> build(List work) { 56 | //Create job groups 57 | int workSizeCount = work.size(); 58 | int numGroups = Math.min(workSizeCount, maxTasks); 59 | 60 | logger.debug("Total work size: {} units. maxTasks: {}. numGroups: {}.", workSizeCount, maxTasks, numGroups); 61 | 62 | List> workGroups = ConnectorUtils.groupPartitions(work, numGroups); 63 | logger.debug("Number of work groups created: {}.", workGroups.size()); 64 | 65 | //Create task configs for each group 66 | List> taskConfigs = new ArrayList<>(workGroups.size()); 67 | 68 | for (List group : workGroups) { 69 | //Forward the config from connector to SourceTask so that they have access to the configured TOPIC NAME 70 | Map taskProps = new HashMap<>(connectorCfg.originalsStrings()); 71 | 72 | //Concatenate all job urls that need to be handled by a single task 73 | String commaSeparatedJobUrls = group.stream() 74 | .map(j -> taskCfgExtractor.extract(j)) 75 | .collect(Collectors.joining(",")); 76 | 77 | taskProps.put(taskCfgKey, commaSeparatedJobUrls); 78 | logger.trace("taskProps: {}", taskProps); 79 | taskConfigs.add(taskProps); 80 | } 81 | return taskConfigs; 82 | } 83 | } -------------------------------------------------------------------------------- /src/main/java/org/aravind/oss/kafka/connect/lib/TaskConfigExtractor.java: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.lib; 2 | 3 | /** 4 | * @author Aravind R Yarram 5 | * @since 0.5.0 6 | */ 7 | public interface TaskConfigExtractor { 8 | public String extract(T input); 9 | } -------------------------------------------------------------------------------- /src/site/resources/images/jenkins-resource-relationships.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaravind/kafka-connect-jenkins/1e000473d8317646cdd977392a12545dac10f5eb/src/site/resources/images/jenkins-resource-relationships.png -------------------------------------------------------------------------------- /src/test/groovy/org/aravind/oss/jenkins/JenkinsClientTest.groovy: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins 2 | 3 | import com.github.dreamhead.moco.HttpsCertificate 4 | import com.github.dreamhead.moco.Runner 5 | import spock.lang.Ignore 6 | import spock.lang.Shared 7 | import spock.lang.Specification 8 | 9 | import static com.github.dreamhead.moco.Moco.and 10 | import static com.github.dreamhead.moco.Moco.by 11 | import static com.github.dreamhead.moco.Moco.exist 12 | import static com.github.dreamhead.moco.Moco.header 13 | import static com.github.dreamhead.moco.Moco.httpsServer 14 | import static com.github.dreamhead.moco.Moco.log 15 | import static com.github.dreamhead.moco.Moco.uri 16 | import static com.github.dreamhead.moco.MocoJsonRunner.jsonHttpServer 17 | import static com.github.dreamhead.moco.Runner.runner 18 | import static com.github.dreamhead.moco.Moco.pathResource 19 | 20 | /** 21 | * @author Aravind R Yarram 22 | * @since 0.5.0 23 | */ 24 | class JenkinsClientTest extends Specification { 25 | 26 | public static final int CONN_TIMEOUT = 100 27 | public static final int READ_TIMEOUT = 500 28 | @Shared 29 | Runner mock 30 | 31 | @Shared 32 | Runner httpsMock 33 | 34 | def setupSpec() { 35 | def server = jsonHttpServer(9191, pathResource("jenkins-mock-server-cfg.json")) 36 | mock = runner(server) 37 | mock.start() 38 | 39 | def httpsServer = httpsServer(9443, HttpsCertificate.certificate(pathResource("keystore.jks"), "password", "password")) 40 | httpsMock = runner(httpsServer) 41 | httpsMock.start() 42 | } 43 | 44 | //Error scenarios 45 | 46 | def "Wrong authentication credentials 'username' should return empty response"() { 47 | given: "Wrong username and correct password" 48 | def server = jsonHttpServer(9495, pathResource("jenkins-mock-server-with-authuentication-cfg.json")) 49 | mock = runner(server) 50 | mock.start() 51 | 52 | def userName = "wronguser" 53 | def password = "password" 54 | 55 | when: "A request is submitted" 56 | def url = new URL("http://localhost:9495/api/json") 57 | def jenkins = new JenkinsClient(url, userName, password, CONN_TIMEOUT, READ_TIMEOUT) 58 | def response = jenkins.get() 59 | 60 | then: "response should be empty and an exception is swallowed but logged" 61 | response.isPresent() == false 62 | noExceptionThrown() 63 | 64 | mock.stop() 65 | } 66 | 67 | def "Wrong authentication credentials 'passwordOrApiToken' should return empty response"() { 68 | given: "Wrong username and correct password" 69 | def server = jsonHttpServer(9495, pathResource("jenkins-mock-server-with-authuentication-cfg.json")) 70 | mock = runner(server) 71 | mock.start() 72 | 73 | def userName = "user" 74 | def password = "wrongpassword" 75 | 76 | when: "A request is submitted" 77 | def url = new URL("http://localhost:9495/api/json") 78 | def jenkins = new JenkinsClient(url, userName, password, CONN_TIMEOUT, READ_TIMEOUT) 79 | def response = jenkins.get() 80 | 81 | then: "response should be empty and an exception is swallowed but logged" 82 | response.isPresent() == false 83 | noExceptionThrown() 84 | 85 | mock.stop() 86 | } 87 | 88 | //Happy scenarios 89 | 90 | def "Supports Jenkins without authentication"() { 91 | when: 92 | def url = new URL("http://localhost:9191/") 93 | def jenkins = new JenkinsClient(url, CONN_TIMEOUT, READ_TIMEOUT) 94 | def response = jenkins.get() 95 | 96 | then: 97 | noExceptionThrown() 98 | } 99 | 100 | def "Supports GET"() { 101 | given: 102 | def url = new URL("http://localhost:9191/api/json") 103 | def jenkins = new JenkinsClient(url, CONN_TIMEOUT, READ_TIMEOUT) 104 | 105 | when: 106 | def allJobs = jenkins.get() 107 | 108 | then: 109 | allJobs != null 110 | allJobs.isPresent() == true 111 | } 112 | 113 | def "Supports Jenkins with Basic Authentication"() { 114 | given: "Correct username and password" 115 | def server = jsonHttpServer(9494, pathResource("jenkins-mock-server-with-authuentication-cfg.json")) 116 | mock = runner(server) 117 | mock.start() 118 | 119 | def userName = "user" 120 | def password = "password" 121 | 122 | when: "A request is submitted" 123 | def url = new URL("http://localhost:9494/api/json") 124 | def jenkins = new JenkinsClient(url, userName, password, CONN_TIMEOUT, READ_TIMEOUT) 125 | def response = jenkins.get() 126 | 127 | then: "A valid response should be present" 128 | response.isPresent() == true 129 | 130 | mock.stop() 131 | } 132 | 133 | @Ignore("Ignore until we figure out how to mock https on Snap-CI") 134 | def "Supports Jenkins with SSL"() { 135 | when: 136 | def url = new URL("https://localhost:9443") 137 | def jenkins = new JenkinsClient(url, CONN_TIMEOUT, READ_TIMEOUT) 138 | def response = jenkins.get() 139 | 140 | then: 141 | noExceptionThrown() 142 | } 143 | 144 | @Ignore("Ignore until we figure out how to mock https + basic auth with java api") 145 | def "Supports Jenkins with SSL and Basic Authentication"() { 146 | given: "Correct username and password" 147 | 148 | def httpsServer = httpsServer(10443, HttpsCertificate.certificate(pathResource("keystore.jks"), "password", "password")) 149 | httpsServer.request( 150 | exist(header("Authorization", "Basic dXNlcjpwYXNzd29yZA==")) 151 | ) 152 | .response(pathResource("single-job.json")) 153 | mock = runner(httpsServer) 154 | mock.start() 155 | 156 | def userName = "user" 157 | def password = "password" 158 | 159 | when: 160 | def url = new URL("https://localhost:10443") 161 | def jenkins = new JenkinsClient(url, userName, password, CONN_TIMEOUT, READ_TIMEOUT) 162 | def response = jenkins.get() 163 | 164 | then: "A valid response should be present" 165 | response.isPresent() == true 166 | 167 | mock.stop() 168 | } 169 | 170 | def cleanupSpec() { 171 | if (mock != null) mock.stop() 172 | if (httpsMock != null) httpsMock.stop(); 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/test/groovy/org/aravind/oss/jenkins/domain/BuildTest.groovy: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.jenkins.domain 2 | 3 | import spock.lang.Shared 4 | import spock.lang.Specification 5 | 6 | import static com.github.dreamhead.moco.Moco.pathResource 7 | import static com.github.dreamhead.moco.MocoJsonRunner.jsonHttpServer 8 | import static com.github.dreamhead.moco.Runner.runner 9 | 10 | /** 11 | * @author Aravind R Yarram 12 | * @since 0.5.0 13 | */ 14 | class BuildTest extends Specification { 15 | @Shared 16 | def mock 17 | 18 | def setupSpec() { 19 | def server = jsonHttpServer(1081, pathResource("BuildTest-mock-server-cfg.json")) 20 | mock = runner(server) 21 | mock.start() 22 | } 23 | 24 | def cleanupSpec() { 25 | mock.stop() 26 | } 27 | 28 | def "getDetails"() { 29 | given: 30 | def build = new Build() 31 | build.setNumber(2546) 32 | build.setUrl('http://localhost:1081/job/Abdera-trunk/2546/') 33 | 34 | when: 35 | Optional details = build.getDetails() 36 | 37 | then: 38 | build.getBuildDetailsResource() == 'http://localhost:1081/job/Abdera-trunk/2546/api/json' 39 | details.isPresent() == true 40 | } 41 | 42 | def "getDetails - Error condition"() { 43 | given: "A wrong port number 10" 44 | def build = new Build() 45 | build.setNumber(2546) 46 | build.setUrl('http://localhost:10/job/Abdera-trunk/0000/') 47 | 48 | when: "Asked for the Build details" 49 | Optional details = build.getDetails() 50 | 51 | then: "An exception is logged and Optional.empty() is returned" 52 | build.getBuildDetailsResource() == 'http://localhost:10/job/Abdera-trunk/0000/api/json' 53 | details.isPresent() == false 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/groovy/org/aravind/oss/kafka/connect/jenkins/JenkinsSourceConfigTest.groovy: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins 2 | 3 | import org.apache.kafka.common.config.ConfigException 4 | import spock.lang.Shared 5 | import spock.lang.Specification 6 | 7 | /** 8 | * @author Aravind R Yarram 9 | * @since 0.5.0 10 | */ 11 | class JenkinsSourceConfigTest extends Specification { 12 | 13 | @Shared 14 | def Map cfg 15 | 16 | def setupSpec() { 17 | cfg = ['jenkins.base.url' : 'https://builds.apache.org', 18 | 'jenkins.username' : 'myuser', 19 | 'jenkins.password.or.api.token' : 'mypassword', 20 | 'jenkins.connection.timeoutInMillis': '1000', 21 | 'jenkins.read.timeoutInMillis' : '5000', 22 | 'jenkins.jobs.resource.path' : '/api/json'] 23 | } 24 | 25 | //Error scenarios 26 | 27 | def "Missing 'jenkins.base.url' property should throw exception"() { 28 | when: 29 | def props = [:] 30 | def jenkinsCfg = new JenkinsSourceConfig(props) 31 | 32 | then: 33 | thrown(ConfigException) 34 | } 35 | 36 | def "Not starting the 'jenkins.jobs.resource.path' property value with '/' should throw an exception"() { 37 | when: 38 | def props = ['client.jobs.resource.path': 'api/json'] 39 | def jenkinsCfg = new JenkinsSourceConfig(props) 40 | 41 | then: 42 | thrown(ConfigException) 43 | } 44 | 45 | def "GetJenkinsUrl should throw Exception with invalid URL"() { 46 | given: "Given invalid url" 47 | def props = ['jenkins.base.url': 'invalid url'] 48 | def jenkinsCfg = new JenkinsSourceConfig(props) 49 | 50 | when: 51 | jenkinsCfg.getJenkinsUrl() 52 | 53 | then: 54 | thrown(ConfigException) 55 | } 56 | 57 | def "GetJobsResource should throw Exception with invalid URL"() { 58 | given: "Given invalid url" 59 | def props = ['jenkins.base.url': 'invalid url'] 60 | def jenkinsCfg = new JenkinsSourceConfig(props) 61 | 62 | when: 63 | jenkinsCfg.getJobsResource() 64 | 65 | then: 66 | thrown(ConfigException) 67 | } 68 | 69 | //Happy scenarios 70 | 71 | def "'jenkins.base.url' is the only REQUIRED property"() { 72 | when: 73 | def props = ['jenkins.base.url': 'https://builds.apache.org'] 74 | def jenkinsCfg = new JenkinsSourceConfig(props) 75 | 76 | then: 77 | notThrown(ConfigException) 78 | } 79 | 80 | //Happy scenarios - Defaults 81 | 82 | def "Defaults - '/api/json' is chosen when 'jenkins.jobs.resource.path' property is not specified"() { 83 | when: 84 | def props = ['jenkins.base.url': 'https://builds.apache.org'] 85 | def jenkinsCfg = new JenkinsSourceConfig(props) 86 | 87 | then: 88 | jenkinsCfg.getJobsResource() == new URL('https://builds.apache.org/api/json') 89 | } 90 | 91 | def "Defaults - '500' millis is chosen when 'jenkins.connection.timeoutInMillis' property is not specified"() { 92 | when: 93 | def props = ['jenkins.base.url': 'https://builds.apache.org'] 94 | def jenkinsCfg = new JenkinsSourceConfig(props) 95 | 96 | then: 97 | jenkinsCfg.getJenkinsConnTimeout() == 500 98 | } 99 | 100 | def "Defaults - '3000' is chosen when 'jenkins.read.timeoutInMillis' property is not specified"() { 101 | when: 102 | def props = ['jenkins.base.url': 'https://builds.apache.org'] 103 | def jenkinsCfg = new JenkinsSourceConfig(props) 104 | 105 | then: 106 | jenkinsCfg.getJenkinsReadTimeout() == 3000 107 | } 108 | 109 | //Happy scenarios - Helper methods 110 | 111 | def "GetJenkinsUrl"() { 112 | when: 113 | def jenkinsCfg = new JenkinsSourceConfig(cfg) 114 | 115 | then: "an URL object is returned" 116 | jenkinsCfg.getJenkinsUrl() == new URL('https://builds.apache.org') 117 | } 118 | 119 | def "GetUsername"() { 120 | when: 121 | def jenkinsCfg = new JenkinsSourceConfig(cfg) 122 | 123 | then: 124 | jenkinsCfg.getUsername() == 'myuser' 125 | } 126 | 127 | def "GetPasswordOrApiToken"() { 128 | when: 129 | def jenkinsCfg = new JenkinsSourceConfig(cfg) 130 | 131 | then: 132 | jenkinsCfg.getPasswordOrApiToken() == 'mypassword' 133 | } 134 | 135 | def "IsProtected"() { 136 | when: 137 | def jenkinsCfg = new JenkinsSourceConfig(cfg) 138 | 139 | then: 140 | jenkinsCfg.isProtected() == true 141 | } 142 | 143 | def "GetJobsResource"() { 144 | when: 145 | def jenkinsCfg = new JenkinsSourceConfig(cfg) 146 | 147 | then: 148 | jenkinsCfg.getJobsResource() == new URL('https://builds.apache.org/api/json') 149 | } 150 | 151 | def "GetJenkinsConnTimeout"() { 152 | when: 153 | def jenkinsCfg = new JenkinsSourceConfig(cfg) 154 | 155 | then: 156 | jenkinsCfg.getJenkinsConnTimeout() == 1000 157 | } 158 | 159 | def "GetJenkinsReadTimeout"() { 160 | when: 161 | def jenkinsCfg = new JenkinsSourceConfig(cfg) 162 | 163 | then: 164 | jenkinsCfg.getJenkinsReadTimeout() == 5000 165 | } 166 | } -------------------------------------------------------------------------------- /src/test/groovy/org/aravind/oss/kafka/connect/jenkins/JenkinsSourceConnectorTest.groovy: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins 2 | 3 | import org.apache.kafka.connect.connector.ConnectorContext 4 | import org.apache.kafka.connect.errors.ConnectException 5 | import spock.lang.Shared 6 | import spock.lang.Specification 7 | 8 | import static com.github.dreamhead.moco.Moco.pathResource 9 | import static com.github.dreamhead.moco.MocoJsonRunner.jsonHttpServer 10 | import static com.github.dreamhead.moco.Runner.runner 11 | 12 | /** 13 | * @author Aravind R Yarram 14 | * @since 0.5.0 15 | */ 16 | class JenkinsSourceConnectorTest extends Specification { 17 | @Shared 18 | ConnectorContext context = Mock() 19 | 20 | Map sourceProps = [:] 21 | JenkinsSourceConnector connector 22 | 23 | def setup() { 24 | connector = new JenkinsSourceConnector(); 25 | connector.initialize(context) 26 | } 27 | 28 | def cleanup() { 29 | if (connector != null) connector.stop() 30 | } 31 | 32 | //API Contracts 33 | 34 | def "taskClass()"() { 35 | when: 36 | Class taskClass = connector.taskClass() 37 | 38 | then: 39 | taskClass == JenkinsSourceTask.class 40 | } 41 | 42 | //Error scenarios 43 | 44 | def "Config - Wrong jenkins url should throw exception"() { 45 | when: 46 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://wrong.domain.name') 47 | connector.start(sourceProps) 48 | 49 | then: 50 | thrown(ConnectException) 51 | } 52 | 53 | //Happy scenarios - Config 54 | 55 | def "Config - Correct config should start the connector"() { 56 | given: 57 | def server = jsonHttpServer(8181, pathResource("jenkins-mock-server-cfg.json")) 58 | def mock = runner(server) 59 | mock.start() 60 | 61 | when: 62 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:8181') 63 | connector.start(sourceProps) 64 | 65 | then: 66 | noExceptionThrown() 67 | 68 | mock.stop() 69 | } 70 | 71 | def "Config - The 'topic' property should be forwarded to each task"() { 72 | given: 73 | def server = jsonHttpServer(9295, pathResource("jenkins-mock-server-single-job-cfg.json")) 74 | def mock = runner(server) 75 | mock.start() 76 | 77 | def MAX_TASKS_IS_THREE = 3 78 | 79 | when: 80 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9295') 81 | sourceProps.put(JenkinsSourceConfig.TOPIC_CONFIG, 'test.topic.name') 82 | connector.start(sourceProps) 83 | 84 | def taskCfgs = connector.taskConfigs(MAX_TASKS_IS_THREE) 85 | 86 | then: 87 | Map taskProps = taskCfgs.get(0) 88 | taskProps != null 89 | taskProps[JenkinsSourceConfig.TOPIC_CONFIG] == 'test.topic.name' 90 | 91 | mock.stop() 92 | } 93 | 94 | //Happy scenarios - Defaults 95 | 96 | def "Defaults - 'jenkins.jobs.resource.path' config is optional"() { 97 | given: 98 | def server = jsonHttpServer(9296, pathResource("jenkins-mock-server-single-job-cfg.json")) 99 | def mock = runner(server) 100 | mock.start() 101 | 102 | when: 103 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9296') 104 | connector.start(sourceProps) 105 | 106 | then: "Default value is used" 107 | connector.getJenkinsCfg().getString(JenkinsSourceConfig.JOBS_RESOURCE_PATH_CONFIG) == '/api/json' 108 | } 109 | 110 | def "Defaults - 'topic' config is optional"() { 111 | given: 112 | def server = jsonHttpServer(9297, pathResource("jenkins-mock-server-single-job-cfg.json")) 113 | def mock = runner(server) 114 | mock.start() 115 | 116 | when: 117 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9297') 118 | connector.start(sourceProps) 119 | 120 | then: "Default value is used" 121 | connector.getJenkinsCfg().getString(JenkinsSourceConfig.TOPIC_CONFIG) == 'jenkins.connector.topic' 122 | } 123 | 124 | //Happy scenarios - Partitioning 125 | 126 | def "Partitioning - Single job"() { 127 | given: 128 | def server = jsonHttpServer(9292, pathResource("jenkins-mock-server-single-job-cfg.json")) 129 | def mock = runner(server) 130 | mock.start() 131 | 132 | def MAX_TASKS_IS_ONE = 1 133 | 134 | when: 135 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9292') 136 | connector.start(sourceProps) 137 | 138 | def taskCfgs = connector.taskConfigs(MAX_TASKS_IS_ONE) 139 | 140 | then: 141 | taskCfgs.size() == MAX_TASKS_IS_ONE 142 | 143 | Map taspProps = taskCfgs.get(0) 144 | taspProps != null 145 | taspProps[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Abdera-trunk/' 146 | 147 | mock.stop() 148 | } 149 | 150 | def "Partitioning - Multiple jobs"() { 151 | given: 152 | def server = jsonHttpServer(9293, pathResource("jenkins-mock-server-three-job-cfg.json")) 153 | def mock = runner(server) 154 | mock.start() 155 | 156 | def MAX_TASKS_IS_ONE = 1 157 | 158 | when: 159 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9293') 160 | connector.start(sourceProps) 161 | 162 | def taskCfgs = connector.taskConfigs(MAX_TASKS_IS_ONE) 163 | 164 | then: 165 | taskCfgs.size() == MAX_TASKS_IS_ONE 166 | 167 | Map taskProps = taskCfgs.get(0) 168 | taskProps != null 169 | taskProps[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Abdera-trunk/,https://builds.apache.org/job/Accumulo-1.8/,https://builds.apache.org/job/Allura/' 170 | 171 | mock.stop() 172 | } 173 | 174 | def "Partitioning - Multiple jobs with 2 MAX_TASKS"() { 175 | given: 176 | def server = jsonHttpServer(9293, pathResource("jenkins-mock-server-three-job-cfg.json")) 177 | def mock = runner(server) 178 | mock.start() 179 | 180 | def MAX_TASKS_IS_TWO = 2 181 | 182 | when: 183 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9293') 184 | connector.start(sourceProps) 185 | 186 | def taskCfgs = connector.taskConfigs(MAX_TASKS_IS_TWO) 187 | 188 | then: 189 | taskCfgs.size() == MAX_TASKS_IS_TWO 190 | 191 | Map task1Props = taskCfgs.get(0) 192 | task1Props != null 193 | task1Props[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Abdera-trunk/,https://builds.apache.org/job/Accumulo-1.8/' 194 | 195 | Map task2Props = taskCfgs.get(1) 196 | task2Props != null 197 | task2Props[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Allura/' 198 | 199 | mock.stop() 200 | } 201 | 202 | def "Partitioning - Multiple jobs with 3 MAX_TASKS"() { 203 | given: 204 | def server = jsonHttpServer(9293, pathResource("jenkins-mock-server-three-job-cfg.json")) 205 | def mock = runner(server) 206 | mock.start() 207 | 208 | def MAX_TASKS_IS_THREE = 3 209 | 210 | when: 211 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9293') 212 | connector.start(sourceProps) 213 | 214 | def taskCfgs = connector.taskConfigs(MAX_TASKS_IS_THREE) 215 | 216 | then: 217 | taskCfgs.size() == MAX_TASKS_IS_THREE 218 | 219 | Map task1Props = taskCfgs.get(0) 220 | task1Props != null 221 | task1Props[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Abdera-trunk/' 222 | 223 | Map task2Props = taskCfgs.get(1) 224 | task2Props != null 225 | task2Props[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Accumulo-1.8/' 226 | 227 | Map task3Props = taskCfgs.get(2) 228 | task3Props != null 229 | task3Props[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Allura/' 230 | 231 | mock.stop() 232 | } 233 | 234 | def "Partitioning - Less jobs and more MAX_TASKS"() { 235 | given: 236 | def server = jsonHttpServer(9294, pathResource("jenkins-mock-server-single-job-cfg.json")) 237 | def mock = runner(server) 238 | mock.start() 239 | 240 | def MAX_TASKS_IS_THREE = 3 241 | 242 | when: 243 | sourceProps.put(JenkinsSourceConfig.JENKINS_BASE_URL_CONFIG, 'http://localhost:9294') 244 | connector.start(sourceProps) 245 | 246 | def taskCfgs = connector.taskConfigs(MAX_TASKS_IS_THREE) 247 | 248 | then: 249 | taskCfgs.size() == 1 250 | 251 | Map taskProps = taskCfgs.get(0) 252 | taskProps != null 253 | taskProps[JenkinsSourceTask.JOB_URLS] == 'https://builds.apache.org/job/Abdera-trunk/' 254 | 255 | mock.stop() 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/test/groovy/org/aravind/oss/kafka/connect/jenkins/JenkinsSourceTaskTest.groovy: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins 2 | 3 | import org.apache.kafka.common.utils.Time 4 | import org.apache.kafka.connect.source.SourceTaskContext 5 | import org.apache.kafka.connect.storage.OffsetStorageReader 6 | import spock.lang.Shared 7 | import spock.lang.Specification 8 | 9 | import static com.github.dreamhead.moco.Moco.pathResource 10 | import static com.github.dreamhead.moco.MocoJsonRunner.jsonHttpServer 11 | import static com.github.dreamhead.moco.Runner.runner 12 | 13 | /** 14 | * @author Aravind R Yarram 15 | * @since 0.5.0 16 | */ 17 | class JenkinsSourceTaskTest extends Specification { 18 | @Shared 19 | SourceTaskContext taskContext = Mock() 20 | 21 | @Shared 22 | OffsetStorageReader storageReader = Mock() 23 | 24 | @Shared 25 | def mock 26 | 27 | JenkinsSourceTask sourceTask 28 | 29 | def setupSpec() { 30 | storageReader.offsets(_) >> [:] 31 | taskContext.offsetStorageReader() >> storageReader 32 | 33 | def server = jsonHttpServer(8181, pathResource("JenkinsSourceTaskTest-mock-server-cfg.json")) 34 | mock = runner(server) 35 | mock.start() 36 | } 37 | 38 | def cleanupSpec() { 39 | mock.stop() 40 | } 41 | 42 | def setup() { 43 | sourceTask = new JenkinsSourceTask() 44 | sourceTask.initialize(taskContext) 45 | } 46 | 47 | def cleanup() { 48 | if (sourceTask != null) sourceTask.stop() 49 | } 50 | 51 | def "Should support single job url as taskProps"() { 52 | given: 53 | def taskProps = ['job.urls': 'http://localhost:8181/job/Abdera-trunk/', 'jenkins.pollIntervalInMillis': '0'] 54 | sourceTask.start(taskProps) 55 | 56 | when: 57 | def sourceRecords = sourceTask.poll() 58 | 59 | then: 60 | sourceRecords != null 61 | sourceRecords.size() == 1 62 | } 63 | 64 | def "Should support multiple comma separated job urls as taskProps"() { 65 | given: 66 | def taskProps = ['job.urls': 'http://localhost:8181/job/Abdera-trunk/,http://localhost:8181/job/Accumulo-1.8/', 'jenkins.pollIntervalInMillis': '0'] 67 | sourceTask.start(taskProps) 68 | 69 | when: 70 | def sourceRecords = sourceTask.poll() 71 | 72 | then: 73 | sourceRecords != null 74 | sourceRecords.size() == 2 75 | } 76 | 77 | //Negative tests 78 | 79 | def "Wrong URL should continue without any errors"() { 80 | given: "A wrong job url" 81 | def taskProps = ['job.urls': 'http://wrong.host.name:8181/job/Abdera-trunk/', 'jenkins.pollIntervalInMillis': '0'] 82 | sourceTask.start(taskProps) 83 | 84 | when: 85 | def sourceRecords = sourceTask.poll() 86 | sourceTask.stop() 87 | 88 | then: "Exception is logged and ignored and Source records will be null" 89 | sourceRecords != null 90 | } 91 | 92 | def "A Job without any builds (May be just created) should continue without any errors"() { 93 | given: "A job with no lastBuild" 94 | def taskProps = ['job.urls': 'http://localhost:8181/job/New-Job/', 'jenkins.pollIntervalInMillis': '0'] 95 | sourceTask.start(taskProps) 96 | 97 | when: 98 | def sourceRecords = sourceTask.poll() 99 | sourceTask.stop() 100 | 101 | then: "Exception is logged and ignored and Source records will be null" 102 | sourceRecords != null 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/test/groovy/org/aravind/oss/kafka/connect/jenkins/ReadYourWritesOffsetStorageAdapterTest.groovy: -------------------------------------------------------------------------------- 1 | package org.aravind.oss.kafka.connect.jenkins 2 | 3 | import org.apache.kafka.connect.storage.OffsetStorageReader 4 | import org.aravind.oss.kafka.connect.lib.SourceOffset 5 | import org.aravind.oss.kafka.connect.lib.Partitions 6 | import org.aravind.oss.kafka.connect.lib.SourcePartition 7 | import spock.lang.Shared 8 | import spock.lang.Specification 9 | 10 | /** 11 | * @author Aravind R Yarram 12 | * @since 0.5.0 13 | */ 14 | class ReadYourWritesOffsetStorageAdapterTest extends Specification { 15 | 16 | @Shared 17 | def Partitions partitions = new Partitions(JenkinsSourceTask.JOB_NAME) 18 | 19 | @Shared 20 | def OffsetStorageReader storageReader = Mock() 21 | 22 | @Shared 23 | def ReadYourWritesOffsetStorageAdapter adapter 24 | 25 | @Shared 26 | def Map, Map> offsets = [:] 27 | 28 | def setupSpec() { 29 | offsets.put(['jobName': 'Abdera-Trunk'], ['buildNumber': '72']) 30 | offsets.put(['jobName': 'Hadoop-Trunk'], ['buildNumber': '14']) 31 | 32 | storageReader.offsets(_) >> offsets 33 | adapter = new ReadYourWritesOffsetStorageAdapter(storageReader, 'Abdera-Trunk', partitions) 34 | } 35 | 36 | def "containsPartition"() { 37 | when: 38 | def result = adapter.containsPartition(partitions.make('Abdera-Trunk')) 39 | 40 | then: 41 | result == true 42 | } 43 | 44 | def "getOffset - kay exists in offsets read from OffsetStorageReader"() { 45 | when: 46 | Optional result = adapter.getOffset(partitions.make('Abdera-Trunk')) 47 | 48 | then: 49 | result.get().encoded == ['buildNumber': '72'] 50 | } 51 | 52 | def "getOffset - if key doesn't exist in offsets then read from cache"() { 53 | given: "the current offsets aren't yet flushed" 54 | SourcePartition p = partitions.make('NonExistingJob-Trunk') 55 | Optional result = adapter.getOffset(p) 56 | result.isPresent() == false 57 | 58 | when: "we create a new SourceRecord for the new build" 59 | String partitionValue = 'NonExistingJob-Trunk' 60 | Object offsetValue = '272' 61 | 62 | adapter.cache(p, SourceOffset.make(JenkinsSourceTask.BUILD_NUMBER, offsetValue)) 63 | 64 | then: 65 | adapter.containsPartition(p) == true 66 | adapter.getOffset(p).isPresent() == true 67 | } 68 | 69 | def "getOffset - should return no result for non existing partitions"() { 70 | when: 71 | Optional result = adapter.getOffset(partitions.make('NonExistingJob1-Trunk')) 72 | 73 | then: 74 | result.isPresent() == false 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/test/resources/BuildTest-mock-server-cfg.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "request": { 4 | "uri": "/job/Abdera-trunk/api/json" 5 | }, 6 | "response": { 7 | "status": 200, 8 | "file": "src/test/resources/abdera-trunk-builds.json", 9 | "headers": { 10 | "Content-Type": "application/json;charset=UTF-8", 11 | "X-Jenkins": "1.642.2" 12 | } 13 | } 14 | }, 15 | { 16 | "request": { 17 | "uri": "/job/Abdera-trunk/2546/api/json" 18 | }, 19 | "response": { 20 | "status": 200, 21 | "file": "src/test/resources/abdera-trunk-2546-build-details.json", 22 | "headers": { 23 | "Content-Type": "application/json;charset=UTF-8", 24 | "X-Jenkins": "1.642.2" 25 | } 26 | } 27 | } 28 | ] -------------------------------------------------------------------------------- /src/test/resources/JenkinsSourceTaskTest-mock-server-cfg.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "request": { 4 | "uri": "/job/Abdera-trunk/api/json" 5 | }, 6 | "response": { 7 | "status": 200, 8 | "file": "src/test/resources/abdera-trunk-builds.json", 9 | "headers": { 10 | "Content-Type": "application/json;charset=UTF-8", 11 | "X-Jenkins": "1.642.2" 12 | } 13 | } 14 | }, 15 | { 16 | "request": { 17 | "uri": "/job/Abdera-trunk/2546/api/json" 18 | }, 19 | "response": { 20 | "status": 200, 21 | "file": "src/test/resources/abdera-trunk-2546-build-details.json", 22 | "headers": { 23 | "Content-Type": "application/json;charset=UTF-8", 24 | "X-Jenkins": "1.642.2" 25 | } 26 | } 27 | }, 28 | { 29 | "request": { 30 | "uri": "/job/Accumulo-1.8/api/json" 31 | }, 32 | "response": { 33 | "status": 200, 34 | "file": "src/test/resources/accumulo-trunk-builds.json", 35 | "headers": { 36 | "Content-Type": "application/json;charset=UTF-8", 37 | "X-Jenkins": "1.642.2" 38 | } 39 | } 40 | }, 41 | { 42 | "request": { 43 | "uri": "/job/Accumulo-1.8/18/api/json" 44 | }, 45 | "response": { 46 | "status": 200, 47 | "file": "src/test/resources/accumulo-trunk-18-build-details.json", 48 | "headers": { 49 | "Content-Type": "application/json;charset=UTF-8", 50 | "X-Jenkins": "1.642.2" 51 | } 52 | } 53 | }, 54 | { 55 | "request": { 56 | "uri": "/job/New-Job/api/json" 57 | }, 58 | "response": { 59 | "status": 200, 60 | "file": "src/test/resources/new-job-with-no-builds.json", 61 | "headers": { 62 | "Content-Type": "application/json;charset=UTF-8", 63 | "X-Jenkins": "1.642.2" 64 | } 65 | } 66 | } 67 | ] -------------------------------------------------------------------------------- /src/test/resources/abdera-trunk-2546-build-details.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [ 3 | { 4 | "causes": [ 5 | { 6 | "shortDescription": "Started by an SCM change" 7 | } 8 | ] 9 | }, 10 | {}, 11 | {}, 12 | {}, 13 | {}, 14 | {}, 15 | {}, 16 | { 17 | "failCount": 0, 18 | "skipCount": 1, 19 | "totalCount": 491, 20 | "urlName": "testReport" 21 | }, 22 | {}, 23 | {} 24 | ], 25 | "artifacts": [], 26 | "building": false, 27 | "description": null, 28 | "displayName": "#2546", 29 | "duration": 480582, 30 | "estimatedDuration": 457794, 31 | "executor": null, 32 | "fullDisplayName": "Abdera-trunk #2546", 33 | "id": "2015-08-25_22-08-43", 34 | "keepLog": false, 35 | "number": 2546, 36 | "queueId": -1, 37 | "result": "SUCCESS", 38 | "timestamp": 1440540523000, 39 | "url": "https://builds.apache.org/job/Abdera-trunk/2546/", 40 | "builtOn": "jenkins-ubuntu-1404-4gb-c51", 41 | "changeSet": { 42 | "items": [ 43 | { 44 | "affectedPaths": [ 45 | "client/src/test/java/org/apache/abdera/test/client/util/MultipartRelatedRequestEntityTest.java" 46 | ], 47 | "author": { 48 | "absoluteUrl": "https://builds.apache.org/user/veithen", 49 | "fullName": "veithen" 50 | }, 51 | "commitId": "1697770", 52 | "timestamp": 1440537458260, 53 | "date": "2015-08-25T21:17:38.260515Z", 54 | "msg": "Use factory to create FOMEntry instance.", 55 | "paths": [ 56 | { 57 | "editType": "edit", 58 | "file": "/abdera/java/trunk/client/src/test/java/org/apache/abdera/test/client/util/MultipartRelatedRequestEntityTest.java" 59 | } 60 | ], 61 | "revision": 1697770, 62 | "user": "veithen" 63 | } 64 | ], 65 | "kind": "svn", 66 | "revisions": [ 67 | { 68 | "module": "https://svn.apache.org/repos/asf/abdera/java/trunk", 69 | "revision": 1697770 70 | } 71 | ] 72 | }, 73 | "culprits": [ 74 | { 75 | "absoluteUrl": "https://builds.apache.org/user/veithen", 76 | "fullName": "veithen" 77 | } 78 | ], 79 | "mavenArtifacts": {}, 80 | "mavenVersionUsed": "3.0.4" 81 | } -------------------------------------------------------------------------------- /src/test/resources/abdera-trunk-builds.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [ 3 | {}, 4 | {}, 5 | {}, 6 | null, 7 | {}, 8 | {}, 9 | {} 10 | ], 11 | "description": "Apache Abdera Trunk", 12 | "displayName": "Abdera-trunk", 13 | "displayNameOrNull": null, 14 | "name": "Abdera-trunk", 15 | "url": "http://localhost:8181/job/Abdera-trunk/", 16 | "buildable": true, 17 | "builds": [ 18 | { 19 | "number": 2546, 20 | "url": "http://localhost:8181/job/Abdera-trunk/2546/" 21 | }, 22 | { 23 | "number": 2545, 24 | "url": "http://localhost:8181/job/Abdera-trunk/2545/" 25 | }, 26 | { 27 | "number": 2544, 28 | "url": "http://localhost:8181/job/Abdera-trunk/2544/" 29 | } 30 | ], 31 | "color": "blue", 32 | "firstBuild": { 33 | "number": 2544, 34 | "url": "http://localhost:8181/job/Abdera-trunk/2544/" 35 | }, 36 | "healthReport": [ 37 | { 38 | "description": "Test Result: 0 tests failing out of a total of 491 tests.", 39 | "iconClassName": "icon-health-80plus", 40 | "iconUrl": "health-80plus.png", 41 | "score": 100 42 | }, 43 | { 44 | "description": "Build stability: No recent builds failed.", 45 | "iconClassName": "icon-health-80plus", 46 | "iconUrl": "health-80plus.png", 47 | "score": 100 48 | } 49 | ], 50 | "inQueue": false, 51 | "keepDependencies": false, 52 | "lastBuild": { 53 | "number": 2546, 54 | "url": "http://localhost:8181/job/Abdera-trunk/2546/" 55 | }, 56 | "lastCompletedBuild": { 57 | "number": 2546, 58 | "url": "http://localhost:8181/job/Abdera-trunk/2546/" 59 | }, 60 | "lastFailedBuild": null, 61 | "lastStableBuild": { 62 | "number": 2546, 63 | "url": "http://localhost:8181/job/Abdera-trunk/2546/" 64 | }, 65 | "lastSuccessfulBuild": { 66 | "number": 2546, 67 | "url": "http://localhost:8181/job/Abdera-trunk/2546/" 68 | }, 69 | "lastUnstableBuild": null, 70 | "lastUnsuccessfulBuild": null, 71 | "nextBuildNumber": 2547, 72 | "property": [ 73 | {}, 74 | {}, 75 | {} 76 | ], 77 | "queueItem": null, 78 | "concurrentBuild": false, 79 | "downstreamProjects": [], 80 | "scm": {}, 81 | "upstreamProjects": [], 82 | "modules": [ 83 | { 84 | "name": "org.apache.abdera:abdera", 85 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera/", 86 | "color": "blue", 87 | "displayName": "Apache Abdera" 88 | }, 89 | { 90 | "name": "org.apache.abdera:abdera-adapters-parent", 91 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-adapters-parent/", 92 | "color": "blue", 93 | "displayName": "Abdera Adapters" 94 | }, 95 | { 96 | "name": "org.apache.abdera:abdera-bundle", 97 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-bundle/", 98 | "color": "blue", 99 | "displayName": "Abdera Bundle Jar" 100 | }, 101 | { 102 | "name": "org.apache.abdera:abdera-client", 103 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-client/", 104 | "color": "blue", 105 | "displayName": "Abdera Client" 106 | }, 107 | { 108 | "name": "org.apache.abdera:abdera-core", 109 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-core/", 110 | "color": "blue", 111 | "displayName": "Abdera Core" 112 | }, 113 | { 114 | "name": "org.apache.abdera:abdera-examples", 115 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-examples/", 116 | "color": "blue", 117 | "displayName": "Abdera Examples" 118 | }, 119 | { 120 | "name": "org.apache.abdera:abdera-extensions-features", 121 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-features/", 122 | "color": "blue", 123 | "displayName": "Abdera Extensions - Features" 124 | }, 125 | { 126 | "name": "org.apache.abdera:abdera-extensions-gdata", 127 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-gdata/", 128 | "color": "blue", 129 | "displayName": "Abdera Extensions - GData" 130 | }, 131 | { 132 | "name": "org.apache.abdera:abdera-extensions-geo", 133 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-geo/", 134 | "color": "blue", 135 | "displayName": "Abdera Extensions - Geo" 136 | }, 137 | { 138 | "name": "org.apache.abdera:abdera-extensions-html", 139 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-html/", 140 | "color": "blue", 141 | "displayName": "Abdera Extensions - HTML" 142 | }, 143 | { 144 | "name": "org.apache.abdera:abdera-extensions-json", 145 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-json/", 146 | "color": "blue", 147 | "displayName": "Abdera Extensions - JSON" 148 | }, 149 | { 150 | "name": "org.apache.abdera:abdera-extensions-main", 151 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-main/", 152 | "color": "blue", 153 | "displayName": "Abdera Extensions - Main" 154 | }, 155 | { 156 | "name": "org.apache.abdera:abdera-extensions-media", 157 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-media/", 158 | "color": "blue", 159 | "displayName": "Abdera Extensions - Media" 160 | }, 161 | { 162 | "name": "org.apache.abdera:abdera-extensions-oauth", 163 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-oauth/", 164 | "color": "blue", 165 | "displayName": "Abdera Extensions - OAuth" 166 | }, 167 | { 168 | "name": "org.apache.abdera:abdera-extensions-opensearch", 169 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-opensearch/", 170 | "color": "blue", 171 | "displayName": "Abdera Extensions - OpenSearch" 172 | }, 173 | { 174 | "name": "org.apache.abdera:abdera-extensions-parent", 175 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-parent/", 176 | "color": "blue", 177 | "displayName": "Abdera Extensions" 178 | }, 179 | { 180 | "name": "org.apache.abdera:abdera-extensions-rss", 181 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-rss/", 182 | "color": "blue", 183 | "displayName": "Abdera Extensions - RSS" 184 | }, 185 | { 186 | "name": "org.apache.abdera:abdera-extensions-serializer", 187 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-serializer/", 188 | "color": "blue", 189 | "displayName": "Abdera Extensions - Serializer" 190 | }, 191 | { 192 | "name": "org.apache.abdera:abdera-extensions-sharing", 193 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-sharing/", 194 | "color": "blue", 195 | "displayName": "Abdera Extensions - Sharing" 196 | }, 197 | { 198 | "name": "org.apache.abdera:abdera-extensions-wsse", 199 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-extensions-wsse/", 200 | "color": "blue", 201 | "displayName": "Abdera Extensions - WSSE" 202 | }, 203 | { 204 | "name": "org.apache.abdera:abdera-filesystem", 205 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-filesystem/", 206 | "color": "blue", 207 | "displayName": "Abdera Filesystem Adapter" 208 | }, 209 | { 210 | "name": "org.apache.abdera:abdera-i18n", 211 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-i18n/", 212 | "color": "blue", 213 | "displayName": "I18N Libraries" 214 | }, 215 | { 216 | "name": "org.apache.abdera:abdera-jcr", 217 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-jcr/", 218 | "color": "blue", 219 | "displayName": "Abdera JCR Adapter" 220 | }, 221 | { 222 | "name": "org.apache.abdera:abdera-jdbc", 223 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-jdbc/", 224 | "color": "blue", 225 | "displayName": "Abdera JDBC Adapter" 226 | }, 227 | { 228 | "name": "org.apache.abdera:abdera-parser", 229 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-parser/", 230 | "color": "blue", 231 | "displayName": "Abdera Parser" 232 | }, 233 | { 234 | "name": "org.apache.abdera:abdera-security", 235 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-security/", 236 | "color": "blue", 237 | "displayName": "Abdera Security" 238 | }, 239 | { 240 | "name": "org.apache.abdera:abdera-server", 241 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-server/", 242 | "color": "blue", 243 | "displayName": "Abdera Server" 244 | }, 245 | { 246 | "name": "org.apache.abdera:abdera-spring", 247 | "url": "http://localhost:8181/job/Abdera-trunk/org.apache.abdera$abdera-spring/", 248 | "color": "blue", 249 | "displayName": "Abdera Spring Integration" 250 | } 251 | ] 252 | } -------------------------------------------------------------------------------- /src/test/resources/accumulo-trunk-18-build-details.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [ 3 | { 4 | "causes": [ 5 | { 6 | "shortDescription": "Started by an SCM change" 7 | } 8 | ] 9 | }, 10 | {}, 11 | {}, 12 | {}, 13 | { 14 | "buildsByBranchName": { 15 | "origin/1.8": { 16 | "buildNumber": 18, 17 | "buildResult": null, 18 | "marked": { 19 | "SHA1": "85ff374f30d1e0ce9351d496b2300d4d6a2c013d", 20 | "branch": [ 21 | { 22 | "SHA1": "85ff374f30d1e0ce9351d496b2300d4d6a2c013d", 23 | "name": "origin/1.8" 24 | } 25 | ] 26 | }, 27 | "revision": { 28 | "SHA1": "85ff374f30d1e0ce9351d496b2300d4d6a2c013d", 29 | "branch": [ 30 | { 31 | "SHA1": "85ff374f30d1e0ce9351d496b2300d4d6a2c013d", 32 | "name": "origin/1.8" 33 | } 34 | ] 35 | } 36 | } 37 | }, 38 | "lastBuiltRevision": { 39 | "SHA1": "85ff374f30d1e0ce9351d496b2300d4d6a2c013d", 40 | "branch": [ 41 | { 42 | "SHA1": "85ff374f30d1e0ce9351d496b2300d4d6a2c013d", 43 | "name": "origin/1.8" 44 | } 45 | ] 46 | }, 47 | "remoteUrls": [ 48 | "git://git.apache.org/accumulo.git" 49 | ], 50 | "scmName": "" 51 | }, 52 | {}, 53 | {}, 54 | {}, 55 | { 56 | "failCount": 0, 57 | "skipCount": 7, 58 | "totalCount": 1507, 59 | "urlName": "testReport" 60 | }, 61 | {}, 62 | {}, 63 | {}, 64 | {} 65 | ], 66 | "artifacts": [], 67 | "building": false, 68 | "description": null, 69 | "displayName": "#18", 70 | "duration": 1845465, 71 | "estimatedDuration": 1884518, 72 | "executor": null, 73 | "fullDisplayName": "Accumulo-1.8 #18", 74 | "id": "18", 75 | "keepLog": false, 76 | "number": 18, 77 | "queueId": 34155, 78 | "result": "SUCCESS", 79 | "timestamp": 1464384308581, 80 | "url": "https://builds.apache.org/job/Accumulo-1.8/18/", 81 | "builtOn": "ubuntu-2", 82 | "changeSet": { 83 | "items": [ 84 | { 85 | "affectedPaths": [ 86 | "test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java", 87 | "test/src/test/java/org/apache/accumulo/test/fate/zookeeper/ZooLockTest.java" 88 | ], 89 | "commitId": "5b971d675c8a395f6bc2924285b094d973e39ebe", 90 | "timestamp": 1464375995000, 91 | "author": { 92 | "absoluteUrl": "https://builds.apache.org/user/ctubbsii", 93 | "fullName": "ctubbsii" 94 | }, 95 | "comment": "ACCUMULO-4312 Fix jar sealing problem\nMove ZooLockTest.java to another package so it doesn't conflict with the \npackage in the fate module (needed because the jars are sealed).\n", 96 | "date": "2016-05-27 15:06:35 -0400", 97 | "id": "5b971d675c8a395f6bc2924285b094d973e39ebe", 98 | "msg": "ACCUMULO-4312 Fix jar sealing problem", 99 | "paths": [ 100 | { 101 | "editType": "add", 102 | "file": "test/src/test/java/org/apache/accumulo/test/fate/zookeeper/ZooLockTest.java" 103 | }, 104 | { 105 | "editType": "delete", 106 | "file": "test/src/test/java/org/apache/accumulo/fate/zookeeper/ZooLockTest.java" 107 | } 108 | ] 109 | }, 110 | { 111 | "affectedPaths": [ 112 | "test/src/test/java/org/apache/accumulo/proxy/TestProxyInstanceOperations.java", 113 | "server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java", 114 | "test/src/test/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java", 115 | "test/src/test/java/org/apache/accumulo/proxy/TestProxySecurityOperations.java", 116 | "test/src/test/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java", 117 | "test/src/test/java/org/apache/accumulo/proxy/TJsonProtocolProxyIT.java", 118 | "test/src/test/java/org/apache/accumulo/test/proxy/TBinaryProxyIT.java", 119 | "server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java", 120 | "test/src/test/java/org/apache/accumulo/proxy/TestProxyTableOperations.java", 121 | "test/src/test/java/org/apache/accumulo/proxy/TTupleProxyIT.java", 122 | "test/src/test/java/org/apache/accumulo/test/proxy/TCompactProxyIT.java", 123 | "test/src/test/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java", 124 | "test/src/test/java/org/apache/accumulo/proxy/TBinaryProxyIT.java", 125 | "test/src/test/java/org/apache/accumulo/test/proxy/TTupleProxyIT.java", 126 | "test/src/test/java/org/apache/accumulo/proxy/SimpleProxyBase.java", 127 | "test/src/test/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java", 128 | "server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java", 129 | "test/src/test/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java", 130 | "test/src/test/java/org/apache/accumulo/test/proxy/SimpleProxyBase.java", 131 | "test/src/test/java/org/apache/accumulo/test/proxy/TJsonProtocolProxyIT.java", 132 | "test/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java", 133 | "test/src/test/java/org/apache/accumulo/proxy/TCompactProxyIT.java", 134 | "test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java" 135 | ], 136 | "commitId": "d10840b8e5177f1f026f28705f93943a1921ad82", 137 | "timestamp": 1464379175000, 138 | "author": { 139 | "absoluteUrl": "https://builds.apache.org/user/ctubbsii", 140 | "fullName": "ctubbsii" 141 | }, 142 | "comment": "ACCUMULO-4312 Fix additional sealing problems in 1.7 branch\n", 143 | "date": "2016-05-27 15:59:35 -0400", 144 | "id": "d10840b8e5177f1f026f28705f93943a1921ad82", 145 | "msg": "ACCUMULO-4312 Fix additional sealing problems in 1.7 branch", 146 | "paths": [ 147 | { 148 | "editType": "add", 149 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TestProxyInstanceOperations.java" 150 | }, 151 | { 152 | "editType": "delete", 153 | "file": "test/src/test/java/org/apache/accumulo/proxy/TBinaryProxyIT.java" 154 | }, 155 | { 156 | "editType": "delete", 157 | "file": "test/src/test/java/org/apache/accumulo/proxy/TCompactProxyIT.java" 158 | }, 159 | { 160 | "editType": "edit", 161 | "file": "server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java" 162 | }, 163 | { 164 | "editType": "delete", 165 | "file": "server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java" 166 | }, 167 | { 168 | "editType": "delete", 169 | "file": "test/src/test/java/org/apache/accumulo/proxy/SimpleProxyBase.java" 170 | }, 171 | { 172 | "editType": "delete", 173 | "file": "test/src/test/java/org/apache/accumulo/proxy/TestProxyTableOperations.java" 174 | }, 175 | { 176 | "editType": "add", 177 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TBinaryProxyIT.java" 178 | }, 179 | { 180 | "editType": "add", 181 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TTupleProxyIT.java" 182 | }, 183 | { 184 | "editType": "delete", 185 | "file": "test/src/test/java/org/apache/accumulo/proxy/TestProxyInstanceOperations.java" 186 | }, 187 | { 188 | "editType": "add", 189 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TestProxyTableOperations.java" 190 | }, 191 | { 192 | "editType": "add", 193 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TestProxySecurityOperations.java" 194 | }, 195 | { 196 | "editType": "delete", 197 | "file": "test/src/test/java/org/apache/accumulo/proxy/TTupleProxyIT.java" 198 | }, 199 | { 200 | "editType": "add", 201 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TCompactProxyIT.java" 202 | }, 203 | { 204 | "editType": "add", 205 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TestProxyReadWrite.java" 206 | }, 207 | { 208 | "editType": "delete", 209 | "file": "test/src/test/java/org/apache/accumulo/proxy/TestProxyReadWrite.java" 210 | }, 211 | { 212 | "editType": "add", 213 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java" 214 | }, 215 | { 216 | "editType": "delete", 217 | "file": "test/src/test/java/org/apache/accumulo/proxy/TJsonProtocolProxyIT.java" 218 | }, 219 | { 220 | "editType": "delete", 221 | "file": "test/src/test/java/org/apache/accumulo/proxy/TestProxySecurityOperations.java" 222 | }, 223 | { 224 | "editType": "add", 225 | "file": "server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java" 226 | }, 227 | { 228 | "editType": "add", 229 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/SimpleProxyBase.java" 230 | }, 231 | { 232 | "editType": "add", 233 | "file": "test/src/test/java/org/apache/accumulo/test/proxy/TJsonProtocolProxyIT.java" 234 | }, 235 | { 236 | "editType": "delete", 237 | "file": "test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java" 238 | } 239 | ] 240 | } 241 | ], 242 | "kind": "git" 243 | }, 244 | "culprits": [ 245 | { 246 | "absoluteUrl": "https://builds.apache.org/user/ctubbsii", 247 | "fullName": "ctubbsii" 248 | } 249 | ], 250 | "mavenArtifacts": {}, 251 | "mavenVersionUsed": "3.3.3" 252 | } -------------------------------------------------------------------------------- /src/test/resources/accumulo-trunk-builds.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [ 3 | {}, 4 | {}, 5 | {}, 6 | {}, 7 | null, 8 | {}, 9 | {}, 10 | {}, 11 | {}, 12 | {} 13 | ], 14 | "description": "Builds the Accumulo 1.8 branch. Includes findbugs and checkstyle invocation. Publishes binaries, source and javadoc jars.", 15 | "displayName": "Accumulo-1.8", 16 | "displayNameOrNull": null, 17 | "name": "Accumulo-1.8", 18 | "url": "http://localhost:8181/job/Accumulo-1.8/", 19 | "buildable": true, 20 | "builds": [ 21 | { 22 | "number": 18, 23 | "url": "http://localhost:8181/job/Accumulo-1.8/18/" 24 | }, 25 | { 26 | "number": 17, 27 | "url": "http://localhost:8181/job/Accumulo-1.8/17/" 28 | }, 29 | { 30 | "number": 16, 31 | "url": "http://localhost:8181/job/Accumulo-1.8/16/" 32 | }, 33 | { 34 | "number": 15, 35 | "url": "http://localhost:8181/job/Accumulo-1.8/15/" 36 | }, 37 | { 38 | "number": 14, 39 | "url": "http://localhost:8181/job/Accumulo-1.8/14/" 40 | } 41 | ], 42 | "color": "blue", 43 | "firstBuild": { 44 | "number": 14, 45 | "url": "http://localhost:8181/job/Accumulo-1.8/14/" 46 | }, 47 | "healthReport": [ 48 | { 49 | "description": "Build stability: 2 out of the last 5 builds failed.", 50 | "iconClassName": "icon-health-40to59", 51 | "iconUrl": "health-40to59.png", 52 | "score": 60 53 | }, 54 | { 55 | "description": "Test Result: 0 tests failing out of a total of 1,507 tests.", 56 | "iconClassName": "icon-health-80plus", 57 | "iconUrl": "health-80plus.png", 58 | "score": 100 59 | } 60 | ], 61 | "inQueue": false, 62 | "keepDependencies": false, 63 | "lastBuild": { 64 | "number": 18, 65 | "url": "http://localhost:8181/job/Accumulo-1.8/18/" 66 | }, 67 | "lastCompletedBuild": { 68 | "number": 18, 69 | "url": "http://localhost:8181/job/Accumulo-1.8/18/" 70 | }, 71 | "lastFailedBuild": { 72 | "number": 15, 73 | "url": "http://localhost:8181/job/Accumulo-1.8/15/" 74 | }, 75 | "lastStableBuild": { 76 | "number": 18, 77 | "url": "http://localhost:8181/job/Accumulo-1.8/18/" 78 | }, 79 | "lastSuccessfulBuild": { 80 | "number": 18, 81 | "url": "http://localhost:8181/job/Accumulo-1.8/18/" 82 | }, 83 | "lastUnstableBuild": null, 84 | "lastUnsuccessfulBuild": { 85 | "number": 15, 86 | "url": "http://localhost:8181/job/Accumulo-1.8/15/" 87 | }, 88 | "nextBuildNumber": 19, 89 | "property": [ 90 | {}, 91 | {}, 92 | {}, 93 | {} 94 | ], 95 | "queueItem": null, 96 | "concurrentBuild": false, 97 | "downstreamProjects": [], 98 | "scm": {}, 99 | "upstreamProjects": [], 100 | "modules": [ 101 | { 102 | "name": "org.apache.accumulo:accumulo", 103 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo/", 104 | "color": "blue", 105 | "displayName": "Apache Accumulo" 106 | }, 107 | { 108 | "name": "org.apache.accumulo:accumulo-core", 109 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-core/", 110 | "color": "blue", 111 | "displayName": "Apache Accumulo Core" 112 | }, 113 | { 114 | "name": "org.apache.accumulo:accumulo-docs", 115 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-docs/", 116 | "color": "blue", 117 | "displayName": "Apache Accumulo Documentation" 118 | }, 119 | { 120 | "name": "org.apache.accumulo:accumulo-examples-simple", 121 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-examples-simple/", 122 | "color": "blue", 123 | "displayName": "Apache Accumulo Simple Examples" 124 | }, 125 | { 126 | "name": "org.apache.accumulo:accumulo-fate", 127 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-fate/", 128 | "color": "blue", 129 | "displayName": "Apache Accumulo Fate" 130 | }, 131 | { 132 | "name": "org.apache.accumulo:accumulo-gc", 133 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-gc/", 134 | "color": "blue", 135 | "displayName": "Apache Accumulo GC Server" 136 | }, 137 | { 138 | "name": "org.apache.accumulo:accumulo-iterator-test-harness", 139 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-iterator-test-harness/", 140 | "color": "blue", 141 | "displayName": "Apache Accumulo Iterator Test Harness" 142 | }, 143 | { 144 | "name": "org.apache.accumulo:accumulo-master", 145 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-master/", 146 | "color": "blue", 147 | "displayName": "Apache Accumulo Master Server" 148 | }, 149 | { 150 | "name": "org.apache.accumulo:accumulo-maven-plugin", 151 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-maven-plugin/", 152 | "color": "blue", 153 | "displayName": "Apache Accumulo Maven Plugin" 154 | }, 155 | { 156 | "name": "org.apache.accumulo:accumulo-minicluster", 157 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-minicluster/", 158 | "color": "blue", 159 | "displayName": "Apache Accumulo MiniCluster" 160 | }, 161 | { 162 | "name": "org.apache.accumulo:accumulo-monitor", 163 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-monitor/", 164 | "color": "blue", 165 | "displayName": "Apache Accumulo Monitor Server" 166 | }, 167 | { 168 | "name": "org.apache.accumulo:accumulo-native", 169 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-native/", 170 | "color": "blue", 171 | "displayName": "Apache Accumulo Native Libraries" 172 | }, 173 | { 174 | "name": "org.apache.accumulo:accumulo-project", 175 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-project/", 176 | "color": "blue", 177 | "displayName": "Apache Accumulo Project" 178 | }, 179 | { 180 | "name": "org.apache.accumulo:accumulo-proxy", 181 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-proxy/", 182 | "color": "blue", 183 | "displayName": "Apache Accumulo Proxy" 184 | }, 185 | { 186 | "name": "org.apache.accumulo:accumulo-server-base", 187 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-server-base/", 188 | "color": "blue", 189 | "displayName": "Apache Accumulo Server Base" 190 | }, 191 | { 192 | "name": "org.apache.accumulo:accumulo-shell", 193 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-shell/", 194 | "color": "blue", 195 | "displayName": "Apache Accumulo Shell" 196 | }, 197 | { 198 | "name": "org.apache.accumulo:accumulo-start", 199 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-start/", 200 | "color": "blue", 201 | "displayName": "Apache Accumulo Start" 202 | }, 203 | { 204 | "name": "org.apache.accumulo:accumulo-test", 205 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-test/", 206 | "color": "blue", 207 | "displayName": "Apache Accumulo Testing" 208 | }, 209 | { 210 | "name": "org.apache.accumulo:accumulo-trace", 211 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-trace/", 212 | "color": "blue", 213 | "displayName": "Apache Accumulo Trace" 214 | }, 215 | { 216 | "name": "org.apache.accumulo:accumulo-tracer", 217 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-tracer/", 218 | "color": "blue", 219 | "displayName": "Apache Accumulo Tracer Server" 220 | }, 221 | { 222 | "name": "org.apache.accumulo:accumulo-tserver", 223 | "url": "http://localhost:8181/job/Accumulo-1.8/org.apache.accumulo$accumulo-tserver/", 224 | "color": "blue", 225 | "displayName": "Apache Accumulo Tablet Server" 226 | } 227 | ] 228 | } -------------------------------------------------------------------------------- /src/test/resources/cert.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaravind/kafka-connect-jenkins/1e000473d8317646cdd977392a12545dac10f5eb/src/test/resources/cert.jks -------------------------------------------------------------------------------- /src/test/resources/jenkins-mock-server-cfg.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "request": { 4 | "uri": "/api/json" 5 | }, 6 | "response": { 7 | "status": 200, 8 | "file": "src/test/resources/all-jobs.json", 9 | "headers": { 10 | "Content-Type": "application/json;charset=UTF-8", 11 | "X-Jenkins": "1.642.2" 12 | } 13 | } 14 | } 15 | ] -------------------------------------------------------------------------------- /src/test/resources/jenkins-mock-server-single-job-cfg.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "request": { 4 | "uri": "/api/json" 5 | }, 6 | "response": { 7 | "status": 200, 8 | "file": "src/test/resources/single-job.json", 9 | "headers": { 10 | "Content-Type": "application/json;charset=UTF-8", 11 | "X-Jenkins": "1.642.2" 12 | } 13 | } 14 | } 15 | ] -------------------------------------------------------------------------------- /src/test/resources/jenkins-mock-server-three-job-cfg.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "request": { 4 | "uri": "/api/json" 5 | }, 6 | "response": { 7 | "status": 200, 8 | "file": "src/test/resources/three-jobs.json", 9 | "headers": { 10 | "Content-Type": "application/json;charset=UTF-8", 11 | "X-Jenkins": "1.642.2" 12 | } 13 | } 14 | } 15 | ] -------------------------------------------------------------------------------- /src/test/resources/jenkins-mock-server-with-authuentication-cfg.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "request": { 4 | "uri": "/api/json", 5 | "headers": { 6 | "Authorization": "Basic dXNlcjpwYXNzd29yZA==" 7 | } 8 | }, 9 | "response": { 10 | "status": 200, 11 | "file": "src/test/resources/all-jobs.json", 12 | "headers": { 13 | "Content-Type": "application/json;charset=UTF-8", 14 | "X-Jenkins": "1.642.2" 15 | } 16 | } 17 | } 18 | ] -------------------------------------------------------------------------------- /src/test/resources/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaravind/kafka-connect-jenkins/1e000473d8317646cdd977392a12545dac10f5eb/src/test/resources/keystore.jks -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | # Direct log messages to stdout 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.Target=System.out 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 8 | log4j.logger.org.aravind.oss.kafka.connect.jenkins=DEBUG 9 | log4j.logger.org.aravind.oss=ERROR 10 | log4j.logger.com.github.dreamhead.moco=ERROR -------------------------------------------------------------------------------- /src/test/resources/moco-cert.cer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaravind/kafka-connect-jenkins/1e000473d8317646cdd977392a12545dac10f5eb/src/test/resources/moco-cert.cer -------------------------------------------------------------------------------- /src/test/resources/new-job-with-no-builds.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [ 3 | {}, 4 | {} 5 | ], 6 | "description": "", 7 | "displayName": "archetype", 8 | "displayNameOrNull": null, 9 | "name": "archetype", 10 | "url": "http://localhost:8181/jenkins/job/New-Job/", 11 | "buildable": true, 12 | "builds": [], 13 | "color": "notbuilt", 14 | "firstBuild": null, 15 | "healthReport": [], 16 | "inQueue": false, 17 | "keepDependencies": false, 18 | "lastBuild": null, 19 | "lastCompletedBuild": null, 20 | "lastFailedBuild": null, 21 | "lastStableBuild": null, 22 | "lastSuccessfulBuild": null, 23 | "lastUnstableBuild": null, 24 | "lastUnsuccessfulBuild": null, 25 | "nextBuildNumber": 24, 26 | "property": [], 27 | "queueItem": null, 28 | "concurrentBuild": false, 29 | "downstreamProjects": [], 30 | "scm": {}, 31 | "upstreamProjects": [] 32 | } -------------------------------------------------------------------------------- /src/test/resources/single-job.json: -------------------------------------------------------------------------------- 1 | { 2 | "assignedLabels": [ 3 | {} 4 | ], 5 | "mode": "EXCLUSIVE", 6 | "nodeDescription": "the master Jenkins node", 7 | "nodeName": "", 8 | "numExecutors": 0, 9 | "description": "\r\n

\r\nThis is a public build and test server for projects of the\r\nApache Software Foundation. All times on this server are UTC.\r\n

\r\n

\r\nSee the Jenkins wiki page for more information\r\nabout this service.\r\n

", 10 | "jobs": [ 11 | { 12 | "name": "Abdera-trunk", 13 | "url": "https://builds.apache.org/job/Abdera-trunk/", 14 | "color": "blue" 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /src/test/resources/three-jobs.json: -------------------------------------------------------------------------------- 1 | { 2 | "assignedLabels": [ 3 | {} 4 | ], 5 | "mode": "EXCLUSIVE", 6 | "nodeDescription": "the master Jenkins node", 7 | "nodeName": "", 8 | "numExecutors": 0, 9 | "description": "\r\n

\r\nThis is a public build and test server for projects of the\r\nApache Software Foundation. All times on this server are UTC.\r\n

\r\n

\r\nSee the Jenkins wiki page for more information\r\nabout this service.\r\n

", 10 | "jobs": [ 11 | { 12 | "name": "Abdera-trunk", 13 | "url": "https://builds.apache.org/job/Abdera-trunk/", 14 | "color": "blue" 15 | }, 16 | { 17 | "name": "Accumulo-1.8", 18 | "url": "https://builds.apache.org/job/Accumulo-1.8/", 19 | "color": "blue" 20 | }, 21 | { 22 | "name": "Allura", 23 | "url": "https://builds.apache.org/job/Allura/", 24 | "color": "blue" 25 | } 26 | ] 27 | } --------------------------------------------------------------------------------