├── .classpath ├── .dir-locals.el ├── .gitignore ├── .project ├── .settings ├── org.eclipse.jdt.core.prefs ├── org.eclipse.jdt.ui.prefs └── org.eclipse.m2e.core.prefs ├── .travis.yml ├── LICENSE ├── README.md ├── assembly.xml ├── doc └── example.png ├── pom.xml └── src ├── main ├── java │ └── org │ │ └── pentaho │ │ └── di │ │ ├── trans │ │ └── kafka │ │ │ └── consumer │ │ │ ├── KafkaConsumer.java │ │ │ ├── KafkaConsumerCallable.java │ │ │ ├── KafkaConsumerData.java │ │ │ ├── KafkaConsumerMeta.java │ │ │ └── Messages.java │ │ └── ui │ │ └── trans │ │ └── kafka │ │ └── consumer │ │ └── KafkaConsumerDialog.java └── resources │ ├── org │ └── pentaho │ │ └── di │ │ └── trans │ │ └── kafka │ │ └── consumer │ │ ├── messages │ │ └── messages_en_US.properties │ │ └── resources │ │ └── kafka_consumer.png │ └── version.xml └── test └── java └── org └── pentaho └── di └── trans └── kafka └── consumer ├── KafkaConsumerDataTest.java ├── KafkaConsumerMetaTest.java └── KafkaConsumerTest.java /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((indent-tabs-mode . t) 2 | (tab-width . 4) 3 | (c-basic-offset . 4) 4 | (fill-column . 80)))) 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.jar 3 | *.war 4 | *.ear 5 | /target 6 | /TAGS 7 | 8 | # IntelliJ files 9 | .idea/ 10 | *.iml 11 | 12 | .DS_Store 13 | 14 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | pentaho-kafka-consumer 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 4 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 5 | org.eclipse.jdt.core.compiler.compliance=1.6 6 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 7 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 8 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 9 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 10 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 11 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 12 | org.eclipse.jdt.core.compiler.source=1.6 13 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | cleanup_settings_version=2 2 | eclipse.preferences.version=1 3 | editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true 4 | formatter_settings_version=12 5 | sp_cleanup.add_default_serial_version_id=true 6 | sp_cleanup.add_generated_serial_version_id=false 7 | sp_cleanup.add_missing_annotations=false 8 | sp_cleanup.add_missing_deprecated_annotations=true 9 | sp_cleanup.add_missing_methods=false 10 | sp_cleanup.add_missing_nls_tags=false 11 | sp_cleanup.add_missing_override_annotations=true 12 | sp_cleanup.add_missing_override_annotations_interface_methods=true 13 | sp_cleanup.add_serial_version_id=false 14 | sp_cleanup.always_use_blocks=true 15 | sp_cleanup.always_use_parentheses_in_expressions=false 16 | sp_cleanup.always_use_this_for_non_static_field_access=false 17 | sp_cleanup.always_use_this_for_non_static_method_access=false 18 | sp_cleanup.convert_to_enhanced_for_loop=true 19 | sp_cleanup.correct_indentation=true 20 | sp_cleanup.format_source_code=true 21 | sp_cleanup.format_source_code_changes_only=false 22 | sp_cleanup.make_local_variable_final=false 23 | sp_cleanup.make_parameters_final=false 24 | sp_cleanup.make_private_fields_final=true 25 | sp_cleanup.make_type_abstract_if_missing_method=false 26 | sp_cleanup.make_variable_declarations_final=false 27 | sp_cleanup.never_use_blocks=false 28 | sp_cleanup.never_use_parentheses_in_expressions=true 29 | sp_cleanup.on_save_use_additional_actions=true 30 | sp_cleanup.organize_imports=true 31 | sp_cleanup.qualify_static_field_accesses_with_declaring_class=false 32 | sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true 33 | sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true 34 | sp_cleanup.qualify_static_member_accesses_with_declaring_class=false 35 | sp_cleanup.qualify_static_method_accesses_with_declaring_class=false 36 | sp_cleanup.remove_private_constructors=true 37 | sp_cleanup.remove_trailing_whitespaces=true 38 | sp_cleanup.remove_trailing_whitespaces_all=true 39 | sp_cleanup.remove_trailing_whitespaces_ignore_empty=false 40 | sp_cleanup.remove_unnecessary_casts=true 41 | sp_cleanup.remove_unnecessary_nls_tags=true 42 | sp_cleanup.remove_unused_imports=true 43 | sp_cleanup.remove_unused_local_variables=false 44 | sp_cleanup.remove_unused_private_fields=true 45 | sp_cleanup.remove_unused_private_members=false 46 | sp_cleanup.remove_unused_private_methods=true 47 | sp_cleanup.remove_unused_private_types=true 48 | sp_cleanup.sort_members=false 49 | sp_cleanup.sort_members_all=false 50 | sp_cleanup.use_blocks=true 51 | sp_cleanup.use_blocks_only_for_return_and_throw=false 52 | sp_cleanup.use_parentheses_in_expressions=false 53 | sp_cleanup.use_this_for_non_static_field_access=false 54 | sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=true 55 | sp_cleanup.use_this_for_non_static_method_access=false 56 | sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true 57 | -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | notifications: 3 | email: false 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | pentaho-kafka-consumer 2 | ====================== 3 | 4 | Apache Kafka consumer step plug-in for Pentaho Kettle. 5 | 6 | [![Build Status](https://travis-ci.org/RuckusWirelessIL/pentaho-kafka-consumer.png)](https://travis-ci.org/RuckusWirelessIL/pentaho-kafka-consumer) 7 | 8 | 9 | ### Screenshots ### 10 | 11 | ![Using Apache Kafka Consumer in Kettle](https://raw.github.com/RuckusWirelessIL/pentaho-kafka-consumer/master/doc/example.png) 12 | 13 | 14 | ### Apache Kafka Compatibility ### 15 | 16 | The consumer depends on Apache Kafka 0.8.1.1, which means that the broker must be of 0.8.x version or later. 17 | 18 | If you want to build the plugin for a different Kafka version you have to 19 | modify the values of kafka.version and kafka.scala.version in the properties 20 | section of the pom.xml. 21 | 22 | ### Maximum Duration Of Consumption ### 23 | 24 | Note that the maximum duration of consumption is a limit on the duration of the 25 | entire step, *not* an individual read. This means that if you have a maximum 26 | duration of 5000ms, your transformation will stop after 5s, whether or 27 | not more data exists and independent of how fast each message is fetched from 28 | the topic. If you want to stop reading messages when the topic has no more 29 | messages, see the section on _Empty topic handling_. 30 | 31 | ### Empty topic handling ### 32 | 33 | If you want the step to halt when there are no more messages available on the 34 | topic, check the "Stop on empty topic" checkbox in the configuration dialog. The 35 | default timeout to wait for messages is 1000ms, but you can override this by 36 | setting the "consumer.timeout.ms" property in the dialog. If you configure a 37 | timeout without checking the box, an empty topic will be considered a failure 38 | case. 39 | 40 | ### Installation ### 41 | 42 | 1. Download ```pentaho-kafka-consumer``` Zip archive from [latest release page](https://github.com/RuckusWirelessIL/pentaho-kafka-consumer/releases/latest). 43 | 2. Extract downloaded archive into *plugins/steps* directory of your Pentaho Data Integration distribution. 44 | 45 | 46 | ### Building from source code ### 47 | 48 | ``` 49 | mvn clean package 50 | ``` 51 | -------------------------------------------------------------------------------- /assembly.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | dist 7 | 8 | ${project.artifactId} 9 | 10 | 11 | zip 12 | 13 | 14 | 15 | 16 | 17 | LICENSE 18 | 19 | 20 | 21 | target 22 | . 23 | 24 | *.jar 25 | version.xml 26 | lib/*.jar 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /doc/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuckusWirelessIL/pentaho-kafka-consumer/50dee62a4c44cb79bcae32bdf64f37003b85527e/doc/example.png -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 4.0.0 5 | com.ruckuswireless 6 | pentaho-kafka-consumer 7 | TRUNK-SNAPSHOT 8 | Apache Kafka Consumer Plug-In for Pentaho 9 | 10 | 11 | UTF-8 12 | 1.6 13 | 1.6 14 | 7.1.0.0-12 15 | 2.10 16 | 0.8.2.1 17 | ${maven.build.timestamp} 18 | 4.13.1 19 | 1.6.6 20 | yyyyMMdd-HHmm 21 | 22 | 23 | 24 | 25 | pentaho-releases 26 | http://nexus.pentaho.org/content/groups/omni 27 | 28 | 29 | maven-eclipse-repo 30 | http://maven-eclipse.github.io/maven 31 | 32 | 33 | 34 | 35 | 36 | org.apache.kafka 37 | kafka_${kafka.scala.version} 38 | ${kafka.version} 39 | 40 | 41 | com.sun.jmx 42 | jmxri 43 | 44 | 45 | com.sun.jdmk 46 | jmxtools 47 | 48 | 49 | javax.jms 50 | jms 51 | 52 | 53 | 54 | 55 | pentaho-kettle 56 | kettle-core 57 | ${kettle.version} 58 | provided 59 | 60 | 61 | pentaho-kettle 62 | kettle-engine 63 | ${kettle.version} 64 | provided 65 | 66 | 67 | pentaho-kettle 68 | kettle-ui-swt 69 | ${kettle.version} 70 | provided 71 | 72 | 73 | pentaho-kettle 74 | kettle-engine-test 75 | ${kettle.version} 76 | test 77 | 78 | 79 | junit 80 | junit 81 | ${junit.version} 82 | test 83 | 84 | 85 | org.powermock 86 | powermock-module-junit4 87 | ${powermock.version} 88 | test 89 | 90 | 91 | org.powermock 92 | powermock-api-mockito 93 | ${powermock.version} 94 | test 95 | 96 | 97 | 98 | 99 | 100 | ${project.artifactId} 101 | 102 | 103 | src/main/resources 104 | 105 | version.xml 106 | 107 | 108 | 109 | src/main/resources 110 | 111 | version.xml 112 | 113 | true 114 | ${project.build.directory} 115 | 116 | 117 | 118 | 119 | 120 | org.pitest 121 | pitest-maven 122 | 1.2.3 123 | 124 | 125 | XML 126 | HTML 127 | 128 | 129 | 130 | 131 | maven-dependency-plugin 132 | 133 | 134 | package 135 | 136 | copy-dependencies 137 | 138 | 139 | provided 140 | runtime 141 | ${project.build.directory}/lib 142 | 143 | 144 | 145 | 146 | 147 | 148 | org.apache.maven.plugins 149 | maven-assembly-plugin 150 | 151 | assembly.xml 152 | ${project.artifactId}-${project.version} 153 | 154 | 0644 155 | 0755 156 | 0755 157 | 158 | 159 | 160 | 161 | package 162 | 163 | single 164 | 165 | 166 | 167 | 168 | 169 | org.sonarsource.scanner.maven 170 | sonar-maven-plugin 171 | 3.3.0.603 172 | 173 | 174 | org.jacoco 175 | jacoco-maven-plugin 176 | 0.7.9 177 | 178 | ${sonar.jacoco.reportPath} 179 | true 180 | 181 | 182 | 183 | agent 184 | 185 | prepare-agent 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | org.eclipse.m2e 197 | lifecycle-mapping 198 | 1.0.0 199 | 200 | 201 | 202 | 203 | 204 | 205 | org.apache.maven.plugins 206 | maven-dependency-plugin 207 | [1.0.0,) 208 | 209 | copy-dependencies 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | sonar 227 | 228 | true 229 | 230 | 231 | http://localhost:9000 232 | ${maven.compiler.source} 233 | 234 | 235 | 236 | 237 | -------------------------------------------------------------------------------- /src/main/java/org/pentaho/di/trans/kafka/consumer/KafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import kafka.consumer.Consumer; 4 | import kafka.consumer.ConsumerConfig; 5 | import kafka.consumer.KafkaStream; 6 | import org.pentaho.di.core.exception.KettleException; 7 | import org.pentaho.di.core.row.RowDataUtil; 8 | import org.pentaho.di.core.row.RowMeta; 9 | import org.pentaho.di.trans.Trans; 10 | import org.pentaho.di.trans.TransMeta; 11 | import org.pentaho.di.trans.step.*; 12 | 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | import java.util.Map.Entry; 17 | import java.util.Properties; 18 | import java.util.concurrent.*; 19 | 20 | /** 21 | * Kafka Consumer step processor 22 | * 23 | * @author Michael Spector 24 | */ 25 | public class KafkaConsumer extends BaseStep implements StepInterface { 26 | public static final String CONSUMER_TIMEOUT_KEY = "consumer.timeout.ms"; 27 | 28 | public KafkaConsumer(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, 29 | Trans trans) { 30 | super(stepMeta, stepDataInterface, copyNr, transMeta, trans); 31 | } 32 | 33 | public boolean init(StepMetaInterface smi, StepDataInterface sdi) { 34 | super.init(smi, sdi); 35 | 36 | KafkaConsumerMeta meta = (KafkaConsumerMeta) smi; 37 | KafkaConsumerData data = (KafkaConsumerData) sdi; 38 | 39 | Properties properties = meta.getKafkaProperties(); 40 | Properties substProperties = new Properties(); 41 | for (Entry e : properties.entrySet()) { 42 | substProperties.put(e.getKey(), environmentSubstitute(e.getValue().toString())); 43 | } 44 | if (meta.isStopOnEmptyTopic()) { 45 | 46 | // If there isn't already a provided value, set a default of 1s 47 | if (!substProperties.containsKey(CONSUMER_TIMEOUT_KEY)) { 48 | substProperties.put(CONSUMER_TIMEOUT_KEY, "1000"); 49 | } 50 | } else { 51 | if (substProperties.containsKey(CONSUMER_TIMEOUT_KEY)) { 52 | logError(Messages.getString("KafkaConsumer.WarnConsumerTimeout")); 53 | } 54 | } 55 | ConsumerConfig consumerConfig = new ConsumerConfig(substProperties); 56 | 57 | logBasic(Messages.getString("KafkaConsumer.CreateKafkaConsumer.Message", consumerConfig.zkConnect())); 58 | data.consumer = Consumer.createJavaConsumerConnector(consumerConfig); 59 | Map topicCountMap = new HashMap(); 60 | String topic = environmentSubstitute(meta.getTopic()); 61 | topicCountMap.put(topic, 1); 62 | Map>> streamsMap = data.consumer.createMessageStreams(topicCountMap); 63 | logDebug("Received streams map: " + streamsMap); 64 | data.streamIterator = streamsMap.get(topic).get(0).iterator(); 65 | 66 | return true; 67 | } 68 | 69 | public void dispose(StepMetaInterface smi, StepDataInterface sdi) { 70 | KafkaConsumerData data = (KafkaConsumerData) sdi; 71 | if (data.consumer != null) { 72 | data.consumer.shutdown(); 73 | 74 | } 75 | super.dispose(smi, sdi); 76 | } 77 | 78 | public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { 79 | Object[] r = getRow(); 80 | if (r == null) { 81 | /* 82 | * If we have no input rows, make sure we at least run once to 83 | * produce output rows. This allows us to consume without requiring 84 | * an input step. 85 | */ 86 | if (!first) { 87 | setOutputDone(); 88 | return false; 89 | } 90 | r = new Object[0]; 91 | } else { 92 | incrementLinesRead(); 93 | } 94 | 95 | final Object[] inputRow = r; 96 | 97 | KafkaConsumerMeta meta = (KafkaConsumerMeta) smi; 98 | final KafkaConsumerData data = (KafkaConsumerData) sdi; 99 | 100 | if (first) { 101 | first = false; 102 | data.inputRowMeta = getInputRowMeta(); 103 | // No input rows means we just dummy data 104 | if (data.inputRowMeta == null) { 105 | data.outputRowMeta = new RowMeta(); 106 | data.inputRowMeta = new RowMeta(); 107 | } else { 108 | data.outputRowMeta = getInputRowMeta().clone(); 109 | } 110 | meta.getFields(data.outputRowMeta, getStepname(), null, null, this, null, null); 111 | } 112 | 113 | try { 114 | long timeout; 115 | String strData = meta.getTimeout(); 116 | 117 | timeout = getTimeout(strData); 118 | 119 | logDebug("Starting message consumption with overall timeout of " + timeout + "ms"); 120 | 121 | KafkaConsumerCallable kafkaConsumer = new KafkaConsumerCallable(meta, data, this) { 122 | protected void messageReceived(byte[] key, byte[] message) throws KettleException { 123 | Object[] newRow = RowDataUtil.addRowData(inputRow.clone(), data.inputRowMeta.size(), 124 | new Object[]{message, key}); 125 | putRow(data.outputRowMeta, newRow); 126 | 127 | if (isRowLevel()) { 128 | logRowlevel(Messages.getString("KafkaConsumer.Log.OutputRow", 129 | Long.toString(getLinesWritten()), data.outputRowMeta.getString(newRow))); 130 | } 131 | } 132 | }; 133 | if (timeout > 0) { 134 | logDebug("Starting timed consumption"); 135 | ExecutorService executor = Executors.newSingleThreadExecutor(); 136 | try { 137 | Future future = executor.submit(kafkaConsumer); 138 | executeFuture(timeout, future); 139 | } finally { 140 | executor.shutdown(); 141 | } 142 | } else { 143 | logDebug("Starting direct consumption"); 144 | kafkaConsumer.call(); 145 | } 146 | } catch (KettleException e) { 147 | if (!getStepMeta().isDoingErrorHandling()) { 148 | logError(Messages.getString("KafkaConsumer.ErrorInStepRunning", e.getMessage())); 149 | setErrors(1); 150 | stopAll(); 151 | setOutputDone(); 152 | return false; 153 | } 154 | putError(getInputRowMeta(), r, 1, e.toString(), null, getStepname()); 155 | } 156 | return true; 157 | } 158 | 159 | private void executeFuture(long timeout, Future future) throws KettleException { 160 | try { 161 | future.get(timeout, TimeUnit.MILLISECONDS); 162 | } catch (TimeoutException e) { 163 | logDebug("Timeout exception on the Future"); 164 | } catch (Exception e) { 165 | throw new KettleException(e); 166 | } 167 | } 168 | 169 | private long getTimeout(String strData) throws KettleException { 170 | long timeout; 171 | try { 172 | timeout = KafkaConsumerMeta.isEmpty(strData) ? 0 : Long.parseLong(environmentSubstitute(strData)); 173 | } catch (NumberFormatException e) { 174 | throw new KettleException("Unable to parse step timeout value", e); 175 | } 176 | return timeout; 177 | } 178 | 179 | public void stopRunning(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { 180 | 181 | KafkaConsumerData data = (KafkaConsumerData) sdi; 182 | data.consumer.shutdown(); 183 | data.canceled = true; 184 | 185 | super.stopRunning(smi, sdi); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/org/pentaho/di/trans/kafka/consumer/KafkaConsumerCallable.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import kafka.consumer.ConsumerTimeoutException; 4 | import kafka.message.MessageAndMetadata; 5 | import org.pentaho.di.core.exception.KettleException; 6 | 7 | import java.util.concurrent.Callable; 8 | 9 | /** 10 | * Kafka reader callable 11 | * 12 | * @author Michael Spector 13 | */ 14 | public abstract class KafkaConsumerCallable implements Callable { 15 | 16 | private KafkaConsumerData data; 17 | private KafkaConsumerMeta meta; 18 | private KafkaConsumer step; 19 | 20 | public KafkaConsumerCallable(KafkaConsumerMeta meta, KafkaConsumerData data, KafkaConsumer step) { 21 | this.meta = meta; 22 | this.data = data; 23 | this.step = step; 24 | } 25 | 26 | /** 27 | * Called when new message arrives from Kafka stream 28 | * 29 | * @param message Kafka message 30 | * @param key Kafka key 31 | */ 32 | protected abstract void messageReceived(byte[] key, byte[] message) throws KettleException; 33 | 34 | public Object call() throws KettleException { 35 | try { 36 | long limit; 37 | String strData = meta.getLimit(); 38 | 39 | limit = getLimit(strData); 40 | if (limit > 0) { 41 | step.logDebug("Collecting up to " + limit + " messages"); 42 | } else { 43 | step.logDebug("Collecting unlimited messages"); 44 | } 45 | while (data.streamIterator.hasNext() && !data.canceled && (limit <= 0 || data.processed < limit)) { 46 | MessageAndMetadata messageAndMetadata = data.streamIterator.next(); 47 | messageReceived(messageAndMetadata.key(), messageAndMetadata.message()); 48 | ++data.processed; 49 | } 50 | } catch (ConsumerTimeoutException cte) { 51 | step.logDebug("Received a consumer timeout after " + data.processed + " messages"); 52 | if (!meta.isStopOnEmptyTopic()) { 53 | // Because we're not set to stop on empty, this is an abnormal 54 | // timeout 55 | throw new KettleException("Unexpected consumer timeout!", cte); 56 | } 57 | } 58 | // Notify that all messages were read successfully 59 | data.consumer.commitOffsets(); 60 | step.setOutputDone(); 61 | return null; 62 | } 63 | 64 | private long getLimit(String strData) throws KettleException { 65 | long limit; 66 | try { 67 | limit = KafkaConsumerMeta.isEmpty(strData) ? 0 : Long.parseLong(step.environmentSubstitute(strData)); 68 | } catch (NumberFormatException e) { 69 | throw new KettleException("Unable to parse messages limit parameter", e); 70 | } 71 | return limit; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/org/pentaho/di/trans/kafka/consumer/KafkaConsumerData.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import kafka.consumer.ConsumerIterator; 4 | import kafka.javaapi.consumer.ConsumerConnector; 5 | import org.pentaho.di.core.row.RowMetaInterface; 6 | import org.pentaho.di.trans.step.BaseStepData; 7 | import org.pentaho.di.trans.step.StepDataInterface; 8 | 9 | /** 10 | * Holds data processed by this step 11 | * 12 | * @author Michael 13 | */ 14 | public class KafkaConsumerData extends BaseStepData implements StepDataInterface { 15 | 16 | ConsumerConnector consumer; 17 | ConsumerIterator streamIterator; 18 | RowMetaInterface outputRowMeta; 19 | RowMetaInterface inputRowMeta; 20 | boolean canceled; 21 | int processed; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/pentaho/di/trans/kafka/consumer/KafkaConsumerMeta.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import kafka.consumer.ConsumerConfig; 4 | import org.pentaho.di.core.CheckResult; 5 | import org.pentaho.di.core.CheckResultInterface; 6 | import org.pentaho.di.core.Const; 7 | import org.pentaho.di.core.annotations.Step; 8 | import org.pentaho.di.core.database.DatabaseMeta; 9 | import org.pentaho.di.core.exception.KettleException; 10 | import org.pentaho.di.core.exception.KettlePluginException; 11 | import org.pentaho.di.core.exception.KettleStepException; 12 | import org.pentaho.di.core.exception.KettleXMLException; 13 | import org.pentaho.di.core.row.RowMetaInterface; 14 | import org.pentaho.di.core.row.ValueMetaInterface; 15 | import org.pentaho.di.core.row.value.ValueMetaFactory; 16 | import org.pentaho.di.core.variables.VariableSpace; 17 | import org.pentaho.di.core.xml.XMLHandler; 18 | import org.pentaho.di.repository.ObjectId; 19 | import org.pentaho.di.repository.Repository; 20 | import org.pentaho.di.trans.Trans; 21 | import org.pentaho.di.trans.TransMeta; 22 | import org.pentaho.di.trans.step.*; 23 | import org.pentaho.metastore.api.IMetaStore; 24 | import org.w3c.dom.Node; 25 | 26 | import java.io.ByteArrayInputStream; 27 | import java.io.ByteArrayOutputStream; 28 | import java.util.HashMap; 29 | import java.util.List; 30 | import java.util.Map; 31 | import java.util.Properties; 32 | 33 | /** 34 | * Kafka Consumer step definitions and serializer to/from XML and to/from Kettle 35 | * repository. 36 | * 37 | * @author Michael Spector 38 | */ 39 | @Step( 40 | id = "KafkaConsumer", 41 | image = "org/pentaho/di/trans/kafka/consumer/resources/kafka_consumer.png", 42 | i18nPackageName = "org.pentaho.di.trans.kafka.consumer", 43 | name = "KafkaConsumerDialog.Shell.Title", 44 | description = "KafkaConsumerDialog.Shell.Tooltip", 45 | documentationUrl = "KafkaConsumerDialog.Shell.DocumentationURL", 46 | casesUrl = "KafkaConsumerDialog.Shell.CasesURL", 47 | categoryDescription = "i18n:org.pentaho.di.trans.step:BaseStep.Category.Input") 48 | public class KafkaConsumerMeta extends BaseStepMeta implements StepMetaInterface { 49 | 50 | @SuppressWarnings("WeakerAccess") 51 | protected static final String[] KAFKA_PROPERTIES_NAMES = new String[]{"zookeeper.connect", "group.id", "consumer.id", 52 | "socket.timeout.ms", "socket.receive.buffer.bytes", "fetch.message.max.bytes", "auto.commit.interval.ms", 53 | "queued.max.message.chunks", "rebalance.max.retries", "fetch.min.bytes", "fetch.wait.max.ms", 54 | "rebalance.backoff.ms", "refresh.leader.backoff.ms", "auto.commit.enable", "auto.offset.reset", 55 | "consumer.timeout.ms", "client.id", "zookeeper.session.timeout.ms", "zookeeper.connection.timeout.ms", 56 | "zookeeper.sync.time.ms"}; 57 | 58 | @SuppressWarnings("WeakerAccess") 59 | protected static final Map KAFKA_PROPERTIES_DEFAULTS = new HashMap(); 60 | 61 | private static final String ATTR_TOPIC = "TOPIC"; 62 | private static final String ATTR_FIELD = "FIELD"; 63 | private static final String ATTR_KEY_FIELD = "KEY_FIELD"; 64 | private static final String ATTR_LIMIT = "LIMIT"; 65 | private static final String ATTR_TIMEOUT = "TIMEOUT"; 66 | private static final String ATTR_STOP_ON_EMPTY_TOPIC = "STOP_ON_EMPTY_TOPIC"; 67 | private static final String ATTR_KAFKA = "KAFKA"; 68 | 69 | static { 70 | KAFKA_PROPERTIES_DEFAULTS.put("zookeeper.connect", "localhost:2181"); 71 | KAFKA_PROPERTIES_DEFAULTS.put("group.id", "group"); 72 | } 73 | 74 | private Properties kafkaProperties = new Properties(); 75 | private String topic; 76 | private String field; 77 | private String keyField; 78 | private String limit; 79 | private String timeout; 80 | private boolean stopOnEmptyTopic; 81 | 82 | public static String[] getKafkaPropertiesNames() { 83 | return KAFKA_PROPERTIES_NAMES; 84 | } 85 | 86 | public static Map getKafkaPropertiesDefaults() { 87 | return KAFKA_PROPERTIES_DEFAULTS; 88 | } 89 | 90 | public KafkaConsumerMeta() { 91 | super(); 92 | } 93 | 94 | public Properties getKafkaProperties() { 95 | return kafkaProperties; 96 | } 97 | 98 | @SuppressWarnings("unused") 99 | public Map getKafkaPropertiesMap() { 100 | return getKafkaProperties(); 101 | } 102 | 103 | public void setKafkaProperties(Properties kafkaProperties) { 104 | this.kafkaProperties = kafkaProperties; 105 | } 106 | 107 | @SuppressWarnings("unused") 108 | public void setKafkaPropertiesMap(Map propertiesMap) { 109 | Properties props = new Properties(); 110 | props.putAll(propertiesMap); 111 | setKafkaProperties(props); 112 | } 113 | 114 | /** 115 | * @return Kafka topic name 116 | */ 117 | public String getTopic() { 118 | return topic; 119 | } 120 | 121 | /** 122 | * @param topic Kafka topic name 123 | */ 124 | public void setTopic(String topic) { 125 | this.topic = topic; 126 | } 127 | 128 | /** 129 | * @return Target field name in Kettle stream 130 | */ 131 | public String getField() { 132 | return field; 133 | } 134 | 135 | /** 136 | * @param field Target field name in Kettle stream 137 | */ 138 | public void setField(String field) { 139 | this.field = field; 140 | } 141 | 142 | /** 143 | * @return Target key field name in Kettle stream 144 | */ 145 | public String getKeyField() { 146 | return keyField; 147 | } 148 | 149 | /** 150 | * @param keyField Target key field name in Kettle stream 151 | */ 152 | public void setKeyField(String keyField) { 153 | this.keyField = keyField; 154 | } 155 | 156 | /** 157 | * @return Limit number of entries to read from Kafka queue 158 | */ 159 | public String getLimit() { 160 | return limit; 161 | } 162 | 163 | /** 164 | * @param limit Limit number of entries to read from Kafka queue 165 | */ 166 | public void setLimit(String limit) { 167 | this.limit = limit; 168 | } 169 | 170 | /** 171 | * @return Time limit for reading entries from Kafka queue (in ms) 172 | */ 173 | public String getTimeout() { 174 | return timeout; 175 | } 176 | 177 | /** 178 | * @param timeout Time limit for reading entries from Kafka queue (in ms) 179 | */ 180 | public void setTimeout(String timeout) { 181 | this.timeout = timeout; 182 | } 183 | 184 | /** 185 | * @return 'true' if the consumer should stop when no more messages are 186 | * available 187 | */ 188 | public boolean isStopOnEmptyTopic() { 189 | return stopOnEmptyTopic; 190 | } 191 | 192 | /** 193 | * @param stopOnEmptyTopic If 'true', stop the consumer when no more messages are 194 | * available on the topic 195 | */ 196 | public void setStopOnEmptyTopic(boolean stopOnEmptyTopic) { 197 | this.stopOnEmptyTopic = stopOnEmptyTopic; 198 | } 199 | 200 | public void check(List remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, 201 | String[] input, String[] output, RowMetaInterface info, VariableSpace space, Repository repository, 202 | IMetaStore metaStore) { 203 | 204 | if (topic == null) { 205 | remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, 206 | Messages.getString("KafkaConsumerMeta.Check.InvalidTopic"), stepMeta)); 207 | } 208 | if (field == null) { 209 | remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, 210 | Messages.getString("KafkaConsumerMeta.Check.InvalidField"), stepMeta)); 211 | } 212 | if (keyField == null) { 213 | remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, 214 | Messages.getString("KafkaConsumerMeta.Check.InvalidKeyField"), stepMeta)); 215 | } 216 | try { 217 | new ConsumerConfig(kafkaProperties); 218 | } catch (IllegalArgumentException e) { 219 | remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, e.getMessage(), stepMeta)); 220 | } 221 | } 222 | 223 | public StepInterface getStep(StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr, TransMeta transMeta, 224 | Trans trans) { 225 | return new KafkaConsumer(stepMeta, stepDataInterface, cnr, transMeta, trans); 226 | } 227 | 228 | public StepDataInterface getStepData() { 229 | return new KafkaConsumerData(); 230 | } 231 | 232 | @Override 233 | public void loadXML(Node stepnode, List databases, IMetaStore metaStore) 234 | throws KettleXMLException { 235 | 236 | try { 237 | topic = XMLHandler.getTagValue(stepnode, ATTR_TOPIC); 238 | field = XMLHandler.getTagValue(stepnode, ATTR_FIELD); 239 | keyField = XMLHandler.getTagValue(stepnode, ATTR_KEY_FIELD); 240 | limit = XMLHandler.getTagValue(stepnode, ATTR_LIMIT); 241 | timeout = XMLHandler.getTagValue(stepnode, ATTR_TIMEOUT); 242 | // This tag only exists if the value is "true", so we can directly 243 | // populate the field 244 | stopOnEmptyTopic = XMLHandler.getTagValue(stepnode, ATTR_STOP_ON_EMPTY_TOPIC) != null; 245 | Node kafkaNode = XMLHandler.getSubNode(stepnode, ATTR_KAFKA); 246 | String[] kafkaElements = XMLHandler.getNodeElements(kafkaNode); 247 | if (kafkaElements != null) { 248 | for (String propName : kafkaElements) { 249 | String value = XMLHandler.getTagValue(kafkaNode, propName); 250 | if (value != null) { 251 | kafkaProperties.put(propName, value); 252 | } 253 | } 254 | } 255 | } catch (Exception e) { 256 | throw new KettleXMLException(Messages.getString("KafkaConsumerMeta.Exception.loadXml"), e); 257 | } 258 | } 259 | 260 | @Override 261 | public String getXML() throws KettleException { 262 | StringBuilder retval = new StringBuilder(); 263 | if (topic != null) { 264 | retval.append(" ").append(XMLHandler.addTagValue(ATTR_TOPIC, topic)); 265 | } 266 | if (field != null) { 267 | retval.append(" ").append(XMLHandler.addTagValue(ATTR_FIELD, field)); 268 | } 269 | if (keyField != null) { 270 | retval.append(" ").append(XMLHandler.addTagValue(ATTR_KEY_FIELD, keyField)); 271 | } 272 | if (limit != null) { 273 | retval.append(" ").append(XMLHandler.addTagValue(ATTR_LIMIT, limit)); 274 | } 275 | if (timeout != null) { 276 | retval.append(" ").append(XMLHandler.addTagValue(ATTR_TIMEOUT, timeout)); 277 | } 278 | if (stopOnEmptyTopic) { 279 | retval.append(" ").append(XMLHandler.addTagValue(ATTR_STOP_ON_EMPTY_TOPIC, "true")); 280 | } 281 | retval.append(" ").append(XMLHandler.openTag(ATTR_KAFKA)).append(Const.CR); 282 | for (String name : kafkaProperties.stringPropertyNames()) { 283 | String value = kafkaProperties.getProperty(name); 284 | if (value != null) { 285 | retval.append(" ").append(XMLHandler.addTagValue(name, value)); 286 | } 287 | } 288 | retval.append(" ").append(XMLHandler.closeTag(ATTR_KAFKA)).append(Const.CR); 289 | return retval.toString(); 290 | } 291 | 292 | @Override 293 | public void readRep(Repository rep, IMetaStore metaStore, ObjectId stepId, List databases) 294 | throws KettleException { 295 | try { 296 | topic = rep.getStepAttributeString(stepId, ATTR_TOPIC); 297 | field = rep.getStepAttributeString(stepId, ATTR_FIELD); 298 | keyField = rep.getStepAttributeString(stepId, ATTR_KEY_FIELD); 299 | limit = rep.getStepAttributeString(stepId, ATTR_LIMIT); 300 | timeout = rep.getStepAttributeString(stepId, ATTR_TIMEOUT); 301 | stopOnEmptyTopic = rep.getStepAttributeBoolean(stepId, ATTR_STOP_ON_EMPTY_TOPIC); 302 | String kafkaPropsXML = rep.getStepAttributeString(stepId, ATTR_KAFKA); 303 | if (kafkaPropsXML != null) { 304 | kafkaProperties.loadFromXML(new ByteArrayInputStream(kafkaPropsXML.getBytes())); 305 | } 306 | // Support old versions: 307 | for (String name : KAFKA_PROPERTIES_NAMES) { 308 | String value = rep.getStepAttributeString(stepId, name); 309 | if (value != null) { 310 | kafkaProperties.put(name, value); 311 | } 312 | } 313 | } catch (Exception e) { 314 | throw new KettleException("KafkaConsumerMeta.Exception.loadRep", e); 315 | } 316 | } 317 | 318 | @Override 319 | public void saveRep(Repository rep, IMetaStore metaStore, ObjectId transformationId, ObjectId stepId) throws KettleException { 320 | try { 321 | if (topic != null) { 322 | rep.saveStepAttribute(transformationId, stepId, ATTR_TOPIC, topic); 323 | } 324 | if (field != null) { 325 | rep.saveStepAttribute(transformationId, stepId, ATTR_FIELD, field); 326 | } 327 | if (keyField != null) { 328 | rep.saveStepAttribute(transformationId, stepId, ATTR_KEY_FIELD, keyField); 329 | } 330 | if (limit != null) { 331 | rep.saveStepAttribute(transformationId, stepId, ATTR_LIMIT, limit); 332 | } 333 | if (timeout != null) { 334 | rep.saveStepAttribute(transformationId, stepId, ATTR_TIMEOUT, timeout); 335 | } 336 | rep.saveStepAttribute(transformationId, stepId, ATTR_STOP_ON_EMPTY_TOPIC, stopOnEmptyTopic); 337 | 338 | ByteArrayOutputStream buf = new ByteArrayOutputStream(); 339 | kafkaProperties.storeToXML(buf, null); 340 | rep.saveStepAttribute(transformationId, stepId, ATTR_KAFKA, buf.toString()); 341 | } catch (Exception e) { 342 | throw new KettleException("KafkaConsumerMeta.Exception.saveRep", e); 343 | } 344 | } 345 | 346 | /** 347 | * Set default values to the transformation 348 | */ 349 | public void setDefault() { 350 | setTopic(""); 351 | } 352 | 353 | public void getFields(RowMetaInterface rowMeta, String origin, RowMetaInterface[] info, StepMeta nextStep, 354 | VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException { 355 | 356 | try { 357 | ValueMetaInterface fieldValueMeta = ValueMetaFactory.createValueMeta(getField(), ValueMetaInterface.TYPE_BINARY); 358 | fieldValueMeta.setOrigin(origin); 359 | rowMeta.addValueMeta(fieldValueMeta); 360 | 361 | ValueMetaInterface keyFieldValueMeta = ValueMetaFactory.createValueMeta(getKeyField(), ValueMetaInterface.TYPE_BINARY); 362 | keyFieldValueMeta.setOrigin(origin); 363 | rowMeta.addValueMeta(keyFieldValueMeta); 364 | 365 | } catch (KettlePluginException e) { 366 | throw new KettleStepException("KafkaConsumerMeta.Exception.getFields", e); 367 | } 368 | 369 | } 370 | 371 | public static boolean isEmpty(String str) { 372 | return str == null || str.length() == 0; 373 | } 374 | 375 | } 376 | -------------------------------------------------------------------------------- /src/main/java/org/pentaho/di/trans/kafka/consumer/Messages.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import org.pentaho.di.i18n.BaseMessages; 4 | 5 | @SuppressWarnings("unused") 6 | public class Messages { 7 | private Messages() { 8 | } 9 | 10 | private static final Class clazz = Messages.class; 11 | 12 | public static String getString(String key) { 13 | return BaseMessages.getString(clazz, key); 14 | } 15 | 16 | public static String getString(String key, String param1) { 17 | return BaseMessages.getString(clazz, key, param1); 18 | } 19 | 20 | public static String getString(String key, String param1, String param2) { 21 | return BaseMessages.getString(clazz, key, param1, param2); 22 | } 23 | 24 | public static String getString(String key, String param1, String param2, String param3) { 25 | return BaseMessages.getString(clazz, key, param1, param2, param3); 26 | } 27 | 28 | public static String getString(String key, String param1, String param2, String param3, String param4) { 29 | return BaseMessages.getString(clazz, key, param1, param2, param3, param4); 30 | } 31 | 32 | public static String getString(String key, String param1, String param2, String param3, String param4, String param5) { 33 | return BaseMessages.getString(clazz, key, param1, param2, param3, param4, param5); 34 | } 35 | 36 | public static String getString(String key, String param1, String param2, String param3, String param4, 37 | String param5, String param6) { 38 | return BaseMessages.getString(clazz, key, param1, param2, param3, param4, param5, param6); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/pentaho/di/ui/trans/kafka/consumer/KafkaConsumerDialog.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.ui.trans.kafka.consumer; 2 | 3 | import org.eclipse.swt.SWT; 4 | import org.eclipse.swt.events.*; 5 | import org.eclipse.swt.layout.FormAttachment; 6 | import org.eclipse.swt.layout.FormData; 7 | import org.eclipse.swt.layout.FormLayout; 8 | import org.eclipse.swt.widgets.*; 9 | import org.pentaho.di.core.Const; 10 | import org.pentaho.di.i18n.BaseMessages; 11 | import org.pentaho.di.trans.TransMeta; 12 | import org.pentaho.di.trans.kafka.consumer.KafkaConsumerMeta; 13 | import org.pentaho.di.trans.kafka.consumer.Messages; 14 | import org.pentaho.di.trans.step.BaseStepMeta; 15 | import org.pentaho.di.trans.step.StepDialogInterface; 16 | import org.pentaho.di.ui.core.widget.ColumnInfo; 17 | import org.pentaho.di.ui.core.widget.TableView; 18 | import org.pentaho.di.ui.core.widget.TextVar; 19 | import org.pentaho.di.ui.trans.step.BaseStepDialog; 20 | 21 | import java.util.Arrays; 22 | import java.util.Properties; 23 | import java.util.TreeSet; 24 | 25 | /** 26 | * UI for the Kafka Consumer step 27 | * 28 | * @author Michael Spector 29 | */ 30 | public class KafkaConsumerDialog extends BaseStepDialog implements StepDialogInterface { 31 | 32 | private KafkaConsumerMeta consumerMeta; 33 | private TextVar wTopicName; 34 | private TextVar wFieldName; 35 | private TextVar wKeyFieldName; 36 | private TableView wProps; 37 | private TextVar wLimit; 38 | private TextVar wTimeout; 39 | private Button wStopOnEmptyTopic; 40 | 41 | public KafkaConsumerDialog(Shell parent, Object in, TransMeta tr, String sname) { 42 | super(parent, (BaseStepMeta) in, tr, sname); 43 | consumerMeta = (KafkaConsumerMeta) in; 44 | } 45 | 46 | public KafkaConsumerDialog(Shell parent, BaseStepMeta baseStepMeta, TransMeta transMeta, String stepname) { 47 | super(parent, baseStepMeta, transMeta, stepname); 48 | consumerMeta = (KafkaConsumerMeta) baseStepMeta; 49 | } 50 | 51 | public KafkaConsumerDialog(Shell parent, int nr, BaseStepMeta in, TransMeta tr) { 52 | super(parent, nr, in, tr); 53 | consumerMeta = (KafkaConsumerMeta) in; 54 | } 55 | 56 | public String open() { 57 | Shell parent = getParent(); 58 | Display display = parent.getDisplay(); 59 | 60 | shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX); 61 | props.setLook(shell); 62 | setShellImage(shell, consumerMeta); 63 | 64 | ModifyListener lsMod = new ModifyListener() { 65 | public void modifyText(ModifyEvent e) { 66 | consumerMeta.setChanged(); 67 | } 68 | }; 69 | changed = consumerMeta.hasChanged(); 70 | 71 | FormLayout formLayout = new FormLayout(); 72 | formLayout.marginWidth = Const.FORM_MARGIN; 73 | formLayout.marginHeight = Const.FORM_MARGIN; 74 | 75 | shell.setLayout(formLayout); 76 | shell.setText(Messages.getString("KafkaConsumerDialog.Shell.Title")); 77 | 78 | int middle = props.getMiddlePct(); 79 | int margin = Const.MARGIN; 80 | 81 | // Step name 82 | wlStepname = new Label(shell, SWT.RIGHT); 83 | wlStepname.setText(Messages.getString("KafkaConsumerDialog.StepName.Label")); 84 | props.setLook(wlStepname); 85 | fdlStepname = new FormData(); 86 | fdlStepname.left = new FormAttachment(0, 0); 87 | fdlStepname.right = new FormAttachment(middle, -margin); 88 | fdlStepname.top = new FormAttachment(0, margin); 89 | wlStepname.setLayoutData(fdlStepname); 90 | wStepname = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); 91 | props.setLook(wStepname); 92 | wStepname.addModifyListener(lsMod); 93 | fdStepname = new FormData(); 94 | fdStepname.left = new FormAttachment(middle, 0); 95 | fdStepname.top = new FormAttachment(0, margin); 96 | fdStepname.right = new FormAttachment(100, 0); 97 | wStepname.setLayoutData(fdStepname); 98 | Control lastControl = wStepname; 99 | 100 | // Topic name 101 | Label wlTopicName = new Label(shell, SWT.RIGHT); 102 | wlTopicName.setText(Messages.getString("KafkaConsumerDialog.TopicName.Label")); 103 | props.setLook(wlTopicName); 104 | FormData fdlTopicName = new FormData(); 105 | fdlTopicName.top = new FormAttachment(lastControl, margin); 106 | fdlTopicName.left = new FormAttachment(0, 0); 107 | fdlTopicName.right = new FormAttachment(middle, -margin); 108 | wlTopicName.setLayoutData(fdlTopicName); 109 | wTopicName = new TextVar(transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); 110 | props.setLook(wTopicName); 111 | wTopicName.addModifyListener(lsMod); 112 | FormData fdTopicName = new FormData(); 113 | fdTopicName.top = new FormAttachment(lastControl, margin); 114 | fdTopicName.left = new FormAttachment(middle, 0); 115 | fdTopicName.right = new FormAttachment(100, 0); 116 | wTopicName.setLayoutData(fdTopicName); 117 | lastControl = wTopicName; 118 | 119 | // Field name 120 | Label wlFieldName = new Label(shell, SWT.RIGHT); 121 | wlFieldName.setText(Messages.getString("KafkaConsumerDialog.FieldName.Label")); 122 | props.setLook(wlFieldName); 123 | FormData fdlFieldName = new FormData(); 124 | fdlFieldName.top = new FormAttachment(lastControl, margin); 125 | fdlFieldName.left = new FormAttachment(0, 0); 126 | fdlFieldName.right = new FormAttachment(middle, -margin); 127 | wlFieldName.setLayoutData(fdlFieldName); 128 | wFieldName = new TextVar(transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); 129 | props.setLook(wFieldName); 130 | wFieldName.addModifyListener(lsMod); 131 | FormData fdFieldName = new FormData(); 132 | fdFieldName.top = new FormAttachment(lastControl, margin); 133 | fdFieldName.left = new FormAttachment(middle, 0); 134 | fdFieldName.right = new FormAttachment(100, 0); 135 | wFieldName.setLayoutData(fdFieldName); 136 | lastControl = wFieldName; 137 | 138 | // Key field name 139 | Label wlKeyFieldName = new Label(shell, SWT.RIGHT); 140 | wlKeyFieldName.setText(Messages.getString("KafkaConsumerDialog.KeyFieldName.Label")); 141 | props.setLook(wlKeyFieldName); 142 | FormData fdlKeyFieldName = new FormData(); 143 | fdlKeyFieldName.top = new FormAttachment(lastControl, margin); 144 | fdlKeyFieldName.left = new FormAttachment(0, 0); 145 | fdlKeyFieldName.right = new FormAttachment(middle, -margin); 146 | wlKeyFieldName.setLayoutData(fdlKeyFieldName); 147 | wKeyFieldName = new TextVar(transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); 148 | props.setLook(wKeyFieldName); 149 | wKeyFieldName.addModifyListener(lsMod); 150 | FormData fdKeyFieldName = new FormData(); 151 | fdKeyFieldName.top = new FormAttachment(lastControl, margin); 152 | fdKeyFieldName.left = new FormAttachment(middle, 0); 153 | fdKeyFieldName.right = new FormAttachment(100, 0); 154 | wKeyFieldName.setLayoutData(fdKeyFieldName); 155 | lastControl = wKeyFieldName; 156 | 157 | // Messages limit 158 | Label wlLimit = new Label(shell, SWT.RIGHT); 159 | wlLimit.setText(Messages.getString("KafkaConsumerDialog.Limit.Label")); 160 | props.setLook(wlLimit); 161 | FormData fdlLimit = new FormData(); 162 | fdlLimit.top = new FormAttachment(lastControl, margin); 163 | fdlLimit.left = new FormAttachment(0, 0); 164 | fdlLimit.right = new FormAttachment(middle, -margin); 165 | wlLimit.setLayoutData(fdlLimit); 166 | wLimit = new TextVar(transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); 167 | props.setLook(wLimit); 168 | wLimit.addModifyListener(lsMod); 169 | FormData fdLimit = new FormData(); 170 | fdLimit.top = new FormAttachment(lastControl, margin); 171 | fdLimit.left = new FormAttachment(middle, 0); 172 | fdLimit.right = new FormAttachment(100, 0); 173 | wLimit.setLayoutData(fdLimit); 174 | lastControl = wLimit; 175 | 176 | // Read timeout 177 | Label wlTimeout = new Label(shell, SWT.RIGHT); 178 | wlTimeout.setText(Messages.getString("KafkaConsumerDialog.Timeout.Label")); 179 | props.setLook(wlTimeout); 180 | FormData fdlTimeout = new FormData(); 181 | fdlTimeout.top = new FormAttachment(lastControl, margin); 182 | fdlTimeout.left = new FormAttachment(0, 0); 183 | fdlTimeout.right = new FormAttachment(middle, -margin); 184 | wlTimeout.setLayoutData(fdlTimeout); 185 | wTimeout = new TextVar(transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); 186 | props.setLook(wTimeout); 187 | wTimeout.addModifyListener(lsMod); 188 | FormData fdTimeout = new FormData(); 189 | fdTimeout.top = new FormAttachment(lastControl, margin); 190 | fdTimeout.left = new FormAttachment(middle, 0); 191 | fdTimeout.right = new FormAttachment(100, 0); 192 | wTimeout.setLayoutData(fdTimeout); 193 | lastControl = wTimeout; 194 | 195 | Label wlStopOnEmptyTopic = new Label(shell, SWT.RIGHT); 196 | wlStopOnEmptyTopic.setText(Messages.getString("KafkaConsumerDialog.StopOnEmpty.Label")); 197 | props.setLook(wlStopOnEmptyTopic); 198 | FormData fdlStopOnEmptyTopic = new FormData(); 199 | fdlStopOnEmptyTopic.top = new FormAttachment(lastControl, margin); 200 | fdlStopOnEmptyTopic.left = new FormAttachment(0, 0); 201 | fdlStopOnEmptyTopic.right = new FormAttachment(middle, -margin); 202 | wlStopOnEmptyTopic.setLayoutData(fdlStopOnEmptyTopic); 203 | wStopOnEmptyTopic = new Button(shell, SWT.CHECK | SWT.LEFT | SWT.BORDER); 204 | props.setLook(wStopOnEmptyTopic); 205 | FormData fdStopOnEmptyTopic = new FormData(); 206 | fdStopOnEmptyTopic.top = new FormAttachment(lastControl, margin); 207 | fdStopOnEmptyTopic.left = new FormAttachment(middle, 0); 208 | fdStopOnEmptyTopic.right = new FormAttachment(100, 0); 209 | wStopOnEmptyTopic.setLayoutData(fdStopOnEmptyTopic); 210 | lastControl = wStopOnEmptyTopic; 211 | 212 | // Buttons 213 | wOK = new Button(shell, SWT.PUSH); 214 | wOK.setText(BaseMessages.getString("System.Button.OK")); //$NON-NLS-1$ 215 | wCancel = new Button(shell, SWT.PUSH); 216 | wCancel.setText(BaseMessages.getString("System.Button.Cancel")); //$NON-NLS-1$ 217 | 218 | setButtonPositions(new Button[]{wOK, wCancel}, margin, null); 219 | 220 | // Kafka properties 221 | ColumnInfo[] colinf = new ColumnInfo[]{ 222 | new ColumnInfo(Messages.getString("KafkaConsumerDialog.TableView.NameCol.Label"), 223 | ColumnInfo.COLUMN_TYPE_TEXT, false), 224 | new ColumnInfo(Messages.getString("KafkaConsumerDialog.TableView.ValueCol.Label"), 225 | ColumnInfo.COLUMN_TYPE_TEXT, false),}; 226 | 227 | wProps = new TableView(transMeta, shell, SWT.FULL_SELECTION | SWT.MULTI, colinf, 1, lsMod, props); 228 | FormData fdProps = new FormData(); 229 | fdProps.top = new FormAttachment(lastControl, margin * 2); 230 | fdProps.bottom = new FormAttachment(wOK, -margin * 2); 231 | fdProps.left = new FormAttachment(0, 0); 232 | fdProps.right = new FormAttachment(100, 0); 233 | wProps.setLayoutData(fdProps); 234 | 235 | // Add listeners 236 | lsCancel = new Listener() { 237 | public void handleEvent(Event e) { 238 | cancel(); 239 | } 240 | }; 241 | lsOK = new Listener() { 242 | public void handleEvent(Event e) { 243 | ok(); 244 | } 245 | }; 246 | wCancel.addListener(SWT.Selection, lsCancel); 247 | wOK.addListener(SWT.Selection, lsOK); 248 | 249 | lsDef = new SelectionAdapter() { 250 | public void widgetDefaultSelected(SelectionEvent e) { 251 | ok(); 252 | } 253 | }; 254 | wStepname.addSelectionListener(lsDef); 255 | wTopicName.addSelectionListener(lsDef); 256 | wFieldName.addSelectionListener(lsDef); 257 | wKeyFieldName.addSelectionListener(lsDef); 258 | wLimit.addSelectionListener(lsDef); 259 | wTimeout.addSelectionListener(lsDef); 260 | wStopOnEmptyTopic.addSelectionListener(lsDef); 261 | 262 | // Detect X or ALT-F4 or something that kills this window... 263 | shell.addShellListener(new ShellAdapter() { 264 | public void shellClosed(ShellEvent e) { 265 | cancel(); 266 | } 267 | }); 268 | 269 | // Set the shell size, based upon previous time... 270 | setSize(shell, 400, 350, true); 271 | 272 | getData(consumerMeta, true); 273 | consumerMeta.setChanged(changed); 274 | 275 | shell.open(); 276 | while (!shell.isDisposed()) { 277 | if (!display.readAndDispatch()) { 278 | display.sleep(); 279 | } 280 | } 281 | return stepname; 282 | } 283 | 284 | /** 285 | * Copy information from the meta-data input to the dialog fields. 286 | */ 287 | private void getData(KafkaConsumerMeta consumerMeta, boolean copyStepname) { 288 | if (copyStepname) { 289 | wStepname.setText(stepname); 290 | } 291 | wTopicName.setText(Const.NVL(consumerMeta.getTopic(), "")); 292 | wFieldName.setText(Const.NVL(consumerMeta.getField(), "")); 293 | wKeyFieldName.setText(Const.NVL(consumerMeta.getKeyField(), "")); 294 | wLimit.setText(Const.NVL(consumerMeta.getLimit(), "")); 295 | wTimeout.setText(Const.NVL(consumerMeta.getTimeout(), "")); 296 | wStopOnEmptyTopic.setSelection(consumerMeta.isStopOnEmptyTopic()); 297 | 298 | TreeSet propNames = new TreeSet(); 299 | propNames.addAll(Arrays.asList(KafkaConsumerMeta.getKafkaPropertiesNames())); 300 | propNames.addAll(consumerMeta.getKafkaProperties().stringPropertyNames()); 301 | 302 | Properties kafkaProperties = consumerMeta.getKafkaProperties(); 303 | int i = 0; 304 | for (String propName : propNames) { 305 | String value = kafkaProperties.getProperty(propName); 306 | TableItem item = new TableItem(wProps.table, i++ > 1 ? SWT.BOLD : SWT.NONE); 307 | int colnr = 1; 308 | item.setText(colnr++, Const.NVL(propName, "")); 309 | String defaultValue = KafkaConsumerMeta.getKafkaPropertiesDefaults().get(propName); 310 | if (defaultValue == null) { 311 | defaultValue = "(default)"; 312 | } 313 | item.setText(colnr++, Const.NVL(value, defaultValue)); 314 | } 315 | 316 | wProps.removeEmptyRows(); 317 | wProps.setRowNums(); 318 | wProps.optWidth(true); 319 | 320 | wStepname.selectAll(); 321 | } 322 | 323 | private void cancel() { 324 | stepname = null; 325 | consumerMeta.setChanged(changed); 326 | dispose(); 327 | } 328 | 329 | /** 330 | * Copy information from the dialog fields to the meta-data input 331 | */ 332 | private void setData(KafkaConsumerMeta consumerMeta) { 333 | consumerMeta.setTopic(wTopicName.getText()); 334 | consumerMeta.setField(wFieldName.getText()); 335 | consumerMeta.setKeyField(wKeyFieldName.getText()); 336 | consumerMeta.setLimit(wLimit.getText()); 337 | consumerMeta.setTimeout(wTimeout.getText()); 338 | consumerMeta.setStopOnEmptyTopic(wStopOnEmptyTopic.getSelection()); 339 | 340 | Properties kafkaProperties = consumerMeta.getKafkaProperties(); 341 | int nrNonEmptyFields = wProps.nrNonEmpty(); 342 | for (int i = 0; i < nrNonEmptyFields; i++) { 343 | TableItem item = wProps.getNonEmpty(i); 344 | int colnr = 1; 345 | String name = item.getText(colnr++); 346 | String value = item.getText(colnr++).trim(); 347 | if (value.length() > 0 && !"(default)".equals(value)) { 348 | kafkaProperties.put(name, value); 349 | } else { 350 | kafkaProperties.remove(name); 351 | } 352 | } 353 | wProps.removeEmptyRows(); 354 | wProps.setRowNums(); 355 | wProps.optWidth(true); 356 | 357 | consumerMeta.setChanged(); 358 | } 359 | 360 | private void ok() { 361 | if (KafkaConsumerMeta.isEmpty(wStepname.getText())) { 362 | return; 363 | } 364 | setData(consumerMeta); 365 | stepname = wStepname.getText(); 366 | dispose(); 367 | } 368 | } 369 | -------------------------------------------------------------------------------- /src/main/resources/org/pentaho/di/trans/kafka/consumer/messages/messages_en_US.properties: -------------------------------------------------------------------------------- 1 | KafkaConsumer.CreateKafkaConsumer.Message=Creating Kafka consumer listening on zookeeper\: {0} 2 | KafkaConsumer.Log.OutputRow=Outputting row {0} : {1} 3 | KafkaConsumer.ErrorInStepRunning=Error running step \: {0} 4 | KafkaConsumer.WarnConsumerTimeout=WARNING\! You have set a consumer timeout, but have not requested termination on an empty topic. This could lead to a transformation failure if the queue becomes empty! 5 | KafkaConsumerMeta.Exception.loadXml=Unable to read step information from XML 6 | KafkaConsumerMeta.Exception.loadRep=Unexpected error reading step information from the repository 7 | KafkaConsumerMeta.Exception.saveRep=Unexpected error writing step information to the repository 8 | KafkaConsumerMeta.Exception.getFields=Error initializing the fields 9 | KafkaConsumerMeta.Check.InvalidTopic=Topic name must be set\! 10 | KafkaConsumerMeta.Check.InvalidField=Field name must be set\! 11 | KafkaConsumerMeta.Check.InvalidKeyField=Key field name must be set\! 12 | KafkaConsumerDialog.Shell.Title=Apache Kafka Consumer 13 | KafkaConsumerDialog.Shell.Tooltip=Read messages throug a specific topic from a Kafka stream 14 | KafkaConsumerDialog.Shell.DocumentationURL=http://wiki.pentaho.com/display/EAI/Apache+Kafka+Consumer 15 | KafkaConsumerDialog.Shell.CasesURL=https://github.com/RuckusWirelessIL/pentaho-kafka-consumer/issues 16 | KafkaConsumerDialog.StepName.Label=Step name 17 | KafkaConsumerDialog.TopicName.Label=Topic name 18 | KafkaConsumerDialog.FieldName.Label=Target message field name 19 | KafkaConsumerDialog.KeyFieldName.Label=Target key field name 20 | KafkaConsumerDialog.Limit.Label=Messages limit 21 | KafkaConsumerDialog.Timeout.Label=Maximum duration of consumption (ms) 22 | KafkaConsumerDialog.StopOnEmpty.Label=Stop on empty topic 23 | KafkaConsumerDialog.TableView.Label=Kafka Properties 24 | KafkaConsumerDialog.TableView.NameCol.Label=Name 25 | KafkaConsumerDialog.TableView.ValueCol.Label=Value 26 | -------------------------------------------------------------------------------- /src/main/resources/org/pentaho/di/trans/kafka/consumer/resources/kafka_consumer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuckusWirelessIL/pentaho-kafka-consumer/50dee62a4c44cb79bcae32bdf64f37003b85527e/src/main/resources/org/pentaho/di/trans/kafka/consumer/resources/kafka_consumer.png -------------------------------------------------------------------------------- /src/main/resources/version.xml: -------------------------------------------------------------------------------- 1 | TRUNK-SNAPSHOT 2 | -------------------------------------------------------------------------------- /src/test/java/org/pentaho/di/trans/kafka/consumer/KafkaConsumerDataTest.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class KafkaConsumerDataTest { 7 | @Test 8 | public void testDefaults() { 9 | KafkaConsumerData data = new KafkaConsumerData(); 10 | Assert.assertNull(data.outputRowMeta); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/test/java/org/pentaho/di/trans/kafka/consumer/KafkaConsumerMetaTest.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import org.junit.BeforeClass; 4 | import org.junit.Test; 5 | import org.pentaho.di.core.CheckResultInterface; 6 | import org.pentaho.di.core.KettleEnvironment; 7 | import org.pentaho.di.core.annotations.Step; 8 | import org.pentaho.di.core.exception.KettleException; 9 | import org.pentaho.di.core.exception.KettleStepException; 10 | import org.pentaho.di.core.row.RowMeta; 11 | import org.pentaho.di.core.row.RowMetaInterface; 12 | import org.pentaho.di.core.row.ValueMetaInterface; 13 | import org.pentaho.di.core.util.Utils; 14 | import org.pentaho.di.core.variables.Variables; 15 | import org.pentaho.di.i18n.BaseMessages; 16 | import org.pentaho.di.trans.TransMeta; 17 | import org.pentaho.di.trans.step.StepMeta; 18 | import org.pentaho.di.trans.steps.loadsave.LoadSaveTester; 19 | import org.pentaho.di.trans.steps.loadsave.MemoryRepository; 20 | import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator; 21 | import org.pentaho.di.trans.steps.loadsave.validator.MapLoadSaveValidator; 22 | import org.pentaho.di.trans.steps.loadsave.validator.StringLoadSaveValidator; 23 | 24 | import java.util.*; 25 | 26 | import static org.junit.Assert.*; 27 | 28 | public class KafkaConsumerMetaTest { 29 | 30 | @BeforeClass 31 | public static void setUpBeforeClass() throws KettleException { 32 | KettleEnvironment.init(false); 33 | } 34 | 35 | @Test 36 | public void testGetStepData() { 37 | KafkaConsumerMeta m = new KafkaConsumerMeta(); 38 | assertEquals(KafkaConsumerData.class, m.getStepData().getClass()); 39 | } 40 | 41 | @Test 42 | public void testStepAnnotations() { 43 | 44 | // PDI Plugin Annotation-based Classloader checks 45 | Step stepAnnotation = KafkaConsumerMeta.class.getAnnotation(Step.class); 46 | assertNotNull(stepAnnotation); 47 | assertFalse(Utils.isEmpty(stepAnnotation.id())); 48 | assertFalse(Utils.isEmpty(stepAnnotation.name())); 49 | assertFalse(Utils.isEmpty(stepAnnotation.description())); 50 | assertFalse(Utils.isEmpty(stepAnnotation.image())); 51 | assertFalse(Utils.isEmpty(stepAnnotation.categoryDescription())); 52 | assertFalse(Utils.isEmpty(stepAnnotation.i18nPackageName())); 53 | assertFalse(Utils.isEmpty(stepAnnotation.documentationUrl())); 54 | assertFalse(Utils.isEmpty(stepAnnotation.casesUrl())); 55 | assertEquals(KafkaConsumerMeta.class.getPackage().getName(), stepAnnotation.i18nPackageName()); 56 | hasi18nValue(stepAnnotation.i18nPackageName(), stepAnnotation.name()); 57 | hasi18nValue(stepAnnotation.i18nPackageName(), stepAnnotation.description()); 58 | hasi18nValue(stepAnnotation.i18nPackageName(), stepAnnotation.documentationUrl()); 59 | hasi18nValue(stepAnnotation.i18nPackageName(), stepAnnotation.casesUrl()); 60 | } 61 | 62 | @Test 63 | public void testDefaults() throws KettleStepException { 64 | KafkaConsumerMeta m = new KafkaConsumerMeta(); 65 | m.setDefault(); 66 | 67 | RowMetaInterface rowMeta = new RowMeta(); 68 | m.getFields(rowMeta, "kafka_consumer", null, null, null, null, null); 69 | 70 | // expect two fields to be added to the row stream 71 | assertEquals(2, rowMeta.size()); 72 | 73 | // those fields must strings and named as configured 74 | assertEquals(ValueMetaInterface.TYPE_BINARY, rowMeta.getValueMeta(0).getType()); // TODO change to string 75 | assertEquals(ValueMetaInterface.TYPE_BINARY, rowMeta.getValueMeta(1).getType()); // TODO change to string 76 | assertEquals(ValueMetaInterface.STORAGE_TYPE_NORMAL, rowMeta.getValueMeta(0).getStorageType()); 77 | assertEquals(ValueMetaInterface.STORAGE_TYPE_NORMAL, rowMeta.getValueMeta(1).getStorageType()); 78 | // TODO check naming 79 | //assertEquals( rowMeta.getFieldNames()[0], m.getOutputField() ); 80 | } 81 | 82 | @Test 83 | public void testLoadSave() throws KettleException { 84 | 85 | List attributes = Arrays.asList("topic", "field", "keyField", "limit", "timeout", "kafka", "stopOnEmptyTopic"); 86 | 87 | Map getterMap = new HashMap(); 88 | getterMap.put("topic", "getTopic"); 89 | getterMap.put("field", "getField"); 90 | getterMap.put("keyField", "getKeyField"); 91 | getterMap.put("limit", "getLimit"); 92 | getterMap.put("timeout", "getTimeout"); 93 | getterMap.put("kafka", "getKafkaPropertiesMap"); 94 | getterMap.put("stopOnEmptyTopic", "isStopOnEmptyTopic"); 95 | 96 | Map setterMap = new HashMap(); 97 | setterMap.put("topic", "setTopic"); 98 | setterMap.put("field", "setField"); 99 | setterMap.put("keyField", "setKeyField"); 100 | setterMap.put("limit", "setLimit"); 101 | setterMap.put("timeout", "setTimeout"); 102 | setterMap.put("kafka", "setKafkaPropertiesMap"); 103 | setterMap.put("stopOnEmptyTopic", "setStopOnEmptyTopic"); 104 | 105 | Map> fieldLoadSaveValidatorAttributeMap = 106 | new HashMap>(); 107 | Map> fieldLoadSaveValidatorTypeMap = 108 | new HashMap>(); 109 | fieldLoadSaveValidatorAttributeMap.put("kafka", new MapLoadSaveValidator( 110 | new KeyStringLoadSaveValidator(), new StringLoadSaveValidator())); 111 | 112 | LoadSaveTester tester = new LoadSaveTester(KafkaConsumerMeta.class, attributes, getterMap, setterMap, fieldLoadSaveValidatorAttributeMap, fieldLoadSaveValidatorTypeMap); 113 | 114 | tester.testSerialization(); 115 | } 116 | 117 | @Test 118 | public void testChecksEmpty() { 119 | KafkaConsumerMeta m = new KafkaConsumerMeta(); 120 | 121 | // Test missing Topic name 122 | List checkResults = new ArrayList(); 123 | m.check(checkResults, new TransMeta(), new StepMeta(), null, null, null, null, new Variables(), new MemoryRepository(), null); 124 | assertFalse(checkResults.isEmpty()); 125 | boolean foundMatch = false; 126 | for (CheckResultInterface result : checkResults) { 127 | if (result.getType() == CheckResultInterface.TYPE_RESULT_ERROR 128 | && result.getText().equals(BaseMessages.getString(KafkaConsumerMeta.class, "KafkaConsumerMeta.Check.InvalidTopic"))) { 129 | foundMatch = true; 130 | } 131 | } 132 | assertTrue("The step checks should fail if input topic is not given", foundMatch); 133 | 134 | // Test missing field name 135 | foundMatch = false; 136 | for (CheckResultInterface result : checkResults) { 137 | if (result.getType() == CheckResultInterface.TYPE_RESULT_ERROR 138 | && result.getText().equals(BaseMessages.getString(KafkaConsumerMeta.class, "KafkaConsumerMeta.Check.InvalidField"))) { 139 | foundMatch = true; 140 | } 141 | } 142 | assertTrue("The step checks should fail if field is not given", foundMatch); 143 | 144 | // Test missing Key field name 145 | foundMatch = false; 146 | for (CheckResultInterface result : checkResults) { 147 | if (result.getType() == CheckResultInterface.TYPE_RESULT_ERROR 148 | && result.getText().equals(BaseMessages.getString(KafkaConsumerMeta.class, "KafkaConsumerMeta.Check.InvalidKeyField"))) { 149 | foundMatch = true; 150 | } 151 | } 152 | assertTrue("The step checks should fail if key is not given", foundMatch); 153 | } 154 | 155 | @Test 156 | public void testChecksNotEmpty() { 157 | KafkaConsumerMeta m = new KafkaConsumerMeta(); 158 | m.setTopic(UUID.randomUUID().toString()); 159 | m.setField(UUID.randomUUID().toString()); 160 | m.setKeyField(UUID.randomUUID().toString()); 161 | 162 | // Test present Topic name 163 | List checkResults = new ArrayList(); 164 | m.check(checkResults, new TransMeta(), new StepMeta(), null, null, null, null, new Variables(), new MemoryRepository(), null); 165 | assertFalse(checkResults.isEmpty()); 166 | boolean foundMatch = false; 167 | for (CheckResultInterface result : checkResults) { 168 | if (result.getType() == CheckResultInterface.TYPE_RESULT_ERROR 169 | && result.getText().equals(BaseMessages.getString(KafkaConsumerMeta.class, "KafkaConsumerMeta.Check.InvalidTopic"))) { 170 | foundMatch = true; 171 | } 172 | } 173 | assertFalse("The step checks should not fail if input topic is given", foundMatch); 174 | 175 | // Test missing field name 176 | foundMatch = false; 177 | for (CheckResultInterface result : checkResults) { 178 | if (result.getType() == CheckResultInterface.TYPE_RESULT_ERROR 179 | && result.getText().equals(BaseMessages.getString(KafkaConsumerMeta.class, "KafkaConsumerMeta.Check.InvalidField"))) { 180 | foundMatch = true; 181 | } 182 | } 183 | assertFalse("The step checks should not fail if field is given", foundMatch); 184 | 185 | // Test missing Key field name 186 | foundMatch = false; 187 | for (CheckResultInterface result : checkResults) { 188 | if (result.getType() == CheckResultInterface.TYPE_RESULT_ERROR 189 | && result.getText().equals(BaseMessages.getString(KafkaConsumerMeta.class, "KafkaConsumerMeta.Check.InvalidKeyField"))) { 190 | foundMatch = true; 191 | } 192 | } 193 | assertFalse("The step checks should not fail if key is given", foundMatch); 194 | 195 | } 196 | 197 | @Test 198 | public void testIsEmpty() { 199 | assertTrue("isEmpty should return true with empty string", KafkaConsumerMeta.isEmpty("")); 200 | assertTrue("isEmpty should return true with null string", KafkaConsumerMeta.isEmpty(null)); 201 | } 202 | 203 | /** 204 | * Private class to generate alphabetic xml tags 205 | */ 206 | private class KeyStringLoadSaveValidator extends StringLoadSaveValidator { 207 | @Override 208 | public String getTestObject() { 209 | return "k" + UUID.randomUUID().toString(); 210 | } 211 | } 212 | 213 | private void hasi18nValue(String i18nPackageName, String messageId) { 214 | String fakeId = UUID.randomUUID().toString(); 215 | String fakeLocalized = BaseMessages.getString(i18nPackageName, fakeId); 216 | assertEquals("The way to identify a missing localization key has changed", "!" + fakeId + "!", fakeLocalized); 217 | 218 | // Real Test 219 | String localized = BaseMessages.getString(i18nPackageName, messageId); 220 | assertFalse(Utils.isEmpty(localized)); 221 | assertNotEquals("!" + messageId + "!", localized); 222 | } 223 | 224 | } 225 | -------------------------------------------------------------------------------- /src/test/java/org/pentaho/di/trans/kafka/consumer/KafkaConsumerTest.java: -------------------------------------------------------------------------------- 1 | package org.pentaho.di.trans.kafka.consumer; 2 | 3 | import kafka.consumer.Consumer; 4 | import kafka.consumer.ConsumerConfig; 5 | import kafka.consumer.ConsumerIterator; 6 | import kafka.consumer.KafkaStream; 7 | import kafka.javaapi.consumer.ZookeeperConsumerConnector; 8 | import kafka.message.Message; 9 | import kafka.message.MessageAndMetadata; 10 | import kafka.serializer.DefaultDecoder; 11 | import org.junit.Before; 12 | import org.junit.BeforeClass; 13 | import org.junit.Test; 14 | import org.junit.runner.RunWith; 15 | import org.mockito.ArgumentCaptor; 16 | import org.mockito.Mock; 17 | import org.pentaho.di.core.KettleEnvironment; 18 | import org.pentaho.di.core.RowMetaAndData; 19 | import org.pentaho.di.core.exception.KettleException; 20 | import org.pentaho.di.core.row.RowMeta; 21 | import org.pentaho.di.core.row.RowMetaInterface; 22 | import org.pentaho.di.core.row.value.ValueMetaString; 23 | import org.pentaho.di.core.variables.Variables; 24 | import org.pentaho.di.trans.Trans; 25 | import org.pentaho.di.trans.TransMeta; 26 | import org.pentaho.di.trans.TransTestFactory; 27 | import org.pentaho.di.trans.step.StepMeta; 28 | import org.powermock.api.mockito.PowerMockito; 29 | import org.powermock.core.classloader.annotations.PowerMockIgnore; 30 | import org.powermock.core.classloader.annotations.PrepareForTest; 31 | import org.powermock.modules.junit4.PowerMockRunner; 32 | 33 | import java.util.*; 34 | 35 | import static org.junit.Assert.*; 36 | import static org.mockito.Mockito.*; 37 | 38 | @PowerMockIgnore("javax.management.*") 39 | @RunWith(PowerMockRunner.class) 40 | @PrepareForTest({Consumer.class}) 41 | public class KafkaConsumerTest { 42 | 43 | private static final String STEP_NAME = "Kafka Step"; 44 | private static final String STEP_LIMIT = "10000"; 45 | 46 | @Mock 47 | private HashMap>> streamsMap; 48 | @Mock 49 | private KafkaStream kafkaStream; 50 | @Mock 51 | private ZookeeperConsumerConnector zookeeperConsumerConnector; 52 | @Mock 53 | private ConsumerIterator streamIterator; 54 | @Mock 55 | private ArrayList> stream; 56 | 57 | private StepMeta stepMeta; 58 | private KafkaConsumerMeta meta; 59 | private KafkaConsumerData data; 60 | private TransMeta transMeta; 61 | private Trans trans; 62 | 63 | @BeforeClass 64 | public static void setUpBeforeClass() throws KettleException { 65 | KettleEnvironment.init(false); 66 | } 67 | 68 | @Before 69 | public void setUp() { 70 | data = new KafkaConsumerData(); 71 | meta = new KafkaConsumerMeta(); 72 | meta.setKafkaProperties(getDefaultKafkaProperties()); 73 | meta.setLimit(STEP_LIMIT); 74 | 75 | stepMeta = new StepMeta("KafkaConsumer", meta); 76 | transMeta = new TransMeta(); 77 | transMeta.addStep(stepMeta); 78 | trans = new Trans(transMeta); 79 | 80 | PowerMockito.mockStatic(Consumer.class); 81 | 82 | when(Consumer.createJavaConsumerConnector(any(ConsumerConfig.class))).thenReturn(zookeeperConsumerConnector); 83 | when(zookeeperConsumerConnector.createMessageStreams(anyMapOf(String.class, Integer.class))).thenReturn(streamsMap); 84 | when(streamsMap.get(anyObject())).thenReturn(stream); 85 | when(stream.get(anyInt())).thenReturn(kafkaStream); 86 | when(kafkaStream.iterator()).thenReturn(streamIterator); 87 | when(streamIterator.next()).thenReturn(generateKafkaMessage()); 88 | } 89 | 90 | @Test(expected = IllegalArgumentException.class) 91 | public void stepInitConfigIssue() throws Exception { 92 | KafkaConsumer step = new KafkaConsumer(stepMeta, data, 1, transMeta, trans); 93 | meta.setKafkaProperties(new Properties()); 94 | 95 | step.init(meta, data); 96 | } 97 | 98 | @Test(expected = KettleException.class) 99 | public void illegalTimeout() throws KettleException { 100 | meta.setTimeout("aaa"); 101 | TransMeta tm = TransTestFactory.generateTestTransformation(new Variables(), meta, STEP_NAME); 102 | 103 | TransTestFactory.executeTestTransformation(tm, TransTestFactory.INJECTOR_STEPNAME, 104 | STEP_NAME, TransTestFactory.DUMMY_STEPNAME, new ArrayList()); 105 | 106 | fail("Invalid timeout value should lead to exception"); 107 | } 108 | 109 | @Test(expected = KettleException.class) 110 | public void invalidLimit() throws KettleException { 111 | meta.setLimit("aaa"); 112 | TransMeta tm = TransTestFactory.generateTestTransformation(new Variables(), meta, STEP_NAME); 113 | 114 | TransTestFactory.executeTestTransformation(tm, TransTestFactory.INJECTOR_STEPNAME, 115 | STEP_NAME, TransTestFactory.DUMMY_STEPNAME, new ArrayList()); 116 | 117 | fail("Invalid limit value should lead to exception"); 118 | } 119 | 120 | @Test 121 | public void withStopOnEmptyTopic() throws KettleException { 122 | 123 | meta.setStopOnEmptyTopic(true); 124 | TransMeta tm = TransTestFactory.generateTestTransformation(new Variables(), meta, STEP_NAME); 125 | 126 | TransTestFactory.executeTestTransformation(tm, TransTestFactory.INJECTOR_STEPNAME, 127 | STEP_NAME, TransTestFactory.DUMMY_STEPNAME, new ArrayList()); 128 | 129 | PowerMockito.verifyStatic(); 130 | ArgumentCaptor consumerConfig = ArgumentCaptor.forClass(ConsumerConfig.class); 131 | Consumer.createJavaConsumerConnector(consumerConfig.capture()); 132 | 133 | assertEquals(1000, consumerConfig.getValue().consumerTimeoutMs()); 134 | } 135 | 136 | // If the step does not receive any rows, the transformation should still run successfully 137 | @Test 138 | public void testNoInput() throws KettleException { 139 | TransMeta tm = TransTestFactory.generateTestTransformation(new Variables(), meta, STEP_NAME); 140 | 141 | List result = TransTestFactory.executeTestTransformation(tm, TransTestFactory.INJECTOR_STEPNAME, 142 | STEP_NAME, TransTestFactory.DUMMY_STEPNAME, new ArrayList()); 143 | 144 | assertNotNull(result); 145 | assertEquals(0, result.size()); 146 | } 147 | 148 | // If the step receives rows without any fields, there should be a two output fields (key + value) on each row 149 | @Test 150 | public void testInputNoFields() throws KettleException { 151 | meta.setKeyField("aKeyField"); 152 | meta.setField("aField"); 153 | 154 | when(streamIterator.hasNext()).thenReturn(true); 155 | 156 | TransMeta tm = TransTestFactory.generateTestTransformation(new Variables(), meta, STEP_NAME); 157 | 158 | List result = TransTestFactory.executeTestTransformation(tm, TransTestFactory.INJECTOR_STEPNAME, 159 | STEP_NAME, TransTestFactory.DUMMY_STEPNAME, generateInputData(2, false)); 160 | 161 | assertNotNull(result); 162 | assertEquals(Integer.parseInt(STEP_LIMIT), result.size()); 163 | for (int i = 0; i < Integer.parseInt(STEP_LIMIT); i++) { 164 | assertEquals(2, result.get(i).size()); 165 | assertEquals("aMessage", result.get(i).getString(0, "default value")); 166 | } 167 | } 168 | 169 | // If the step receives rows without any fields, there should be a two output fields (key + value) on each row 170 | @Test 171 | public void testInputFields() throws KettleException { 172 | meta.setKeyField("aKeyField"); 173 | meta.setField("aField"); 174 | 175 | when(streamIterator.hasNext()).thenReturn(true); 176 | 177 | TransMeta tm = TransTestFactory.generateTestTransformation(new Variables(), meta, STEP_NAME); 178 | 179 | List result = TransTestFactory.executeTestTransformation(tm, TransTestFactory.INJECTOR_STEPNAME, 180 | STEP_NAME, TransTestFactory.DUMMY_STEPNAME, generateInputData(3, true)); 181 | 182 | assertNotNull(result); 183 | assertEquals(Integer.parseInt(STEP_LIMIT), result.size()); 184 | for (int i = 0; i < Integer.parseInt(STEP_LIMIT); i++) { 185 | assertEquals(3, result.get(i).size()); 186 | assertEquals("aMessage", result.get(i).getString(1, "default value")); 187 | } 188 | } 189 | 190 | private static Properties getDefaultKafkaProperties() { 191 | Properties p = new Properties(); 192 | p.put("zookeeper.connect", ""); 193 | p.put("group.id", ""); 194 | 195 | return p; 196 | } 197 | 198 | /** 199 | * @param rowCount The number of rows that should be returned 200 | * @param hasFields Whether a "UUID" field should be added to each row 201 | * @return A RowMetaAndData object that can be used for input data in a test transformation 202 | */ 203 | private static List generateInputData(int rowCount, boolean hasFields) { 204 | List retval = new ArrayList(); 205 | RowMetaInterface rowMeta = new RowMeta(); 206 | if (hasFields) { 207 | rowMeta.addValueMeta(new ValueMetaString("UUID")); 208 | } 209 | 210 | for (int i = 0; i < rowCount; i++) { 211 | Object[] data = new Object[0]; 212 | if (hasFields) { 213 | data = new Object[]{UUID.randomUUID().toString()}; 214 | } 215 | retval.add(new RowMetaAndData(rowMeta, data)); 216 | } 217 | return retval; 218 | } 219 | 220 | private static MessageAndMetadata generateKafkaMessage() { 221 | byte[] message = "aMessage".getBytes(); 222 | 223 | return new MessageAndMetadata("topic", 0, new Message(message), 224 | 0, new DefaultDecoder(null), new DefaultDecoder(null)); 225 | } 226 | 227 | } 228 | --------------------------------------------------------------------------------