├── kafka-connect-rest-plugin ├── src │ ├── test │ │ ├── resources │ │ │ ├── test.properties │ │ │ └── logback.xml │ │ └── java │ │ │ └── com │ │ │ └── tm │ │ │ └── kafka │ │ │ └── connect │ │ │ └── rest │ │ │ ├── RestConnectorConfigTest.java │ │ │ ├── selector │ │ │ └── SimpleTopicSelectorConfigTest.java │ │ │ ├── http │ │ │ ├── payload │ │ │ │ ├── templated │ │ │ │ │ ├── VelocityTemplateEngineTest.java │ │ │ │ │ ├── EnvironmentValueProviderTest.java │ │ │ │ │ ├── XPathResponseValueProviderConfigTest.java │ │ │ │ │ ├── RegexResponseValueProviderConfigTest.java │ │ │ │ │ ├── TemplatedPayloadGeneratorConfigTest.java │ │ │ │ │ ├── AbstractValueProviderTest.java │ │ │ │ │ ├── RegexResponseValueProviderTest.java │ │ │ │ │ ├── XPathResponseValueProviderTest.java │ │ │ │ │ └── TemplatedPayloadGeneratorTest.java │ │ │ │ ├── ConstantPayloadGeneratorConfigTest.java │ │ │ │ └── ConstantPayloadGeneratorTest.java │ │ │ └── executor │ │ │ │ └── OkHttpRequestExecutorConfigTest.java │ │ │ ├── config │ │ │ ├── InstanceOfValidatorTest.java │ │ │ └── ServiceProviderInterfaceRecommenderTest.java │ │ │ ├── RestSinkConnectorTest.java │ │ │ ├── RestSourceConnectorConfigTest.java │ │ │ ├── RestSourceConnectorTest.java │ │ │ ├── util │ │ │ └── StringToMapTest.java │ │ │ ├── RestSinkConnectorConfigTest.java │ │ │ └── RestSinkTaskTest.java │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ ├── com.tm.kafka.connect.rest.selector.TopicSelector │ │ │ ├── com.tm.kafka.connect.rest.http.executor.RequestExecutor │ │ │ ├── com.tm.kafka.connect.rest.http.payload.templated.TemplateEngine │ │ │ ├── com.tm.kafka.connect.rest.http.payload.PayloadGenerator │ │ │ └── com.tm.kafka.connect.rest.http.payload.templated.ValueProvider │ │ └── java │ │ └── com │ │ └── tm │ │ └── kafka │ │ └── connect │ │ └── rest │ │ ├── VersionUtil.java │ │ ├── http │ │ ├── handler │ │ │ ├── ResponseHandler.java │ │ │ └── DefaultResponseHandler.java │ │ ├── executor │ │ │ ├── RequestExecutor.java │ │ │ ├── OkHttpRequestExecutor.java │ │ │ └── OkHttpRequestExecutorConfig.java │ │ ├── Response.java │ │ ├── payload │ │ │ ├── templated │ │ │ │ ├── TemplateEngine.java │ │ │ │ ├── EnvironmentValueProvider.java │ │ │ │ ├── ValueProvider.java │ │ │ │ ├── VelocityTemplateEngine.java │ │ │ │ ├── AbstractValueProvider.java │ │ │ │ ├── TemplatedPayloadGenerator.java │ │ │ │ ├── RegexResponseValueProviderConfig.java │ │ │ │ ├── XPathResponseValueProviderConfig.java │ │ │ │ ├── RegexResponseValueProvider.java │ │ │ │ ├── XPathResponseValueProvider.java │ │ │ │ └── TemplatedPayloadGeneratorConfig.java │ │ │ ├── ConstantPayloadGenerator.java │ │ │ ├── PayloadGenerator.java │ │ │ └── ConstantPayloadGeneratorConfig.java │ │ └── Request.java │ │ ├── ExecutionContext.java │ │ ├── selector │ │ ├── TopicSelector.java │ │ ├── SimpleTopicSelector.java │ │ └── SimpleTopicSelectorConfig.java │ │ ├── config │ │ ├── MethodValidator.java │ │ ├── MethodRecommender.java │ │ ├── InstanceOfValidator.java │ │ └── ServiceProviderInterfaceRecommender.java │ │ ├── metrics │ │ └── Metrics.java │ │ ├── RestSinkConnector.java │ │ ├── RestSourceConnector.java │ │ ├── util │ │ └── StringToMap.java │ │ ├── RestSinkTask.java │ │ ├── RestSourceTask.java │ │ └── RestSourceConnectorConfig.java └── pom.xml ├── examples ├── gcf │ ├── index.js │ ├── config │ │ └── source.json │ └── docker-compose.yml ├── spring │ ├── gs-rest-service │ │ ├── .mvn │ │ │ └── wrapper │ │ │ │ ├── maven-wrapper.properties │ │ │ │ └── maven-wrapper.jar │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── manifest.yml │ │ ├── src │ │ │ ├── main │ │ │ │ └── java │ │ │ │ │ └── hello │ │ │ │ │ ├── Application.java │ │ │ │ │ ├── Greeting.java │ │ │ │ │ ├── Calculation.java │ │ │ │ │ └── GreetingController.java │ │ │ └── test │ │ │ │ └── java │ │ │ │ └── hello │ │ │ │ └── GreetingControllerTests.java │ │ ├── build.gradle │ │ ├── pom.xml │ │ ├── gradlew.bat │ │ ├── mvnw.cmd │ │ ├── gradlew │ │ └── mvnw │ ├── config │ │ ├── params-source.json │ │ ├── sink.json │ │ ├── xpath-template-source.json │ │ ├── regex-template-source.json │ │ └── source.json │ └── docker-compose.yml ├── current-datetime.json └── flood-monitoring.json ├── .gitignore ├── kafka-connect-transform-from-json ├── .gitignore ├── kafka-connect-transform-from-json-avro │ ├── src │ │ └── main │ │ │ └── avro │ │ │ └── Greeting.avsc │ └── pom.xml ├── pom.xml ├── .editorconfig └── kafka-connect-transform-from-json-plugin │ ├── pom.xml │ └── src │ └── main │ └── java │ └── org │ └── apache │ └── kafka │ └── connect │ └── transforms │ └── FromJson.java ├── .editorconfig ├── kafka-connect-transform-add-headers ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── kafka │ └── connect │ └── transforms │ └── AddHeaders.java ├── kafka-connect-transform-velocity-eval ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── kafka │ └── connect │ └── transforms │ └── VelocityEval.java └── README.md /kafka-connect-rest-plugin/src/test/resources/test.properties: -------------------------------------------------------------------------------- 1 | property.foo=bar 2 | -------------------------------------------------------------------------------- /examples/gcf/index.js: -------------------------------------------------------------------------------- 1 | exports.hello = (req, res) => { 2 | res.send(`Hello ${req.body.name || 'World'}!`); 3 | }; 4 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip 2 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llofberg/kafka-connect-rest/HEAD/examples/spring/gs-rest-service/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llofberg/kafka-connect-rest/HEAD/examples/spring/gs-rest-service/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: gs-rest-service 4 | memory: 256M 5 | instances: 1 6 | host: rest-service 7 | domain: guides.spring.io 8 | path: build/libs/gs-rest-service-0.1.0.jar 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *~ 3 | 4 | # Build products 5 | target/ 6 | build/ 7 | 8 | # IntelliJ data 9 | *.iml 10 | .idea/ 11 | .ipr 12 | 13 | # Eclipse 14 | .classpath 15 | .project 16 | .settings/ 17 | 18 | # Documentation build output 19 | /docs/_build 20 | 21 | .DS_Store 22 | 23 | *.jar 24 | -------------------------------------------------------------------------------- /kafka-connect-transform-from-json/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *~ 3 | 4 | # Build products 5 | target/ 6 | build/ 7 | 8 | # IntelliJ data 9 | *.iml 10 | .idea/ 11 | .ipr 12 | 13 | # Eclipse 14 | .classpath 15 | .project 16 | .settings/ 17 | 18 | # Documentation build output 19 | /docs/_build 20 | 21 | .DS_Store 22 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Aug 29 13:08:10 CDT 2016 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.13-bin.zip 7 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/resources/META-INF/services/com.tm.kafka.connect.rest.selector.TopicSelector: -------------------------------------------------------------------------------- 1 | # This file lists the available implementations of the TopicSelector SPI interface 2 | # This interface provides output topic selection functionality 3 | com.tm.kafka.connect.rest.selector.SimpleTopicSelector 4 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/resources/META-INF/services/com.tm.kafka.connect.rest.http.executor.RequestExecutor: -------------------------------------------------------------------------------- 1 | # This file lists the available implementations of the RequestExecutor SPI interface 2 | # This interface provides HTTP request execution functionality 3 | com.tm.kafka.connect.rest.http.executor.OkHttpRequestExecutor 4 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/resources/META-INF/services/com.tm.kafka.connect.rest.http.payload.templated.TemplateEngine: -------------------------------------------------------------------------------- 1 | # This file lists the available implementations of the TemplateEngine SPI interface 2 | # This interface provides template interpretation functionality for templated payloads 3 | com.tm.kafka.connect.rest.http.payload.templated.VelocityTemplateEngine 4 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/VersionUtil.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | public class VersionUtil { 4 | public static String getVersion() { 5 | try { 6 | return VersionUtil.class.getPackage().getImplementationVersion(); 7 | } catch (Exception ex) { 8 | return "0.0.0.0"; 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/resources/META-INF/services/com.tm.kafka.connect.rest.http.payload.PayloadGenerator: -------------------------------------------------------------------------------- 1 | # This file lists the available implementations of the PayloadGenerator SPI interface 2 | # This interface provides HTTP payload generation functionality 3 | com.tm.kafka.connect.rest.http.payload.ConstantPayloadGenerator 4 | com.tm.kafka.connect.rest.http.payload.templated.TemplatedPayloadGenerator 5 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/src/main/java/hello/Application.java: -------------------------------------------------------------------------------- 1 | package hello; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class Application { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(Application.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/handler/ResponseHandler.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.handler; 2 | 3 | import com.tm.kafka.connect.rest.ExecutionContext; 4 | import com.tm.kafka.connect.rest.http.Response; 5 | 6 | import java.util.List; 7 | 8 | public interface ResponseHandler { 9 | 10 | List handle(Response response, ExecutionContext ctx); 11 | } 12 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/RestConnectorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import org.junit.Test; 4 | 5 | public class RestConnectorConfigTest { 6 | @Test 7 | public void docSource() { 8 | System.out.println(RestSourceConnectorConfig.conf().toRst()); 9 | } 10 | 11 | @Test 12 | public void docSink() { 13 | System.out.println(RestSinkConnectorConfig.conf().toRst()); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/src/main/java/hello/Greeting.java: -------------------------------------------------------------------------------- 1 | package hello; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | @Data 8 | @NoArgsConstructor 9 | @AllArgsConstructor 10 | class Greeting { 11 | private long id; 12 | private String content; 13 | private String topic; 14 | private Long timestamp; 15 | private String add1; 16 | private String add2; 17 | private String add3; 18 | } 19 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/ExecutionContext.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | public class ExecutionContext { 4 | 5 | private String taskName; 6 | 7 | 8 | public String getTaskName() { 9 | return this.taskName; 10 | } 11 | 12 | public static ExecutionContext create(String taskName) { 13 | ExecutionContext context = new ExecutionContext(); 14 | context.taskName = taskName; 15 | return context; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/resources/META-INF/services/com.tm.kafka.connect.rest.http.payload.templated.ValueProvider: -------------------------------------------------------------------------------- 1 | # This file lists the available implementations of the ValueProvider SPI interface 2 | # This interface provides initial values for templated payloads functionality 3 | com.tm.kafka.connect.rest.http.payload.templated.EnvironmentValueProvider 4 | com.tm.kafka.connect.rest.http.payload.templated.RegexResponseValueProvider 5 | com.tm.kafka.connect.rest.http.payload.templated.XPathResponseValueProvider 6 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/selector/TopicSelector.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.selector; 2 | 3 | 4 | /** 5 | * A class used to select which topic a source message should be sent to. 6 | *

7 | * Note: This is a Service Provider Interface (SPI) 8 | * All implementations should be listed in 9 | * META-INF/services/com.tm.kafka.connect.rest.http.payload.PayloadGenerator 10 | */ 11 | public interface TopicSelector { 12 | 13 | String getTopic(Object data); 14 | } 15 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/config/MethodValidator.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.config; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | 5 | import java.util.Collections; 6 | 7 | public class MethodValidator implements ConfigDef.Validator { 8 | @Override 9 | public void ensureValid(String name, Object provider) { 10 | } 11 | 12 | @Override 13 | public String toString() { 14 | return new MethodRecommender().validValues("", Collections.emptyMap()).toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/executor/RequestExecutor.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.executor; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | 7 | 8 | /** 9 | * Implementation class should declare constructor with HttpProperties as an argument 10 | *

11 | * Note: This is a Service Provider Interface (SPI) 12 | * All implementations should be listed in 13 | * META-INF/services/com.tm.kafka.connect.rest.http.executor.RequestExecutor 14 | */ 15 | public interface RequestExecutor { 16 | 17 | Response execute(Request request) throws Exception; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/config/MethodRecommender.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.config; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | 5 | import java.util.Arrays; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | public class MethodRecommender implements ConfigDef.Recommender { 10 | @Override 11 | public List validValues(String name, Map connectorConfigs) { 12 | return Arrays.asList("GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"); 13 | } 14 | 15 | @Override 16 | public boolean visible(String name, Map connectorConfigs) { 17 | return true; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/selector/SimpleTopicSelector.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.selector; 2 | 3 | 4 | import org.apache.kafka.common.Configurable; 5 | 6 | import java.util.Map; 7 | 8 | 9 | public class SimpleTopicSelector implements TopicSelector, Configurable { 10 | 11 | private String topic; 12 | 13 | @Override 14 | public void configure(Map props) { 15 | final SimpleTopicSelectorConfig config = new SimpleTopicSelectorConfig(props); 16 | 17 | // Always use the first topic in the list 18 | topic = config.getTopics().get(0); 19 | } 20 | 21 | @Override 22 | public String getTopic(Object data) { 23 | return topic; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /kafka-connect-transform-from-json/kafka-connect-transform-from-json-avro/src/main/avro/Greeting.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "hello", 2 | "type": "record", 3 | "name": "Greeting", 4 | "fields": [ 5 | {"name": "id" , "type": ["null", "int" ], "default": null}, 6 | {"name": "content" , "type": ["null", "string"], "default": null}, 7 | {"name": "topic" , "type": ["null", "string"], "default": null}, 8 | {"name": "timestamp" , "type": ["null", "long" ], "default": null}, 9 | {"name": "add1" , "type": ["null", "string"], "default": null}, 10 | {"name": "add2" , "type": ["null", "string"], "default": null}, 11 | {"name": "add3" , "type": ["null", "string"], "default": null} 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /examples/current-datetime.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "source_rest_current-datetime", 3 | "config": { 4 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 5 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 6 | "connector.class": "com.tm.kafka.connect.rest.RestSourceConnector", 7 | "tasks.max": "1", 8 | "rest.source.poll.interval.ms": "60000", 9 | "rest.source.method": "GET", 10 | "rest.source.url": "http://worldclockapi.com/api/json/utc/now", 11 | "rest.source.headers": "Content-Type:application/json,Accept:application/json", 12 | "rest.source.topic.selector": "com.tm.kafka.connect.rest.selector.SimpleTopicSelector", 13 | "rest.source.destination.topics": "current-datetime" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/gcf/config/source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RestSourceConnectorSpring", 3 | "config": { 4 | "connector.class": "com.tm.kafka.connect.rest.RestSourceConnector", 5 | "tasks.max": "1", 6 | "rest.source.poll.interval.ms": "10000", 7 | "rest.source.method": "POST", 8 | "rest.source.url": "http://-.cloudfunctions.net/hello", 9 | "rest.source.data": "{\"name\":\"Kafka Connect\"}", 10 | "rest.source.payload.converter.class": "com.tm.kafka.connect.rest.converter.StringPayloadConverter", 11 | "rest.source.properties": "Content-Type:application/json,Accept::application/json", 12 | "rest.source.topic.selector": "com.tm.kafka.connect.rest.selector.SimpleTopicSelector", 13 | "rest.source.destination.topics": "restSourceDestinationTopic" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/spring/config/params-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParamsRestSourceConnectorSpring", 3 | "config": { 4 | "producer.compression.type": "snappy", 5 | "connector.class": "com.tm.kafka.connect.rest.RestSourceConnector", 6 | "tasks.max": "1", 7 | "rest.source.poll.interval.ms": "10000", 8 | "rest.source.method": "POST", 9 | "rest.source.url": "http://webservice:8080/reverse", 10 | "rest.source.destination.topics": "restSourceDestinationTopic", 11 | "rest.source.body": "{\"content\": \"This is a GREETING!\"}", 12 | "rest.source.param.names": "caps, unused", 13 | "rest.source.param.caps.value": "UPPER", 14 | "rest.source.param.unused.value": "a&b+c", 15 | "rest.source.headers": "Content-Type:application/json, Accept:application/json" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 2 10 | 11 | # Matches multiple files with brace expansion notation 12 | # Set default charset 13 | [*.{js,py}] 14 | charset = utf-8 15 | 16 | # 4 space indentation 17 | [*.py] 18 | indent_style = space 19 | indent_size = 2 20 | 21 | # Tab indentation (no size specified) 22 | [Makefile] 23 | indent_style = space 24 | 25 | # Indentation override for all JS under lib directory 26 | [lib/**.js] 27 | indent_style = space 28 | indent_size = 2 29 | 30 | # Matches the exact files either package.json or .travis.yml 31 | [{package.json,.travis.yml}] 32 | indent_style = space 33 | indent_size = 2 34 | -------------------------------------------------------------------------------- /kafka-connect-transform-from-json/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | com.tm.kafka 9 | kafka-connect-rest 10 | 1.0.3 11 | 12 | 13 | pom 14 | 15 | kafka-connect-transform-from-json 16 | 17 | 18 | kafka-connect-transform-from-json-avro 19 | kafka-connect-transform-from-json-plugin 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /examples/spring/config/sink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RestSinkConnector", 3 | "config": { 4 | "connector.class": "com.tm.kafka.connect.rest.RestSinkConnector", 5 | "tasks.max": "1", 6 | "topics": "restSourceDestinationTopic", 7 | "rest.sink.url": "http://webservice:8080/count", 8 | "rest.sink.method": "POST", 9 | "rest.sink.headers": "Content-Type:application/json", 10 | "transforms": "velocityEval", 11 | "transforms.velocityEval.type": "org.apache.kafka.connect.transforms.VelocityEval$Value", 12 | "transforms.velocityEval.template": "{\"id\":$value.id,\"content\":\"$value.content\",\"topic\":\"$topic\",\"timestamp\":$timestamp,\"add1\":\"$k\",\"add2\":\"$l\",\"add3\":\"$n.kk\"}", 13 | "transforms.velocityEval.context": "{\"k\":\"v\", \"l\":2, \"n\":{\"kk\":\"vv\"}}" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | mavenCentral() 4 | } 5 | dependencies { 6 | classpath("org.springframework.boot:spring-boot-gradle-plugin:1.5.10.RELEASE") 7 | } 8 | } 9 | 10 | apply plugin: 'java' 11 | apply plugin: 'eclipse' 12 | apply plugin: 'idea' 13 | apply plugin: 'org.springframework.boot' 14 | 15 | jar { 16 | baseName = 'gs-rest-service' 17 | version = '0.1.0' 18 | } 19 | 20 | repositories { 21 | mavenCentral() 22 | } 23 | 24 | sourceCompatibility = 1.8 25 | targetCompatibility = 1.8 26 | 27 | dependencies { 28 | compile("org.springframework.boot:spring-boot-starter-web") 29 | testCompile('org.springframework.boot:spring-boot-starter-test') 30 | testCompile('com.jayway.jsonpath:json-path') 31 | } 32 | 33 | -------------------------------------------------------------------------------- /kafka-connect-transform-from-json/.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 2 10 | 11 | # Matches multiple files with brace expansion notation 12 | # Set default charset 13 | [*.{js,py}] 14 | charset = utf-8 15 | 16 | # 4 space indentation 17 | [*.py] 18 | indent_style = space 19 | indent_size = 2 20 | 21 | # Tab indentation (no size specified) 22 | [Makefile] 23 | indent_style = space 24 | 25 | # Indentation override for all JS under lib directory 26 | [lib/**.js] 27 | indent_style = space 28 | indent_size = 2 29 | 30 | # Matches the exact files either package.json or .travis.yml 31 | [{package.json,.travis.yml}] 32 | indent_style = space 33 | indent_size = 2 34 | -------------------------------------------------------------------------------- /examples/flood-monitoring.json: -------------------------------------------------------------------------------- 1 | // See https://environment.data.gov.uk/flood-monitoring/doc/reference 2 | 3 | { 4 | "name": "source_rest_flood-monitoring-L2404", 5 | "config": { 6 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 7 | "value.converter": "org.apache.kafka.connect.storage.StringConverter", 8 | "connector.class": "com.tm.kafka.connect.rest.RestSourceConnector", 9 | "tasks.max": "1", 10 | "rest.source.poll.interval.ms": "900000", 11 | "rest.source.method": "GET", 12 | "rest.source.url": "http://environment.data.gov.uk/flood-monitoring/id/stations/L2404", 13 | "rest.source.headers": "Content-Type:application/json,Accept:application/json", 14 | "rest.source.topic.selector": "com.tm.kafka.connect.rest.selector.SimpleTopicSelector", 15 | "rest.source.destination.topics": "flood-monitoring-L2404" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/Response.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | public class Response { 7 | 8 | private int statusCode; 9 | private String payload; 10 | private Map> headers; 11 | 12 | 13 | public Response(int statusCode, Map> headers, String payload) { 14 | this.statusCode = statusCode; 15 | this.headers = headers; 16 | this.payload = payload; 17 | } 18 | 19 | public String getPayload() { 20 | return payload; 21 | } 22 | 23 | public Map> getHeaders() { 24 | return headers; 25 | } 26 | 27 | public int getStatusCode() { 28 | return statusCode; 29 | } 30 | 31 | public String toString() { 32 | return "StatusCode=" + getStatusCode() + ", Payload=" + getPayload() + ", Headers=" + getHeaders(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/selector/SimpleTopicSelectorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.selector; 2 | 3 | import com.tm.kafka.connect.rest.selector.SimpleTopicSelectorConfig; 4 | import org.junit.Test; 5 | 6 | import java.util.Arrays; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | import static org.junit.Assert.assertEquals; 12 | import static org.junit.Assert.assertNotNull; 13 | 14 | public class SimpleTopicSelectorConfigTest { 15 | 16 | @Test 17 | public void testConfig() { 18 | Map props = new HashMap<>(); 19 | 20 | props.put("rest.source.destination.topics", "test_topic1, test_topic2"); 21 | 22 | SimpleTopicSelectorConfig config = new SimpleTopicSelectorConfig(props); 23 | 24 | List expectedTopics = Arrays.asList("test_topic1", "test_topic2"); 25 | 26 | assertEquals(expectedTopics, config.getTopics()); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/config/InstanceOfValidator.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.config; 2 | 3 | 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigException; 6 | 7 | 8 | /** 9 | * A validator which checks that the object being validated is an instance of the given class. 10 | */ 11 | public class InstanceOfValidator implements ConfigDef.Validator { 12 | 13 | private Class parent; 14 | 15 | public InstanceOfValidator(Class parent) { 16 | this.parent = parent; 17 | } 18 | 19 | @Override 20 | public void ensureValid(String name, Object obj) { 21 | if (obj instanceof Class 22 | && parent.isAssignableFrom((Class) obj)) { 23 | return; 24 | } 25 | throw new ConfigException(name, obj, "Class must extend: " + parent); 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return "Any class implementing: " + parent; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/VelocityTemplateEngineTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import org.junit.Test; 5 | 6 | import static org.hamcrest.Matchers.equalTo; 7 | import static org.junit.Assert.assertThat; 8 | import static org.mockito.Mockito.mock; 9 | import static org.mockito.Mockito.when; 10 | 11 | 12 | public class VelocityTemplateEngineTest { 13 | 14 | ValueProvider valProvider = mock(ValueProvider.class); 15 | 16 | VelocityTemplateEngine engine = new VelocityTemplateEngine(); 17 | 18 | @Test 19 | public void renderTemplate() { 20 | when(valProvider.lookupValue("name")).thenReturn("Noddy"); 21 | assertThat(engine.renderTemplate("Hello ${name}", valProvider), equalTo("Hello Noddy")); 22 | } 23 | 24 | @Test 25 | public void renderTemplate_undefinedValue() { 26 | assertThat(engine.renderTemplate("Hello ${name}", valProvider), equalTo("Hello ${name}")); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/metrics/Metrics.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.metrics; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | import com.codahale.metrics.jmx.JmxReporter; 5 | import com.tm.kafka.connect.rest.ExecutionContext; 6 | 7 | public class Metrics { 8 | 9 | public static final String ERROR_METRIC = "error"; 10 | public static final String RETRIABLE_ERROR_METRIC = "retriable_error"; 11 | public static final String UNRETRIABLE_ERROR_METRIC = "unretriable_error"; 12 | 13 | static final MetricRegistry metrics = new MetricRegistry(); 14 | static final JmxReporter jmxReporter = JmxReporter.forRegistry(metrics).build(); 15 | 16 | static { 17 | jmxReporter.start(); 18 | } 19 | 20 | public static void increaseCounter(String name, ExecutionContext ctx) { 21 | metrics.counter(String.format("kafka_connect_rest_%s_%s", name, ctx.getTaskName())).inc(); 22 | metrics.counter(String.format("kafka_connect_rest_%s", name)).inc(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/config/ServiceProviderInterfaceRecommender.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.config; 2 | 3 | 4 | import org.apache.kafka.common.config.ConfigDef; 5 | 6 | import java.util.*; 7 | 8 | 9 | public class ServiceProviderInterfaceRecommender implements ConfigDef.Recommender { 10 | 11 | private List implementations; 12 | 13 | public ServiceProviderInterfaceRecommender(Class clazz) { 14 | List implementations = new ArrayList<>(); 15 | ServiceLoader loader = ServiceLoader.load(clazz); 16 | for (T impl : loader) { 17 | implementations.add(impl.getClass()); 18 | } 19 | this.implementations = implementations; 20 | } 21 | 22 | @Override 23 | public List validValues(String name, Map connectorConfigs) { 24 | return implementations; 25 | } 26 | 27 | @Override 28 | public boolean visible(String name, Map connectorConfigs) { 29 | return true; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /kafka-connect-transform-from-json/kafka-connect-transform-from-json-avro/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | kafka-connect-transform-from-json 7 | com.tm.kafka 8 | 1.0.3 9 | 10 | 4.0.0 11 | 12 | kafka-connect-transform-from-json-avro 13 | 14 | 15 | 16 | 17 | org.apache.avro 18 | avro 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | org.apache.avro 28 | avro-maven-plugin 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/TemplateEngine.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | 7 | import java.util.Map; 8 | 9 | 10 | /** 11 | * Template engine is responsible for converting a template into a series of outputs, based on a series of context 12 | * entries. 13 | *

14 | * Note: This is a Service Provider Interface (SPI) 15 | * All implementations should be listed in 16 | * META-INF/services/com.tm.kafka.connect.rest.http.payload.templated.TemplateEngine 17 | */ 18 | public interface TemplateEngine { 19 | 20 | /** 21 | * Get a particular interpretation of the template based on the values in the given coutext. 22 | * 23 | * @param context The source from which values are taken. 24 | * @return A completed template where all appliccable placeholders have been replaced with values. 25 | */ 26 | String renderTemplate(String template, ValueProvider context); 27 | } 28 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/executor/OkHttpRequestExecutorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.executor; 2 | 3 | import com.tm.kafka.connect.rest.http.executor.OkHttpRequestExecutorConfig; 4 | import org.junit.Test; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | import static org.junit.Assert.assertNotNull; 11 | 12 | public class OkHttpRequestExecutorConfigTest { 13 | 14 | @Test 15 | public void testConfig() { 16 | Map props = new HashMap<>(); 17 | 18 | props.put("rest.http.connection.connection.timeout", "2000"); 19 | props.put("rest.http.connection.read.timeout", "5000"); 20 | props.put("rest.http.connection.keep.alive.ms", "10000"); 21 | props.put("rest.http.connection.max.idle", "30000"); 22 | 23 | OkHttpRequestExecutorConfig config = new OkHttpRequestExecutorConfig(props); 24 | 25 | assertEquals(2000, config.getConnectionTimeout()); 26 | assertEquals(5000, config.getReadTimeout()); 27 | assertEquals(10000, config.getKeepAliveDuration()); 28 | assertEquals(30000, config.getMaxIdleConnections()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/EnvironmentValueProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import org.junit.Test; 7 | 8 | import static org.hamcrest.Matchers.*; 9 | import static org.junit.Assert.*; 10 | import static org.mockito.Mockito.mock; 11 | 12 | 13 | public class EnvironmentValueProviderTest { 14 | 15 | Request request = mock(Request.class); 16 | Response response = mock(Response.class); 17 | 18 | EnvironmentValueProvider provider = new EnvironmentValueProvider(); 19 | 20 | @Test 21 | public void extractValues() { 22 | provider.extractValues(request, response); 23 | assertThat(provider.getParameters(), not(hasKey(anything()))); 24 | } 25 | 26 | @Test 27 | public void getValue() { 28 | System.setProperty("test", "yeah"); 29 | assertThat(provider.getValue("test"), equalTo("yeah")); 30 | } 31 | 32 | @Test 33 | public void getValue_notDefined() { 34 | System.clearProperty("test"); 35 | assertThat(provider.getValue("test"), nullValue()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /examples/spring/config/xpath-template-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "XPathTemplatedRestSourceConnectorSpring", 3 | "config": { 4 | "producer.compression.type": "snappy", 5 | "connector.class": "com.tm.kafka.connect.rest.RestSourceConnector", 6 | "tasks.max": "1", 7 | "rest.source.poll.interval.ms": "10000", 8 | "rest.source.method": "GET", 9 | "rest.source.url": "http://webservice:8080/sum-xml", 10 | "rest.source.destination.topics": "restSourceDestinationTopic", 11 | 12 | "rest.source.data.generator": "com.tm.kafka.connect.rest.http.payload.templated.TemplatedPayloadGenerator", 13 | "rest.source.param.names": "val1, val2", 14 | "rest.source.param.val1.template": "$!{val2}", 15 | "rest.source.param.val2.template": "$!{res}", 16 | "rest.source.header.template": "Content-Type:text/plain, Accept:text/plain", 17 | 18 | "rest.source.payload.value.provider": "com.tm.kafka.connect.rest.http.payload.templated.XPathResponseValueProvider", 19 | "rest.source.response.var.names": "val1, val2, res", 20 | "rest.source.response.var.val1.xpath": "/calc/expr/val[1]", 21 | "rest.source.response.var.val2.xpath": "/calc/expr/val[2]", 22 | "rest.source.response.var.res.xpath": "/calc/result" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/src/main/java/hello/Calculation.java: -------------------------------------------------------------------------------- 1 | package hello; 2 | 3 | 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import javax.xml.bind.annotation.XmlAccessType; 9 | import javax.xml.bind.annotation.XmlAccessorType; 10 | import javax.xml.bind.annotation.XmlElement; 11 | import javax.xml.bind.annotation.XmlRootElement; 12 | import java.util.List; 13 | 14 | 15 | @Data 16 | @NoArgsConstructor 17 | @AllArgsConstructor 18 | @XmlRootElement(name = "calc") 19 | @XmlAccessorType(XmlAccessType.FIELD) 20 | class Calculation { 21 | @XmlElement 22 | private Expression expr; 23 | @XmlElement 24 | private Double result; 25 | 26 | public Calculation(Expression.Operation operation, List values, Double result) { 27 | this.expr = new Expression(operation, values); 28 | this.result = result; 29 | } 30 | 31 | @Data 32 | @NoArgsConstructor 33 | @AllArgsConstructor 34 | @XmlAccessorType(XmlAccessType.FIELD) 35 | public static class Expression { 36 | @XmlElement 37 | private Operation operation; 38 | @XmlElement(name = "val") 39 | private List values; 40 | 41 | public enum Operation { 42 | SUM; 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/XPathResponseValueProviderConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import org.junit.Test; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import static org.hamcrest.CoreMatchers.allOf; 10 | import static org.hamcrest.Matchers.contains; 11 | import static org.hamcrest.collection.IsMapContaining.hasEntry; 12 | import static org.junit.Assert.assertThat; 13 | 14 | 15 | public class XPathResponseValueProviderConfigTest { 16 | 17 | @Test 18 | public void testConfig() { 19 | Map props = new HashMap<>(); 20 | 21 | props.put("rest.source.response.var.names", "key1, key2"); 22 | props.put("rest.source.response.var.key1.xpath", "/bookstore/book[1]"); 23 | props.put("rest.source.response.var.key2.xpath", "//title[@lang]"); 24 | 25 | XPathResponseValueProviderConfig config = new XPathResponseValueProviderConfig(props); 26 | 27 | assertThat(config.getResponseVariableNames(), contains("key1", "key2")); 28 | assertThat(config.getResponseVariableXPaths(), allOf( 29 | hasEntry("key1", "/bookstore/book[1]"), 30 | hasEntry("key2", "//title[@lang]"))); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/config/InstanceOfValidatorTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.config; 2 | 3 | import org.apache.kafka.common.config.ConfigException; 4 | import org.hamcrest.Matchers; 5 | import org.junit.Test; 6 | 7 | import static org.hamcrest.Matchers.equalTo; 8 | import static org.junit.Assert.*; 9 | 10 | public class InstanceOfValidatorTest { 11 | 12 | InstanceOfValidator validator = new InstanceOfValidator(TestClass.class); 13 | 14 | @Test 15 | public void ensureValidTest_sameClass() { 16 | validator.ensureValid("test", TestClass.class); 17 | } 18 | 19 | @Test 20 | public void ensureValidTest_subclass() { 21 | validator.ensureValid("test", TestSubClass.class); 22 | } 23 | 24 | @Test(expected = ConfigException.class) 25 | public void ensureValidTest_wrongClass() { 26 | validator.ensureValid("test", Object.class); 27 | } 28 | 29 | @Test(expected = ConfigException.class) 30 | public void ensureValidTest_notAClass() { 31 | validator.ensureValid("test", new Object()); 32 | } 33 | 34 | @Test 35 | public void toString1() { 36 | } 37 | 38 | 39 | private static class TestClass { 40 | } 41 | 42 | private static class TestSubClass extends TestClass { 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /examples/spring/config/regex-template-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RegexTemplatedRestSourceConnectorSpring", 3 | "config": { 4 | "producer.compression.type": "snappy", 5 | "connector.class": "com.tm.kafka.connect.rest.RestSourceConnector", 6 | "tasks.max": "1", 7 | "rest.source.poll.interval.ms": "10000", 8 | "rest.source.method": "GET", 9 | "rest.source.url": "http://webservice:8080/sum-plain", 10 | "rest.source.destination.topics": "restSourceDestinationTopic", 11 | 12 | "rest.source.data.generator": "com.tm.kafka.connect.rest.http.payload.templated.TemplatedPayloadGenerator", 13 | "rest.source.param.names": "val1, val2", 14 | "rest.source.param.val1.template": "$!{val2}", 15 | "rest.source.param.val2.template": "$!{res}", 16 | "rest.source.header.template": "Content-Type:text/plain, Accept:text/plain", 17 | 18 | "rest.source.payload.value.provider": "com.tm.kafka.connect.rest.http.payload.templated.RegexResponseValueProvider", 19 | "rest.source.response.var.names": "val1, val2, res", 20 | "rest.source.response.var.val1.regex": "(\\d+)\\s+\\+\\s+\\d+\\s+=\\s+\\d+", 21 | "rest.source.response.var.val2.regex": "\\d+\\s+\\+\\s+(\\d+)\\s+=\\s+\\d+", 22 | "rest.source.response.var.res.regex": "\\d+\\s+\\+\\s+\\d+\\s+=\\s+(\\d+)" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /kafka-connect-transform-add-headers/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | kafka-connect-rest 7 | com.tm.kafka 8 | 1.0.3 9 | 10 | 4.0.0 11 | 12 | kafka-connect-transform-add-headers 13 | 14 | 15 | 16 | 17 | org.apache.kafka 18 | connect-api 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-clients 24 | 25 | 26 | 27 | org.apache.kafka 28 | connect-transforms 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | org.apache.maven.plugins 38 | maven-shade-plugin 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/EnvironmentValueProvider.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | 7 | 8 | /** 9 | * Lookup values used to populate dynamic payloads. 10 | * These values will be substituted into the payload template. 11 | * 12 | * This implementation looks up values in the System properties and then in environment variables. 13 | */ 14 | public class EnvironmentValueProvider extends AbstractValueProvider { 15 | 16 | /** 17 | * This method does nothing as none of the values used by this class are based on the request or response. 18 | * 19 | * @param request The last request made. 20 | * @param response The last response received. 21 | */ 22 | @Override 23 | protected void extractValues(Request request, Response response) { 24 | // Do nothing 25 | } 26 | 27 | /** 28 | * Returns the value of the given key, which will be looked up first in the System properties and then 29 | * in environment variables. 30 | * 31 | * @return The defined value or null if te key is undefined. 32 | */ 33 | @Override 34 | protected String getValue(String key) { 35 | String value = System.getProperty(key); 36 | return (value != null) ? value : System.getenv(key); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/RestSinkConnector.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.connect.connector.Task; 5 | import org.apache.kafka.connect.sink.SinkConnector; 6 | 7 | import java.util.ArrayList; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | public class RestSinkConnector extends SinkConnector { 13 | 14 | private RestSinkConnectorConfig config; 15 | 16 | @Override 17 | public String version() { 18 | return VersionUtil.getVersion(); 19 | } 20 | 21 | @Override 22 | public void start(Map map) { 23 | config = new RestSinkConnectorConfig(map); 24 | } 25 | 26 | @Override 27 | public Class taskClass() { 28 | return RestSinkTask.class; 29 | } 30 | 31 | @Override 32 | public List> taskConfigs(int maxTasks) { 33 | Map taskProps = new HashMap<>(config.originalsStrings()); 34 | List> taskConfigs = new ArrayList<>(maxTasks); 35 | for (int i = 0; i < maxTasks; ++i) { 36 | taskConfigs.add(taskProps); 37 | } 38 | return taskConfigs; 39 | } 40 | 41 | @Override 42 | public void stop() { 43 | } 44 | 45 | @Override 46 | public ConfigDef config() { 47 | return RestSinkConnectorConfig.conf(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/RegexResponseValueProviderConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.payload.ConstantPayloadGeneratorConfig; 5 | import org.hamcrest.Matchers; 6 | import org.junit.Test; 7 | 8 | import java.util.Arrays; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | import static org.hamcrest.CoreMatchers.allOf; 13 | import static org.hamcrest.CoreMatchers.equalTo; 14 | import static org.hamcrest.Matchers.contains; 15 | import static org.hamcrest.collection.IsMapContaining.hasEntry; 16 | import static org.junit.Assert.assertThat; 17 | 18 | 19 | public class RegexResponseValueProviderConfigTest { 20 | 21 | @Test 22 | public void testConfig() { 23 | Map props = new HashMap<>(); 24 | 25 | props.put("rest.source.response.var.names", "key1, key2"); 26 | props.put("rest.source.response.var.key1.regex", ".*"); 27 | props.put("rest.source.response.var.key2.regex", "result: (\\d+)"); 28 | 29 | RegexResponseValueProviderConfig config = new RegexResponseValueProviderConfig(props); 30 | 31 | assertThat(config.getResponseVariableNames(), contains("key1", "key2")); 32 | assertThat(config.getResponseVariableRegexs(), allOf( 33 | hasEntry("key1", ".*"), 34 | hasEntry("key2", "result: (\\d+)"))); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/RestSourceConnector.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.connect.connector.Task; 5 | import org.apache.kafka.connect.source.SourceConnector; 6 | 7 | import java.util.ArrayList; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | public class RestSourceConnector extends SourceConnector { 13 | 14 | private RestSourceConnectorConfig config; 15 | 16 | @Override 17 | public String version() { 18 | return VersionUtil.getVersion(); 19 | } 20 | 21 | @Override 22 | public void start(Map map) { 23 | config = new RestSourceConnectorConfig(map); 24 | } 25 | 26 | @Override 27 | public Class taskClass() { 28 | return RestSourceTask.class; 29 | } 30 | 31 | @Override 32 | public List> taskConfigs(int maxTasks) { 33 | Map taskProps = new HashMap<>(config.originalsStrings()); 34 | List> taskConfigs = new ArrayList<>(maxTasks); 35 | for (int i = 0; i < maxTasks; ++i) { 36 | taskConfigs.add(taskProps); 37 | } 38 | return taskConfigs; 39 | } 40 | 41 | @Override 42 | public void stop() { 43 | } 44 | 45 | @Override 46 | public ConfigDef config() { 47 | return RestSourceConnectorConfig.conf(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/ConstantPayloadGeneratorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.payload.ConstantPayloadGeneratorConfig; 5 | import org.junit.Test; 6 | 7 | import java.util.Arrays; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import static org.hamcrest.CoreMatchers.allOf; 12 | import static org.hamcrest.CoreMatchers.equalTo; 13 | import static org.hamcrest.collection.IsMapContaining.hasEntry; 14 | import static org.junit.Assert.assertThat; 15 | 16 | 17 | public class ConstantPayloadGeneratorConfigTest { 18 | 19 | @Test 20 | public void testConfig() { 21 | Map props = new HashMap<>(); 22 | 23 | props.put("rest.source.body", "This is the body"); 24 | props.put("rest.source.param.names", "key1, key2"); 25 | props.put("rest.source.param.key1.value", "val1"); 26 | props.put("rest.source.param.key2.value", "val2"); 27 | props.put("rest.source.headers", Arrays.asList("Content-Type:application/json", "Accept:application/json")); 28 | 29 | ConstantPayloadGeneratorConfig config = new ConstantPayloadGeneratorConfig(props); 30 | 31 | assertThat(config.getRequestBody(), equalTo("This is the body")); 32 | assertThat(config.getRequestParameters(), allOf(hasEntry("key1", "val1"), hasEntry("key2", "val2"))); 33 | assertThat(config.getRequestHeaders(), allOf( 34 | hasEntry("Content-Type", "application/json"), hasEntry("Accept", "application/json"))); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/TemplatedPayloadGeneratorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.payload.ConstantPayloadGeneratorConfig; 5 | import org.junit.Test; 6 | 7 | import java.util.Arrays; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import static org.hamcrest.CoreMatchers.allOf; 12 | import static org.hamcrest.CoreMatchers.equalTo; 13 | import static org.hamcrest.collection.IsMapContaining.hasEntry; 14 | import static org.junit.Assert.assertThat; 15 | 16 | 17 | public class TemplatedPayloadGeneratorConfigTest { 18 | 19 | @Test 20 | public void testConfig() { 21 | Map props = new HashMap<>(); 22 | 23 | props.put("rest.source.body", "This is the body"); 24 | props.put("rest.source.param.names", "key1, key2"); 25 | props.put("rest.source.param.key1.value", "val1"); 26 | props.put("rest.source.param.key2.value", "val2"); 27 | props.put("rest.source.headers", Arrays.asList("Content-Type:application/json", "Accept:application/json")); 28 | 29 | ConstantPayloadGeneratorConfig config = new ConstantPayloadGeneratorConfig(props); 30 | 31 | assertThat(config.getRequestBody(), equalTo("This is the body")); 32 | assertThat(config.getRequestParameters(), allOf(hasEntry("key1", "val1"), hasEntry("key2", "val2"))); 33 | assertThat(config.getRequestHeaders(), allOf( 34 | hasEntry("Content-Type", "application/json"), hasEntry("Accept", "application/json"))); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.mockito.InjectMocks; 6 | import org.mockito.Mock; 7 | import org.powermock.api.mockito.PowerMockito; 8 | import org.powermock.core.classloader.annotations.PrepareForTest; 9 | import org.powermock.modules.junit4.PowerMockRunner; 10 | 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | import static org.junit.Assert.assertEquals; 16 | import static org.mockito.Mockito.when; 17 | 18 | 19 | @RunWith(PowerMockRunner.class) 20 | @PrepareForTest({RestSinkConnectorConfig.class, VersionUtil.class}) 21 | public class RestSinkConnectorTest { 22 | 23 | @Mock 24 | private RestSinkConnectorConfig config; 25 | 26 | @InjectMocks 27 | private RestSinkConnector subject; 28 | 29 | @Test 30 | public void shouldReturnListOfConfigsOnStartup() { 31 | Map props = new HashMap<>(); 32 | props.put("key", "val"); 33 | 34 | when(config.originalsStrings()).thenReturn(props); 35 | 36 | List> maps = subject.taskConfigs(3); 37 | 38 | assertEquals(maps.get(0), props); 39 | assertEquals(maps.get(1), props); 40 | assertEquals(maps.get(2), props); 41 | } 42 | 43 | @Test 44 | public void shouldReturnVersionWhenRequested() { 45 | PowerMockito.mockStatic(VersionUtil.class); 46 | when(VersionUtil.getVersion()).thenReturn("test"); 47 | 48 | String version = subject.version(); 49 | 50 | assertEquals("test", version); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/RestSourceConnectorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.Arrays; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | import static org.junit.Assert.assertNotNull; 12 | 13 | public class RestSourceConnectorConfigTest { 14 | 15 | @Test 16 | public void testConfig() { 17 | Map props = new HashMap<>(); 18 | 19 | props.put("rest.source.poll.interval.ms", "60000"); 20 | props.put("rest.source.method", "POST"); 21 | props.put("rest.source.url", "http://test.foobar"); 22 | 23 | props.put("rest.source.payload.converter.class", "com.tm.kafka.connect.rest.converter.source.SourceBytesPayloadConverter"); 24 | props.put("rest.source.topic.selector", "com.tm.kafka.connect.rest.selector.SimpleTopicSelector"); 25 | props.put("rest.source.data.generator", "com.tm.kafka.connect.rest.http.payload.ConstantPayloadGenerator"); 26 | props.put("rest.http.executor.class", "com.tm.kafka.connect.rest.http.executor.OkHttpRequestExecutor"); 27 | 28 | props.put("rest.source.destination.topics", "test_topic"); 29 | 30 | RestSourceConnectorConfig config = new RestSourceConnectorConfig(props); 31 | 32 | assertEquals(60000l, config.getPollInterval()); 33 | assertEquals("POST", config.getMethod()); 34 | assertEquals("http://test.foobar", config.getUrl()); 35 | 36 | assertNotNull(config.getTopicSelector()); 37 | assertNotNull(config.getRequestExecutor()); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/RestSourceConnectorTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.mockito.InjectMocks; 6 | import org.mockito.Mock; 7 | import org.powermock.api.mockito.PowerMockito; 8 | import org.powermock.core.classloader.annotations.PrepareForTest; 9 | import org.powermock.modules.junit4.PowerMockRunner; 10 | 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | import static org.junit.Assert.assertEquals; 16 | import static org.mockito.Mockito.when; 17 | 18 | 19 | @RunWith(PowerMockRunner.class) 20 | @PrepareForTest({RestSinkConnectorConfig.class, VersionUtil.class}) 21 | public class RestSourceConnectorTest { 22 | 23 | @Mock 24 | private RestSourceConnectorConfig config; 25 | 26 | @InjectMocks 27 | private RestSourceConnector subject; 28 | 29 | @Test 30 | public void shouldReturnListOfConfigsOnStartup() { 31 | Map props = new HashMap<>(); 32 | props.put("key", "val"); 33 | 34 | when(config.originalsStrings()).thenReturn(props); 35 | 36 | List> maps = subject.taskConfigs(3); 37 | 38 | assertEquals(maps.get(0), props); 39 | assertEquals(maps.get(1), props); 40 | assertEquals(maps.get(2), props); 41 | } 42 | 43 | @Test 44 | public void shouldReturnVersionWhenRequested() { 45 | PowerMockito.mockStatic(VersionUtil.class); 46 | when(VersionUtil.getVersion()).thenReturn("test"); 47 | 48 | String version = subject.version(); 49 | 50 | assertEquals("test", version); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/ValueProvider.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | 7 | import java.util.Map; 8 | 9 | 10 | /** 11 | * Lookup values used to populate dynamic payloads. 12 | * These values will be substituted into the payload template. 13 | *

14 | * Note: This is a Service Provider Interface (SPI) 15 | * All implementations should be listed in 16 | * META-INF/services/com.tm.kafka.connect.rest.http.payload.templated.ValueProvider 17 | */ 18 | public interface ValueProvider { 19 | 20 | /** 21 | * Update the values being provided in the light of the most recent request and response 22 | * 23 | * @param request The last request made. 24 | * @param response The last response received. 25 | */ 26 | void update(Request request, Response response); 27 | 28 | /** 29 | * Returns the value of the given key, or null if the key s undefined. 30 | * 31 | * @return The value of the key. 32 | */ 33 | String lookupValue(String key); 34 | 35 | /** 36 | * Get the map of keys to values that have been requested since the last update. 37 | * 38 | * @return The parameter map. 39 | */ 40 | Map getParameters(); 41 | 42 | /** 43 | * Set the map of keys to values that will be used to generate the next template. 44 | * Note that the update method may overwrite some or all of these mappings. 45 | * 46 | * @param params The parameter map. 47 | */ 48 | void setParameters(Map params); 49 | } 50 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/config/ServiceProviderInterfaceRecommenderTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.config; 2 | 3 | import com.tm.kafka.connect.rest.http.executor.OkHttpRequestExecutor; 4 | import com.tm.kafka.connect.rest.http.executor.RequestExecutor; 5 | import org.hamcrest.Matchers; 6 | import org.junit.Test; 7 | 8 | import java.text.spi.DateFormatProvider; 9 | import java.util.Collections; 10 | import java.util.List; 11 | import java.util.spi.LocaleServiceProvider; 12 | import java.util.stream.Collectors; 13 | 14 | import static org.hamcrest.Matchers.*; 15 | import static org.junit.Assert.*; 16 | 17 | public class ServiceProviderInterfaceRecommenderTest { 18 | 19 | @Test 20 | public void validValuesTest_actualSPI() { 21 | ServiceProviderInterfaceRecommender recommender = 22 | new ServiceProviderInterfaceRecommender<>(RequestExecutor.class); 23 | assertThat(recommender.validValues("test", Collections.emptyMap()), hasItem(OkHttpRequestExecutor.class)); 24 | } 25 | 26 | @Test 27 | public void validValuesTest_notSPI() { 28 | ServiceProviderInterfaceRecommender recommender = 29 | new ServiceProviderInterfaceRecommender<>(ServiceProviderInterfaceRecommenderTest.class); 30 | assertThat(recommender.validValues("test", Collections.emptyMap()), emptyIterable()); 31 | } 32 | 33 | @Test 34 | public void visible() { 35 | ServiceProviderInterfaceRecommender recommender = 36 | new ServiceProviderInterfaceRecommender<>(RequestExecutor.class); 37 | assertThat(recommender.visible("test", Collections.emptyMap()), equalTo(true)); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /kafka-connect-transform-add-headers/src/main/java/org/apache/kafka/connect/transforms/AddHeaders.java: -------------------------------------------------------------------------------- 1 | package org.apache.kafka.connect.transforms; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.connect.connector.ConnectRecord; 5 | import org.apache.kafka.connect.data.Schema; 6 | import org.apache.kafka.connect.transforms.util.SimpleConfig; 7 | 8 | import java.util.Map; 9 | import java.util.stream.Collectors; 10 | 11 | public class AddHeaders> implements Transformation { 12 | 13 | // TODO: Allow removing, modifying headers 14 | 15 | private static final String HEADERS_CONFIG = "headers"; 16 | private static final String HEADERS_DOC = "List of headers."; 17 | 18 | private static final ConfigDef CONFIG_DEF = new ConfigDef() 19 | .define(HEADERS_CONFIG, ConfigDef.Type.LIST, ConfigDef.NO_DEFAULT_VALUE, ConfigDef.Importance.MEDIUM, HEADERS_DOC); 20 | 21 | private Map headers; 22 | 23 | @Override 24 | public void configure(Map props) { 25 | final SimpleConfig config = new SimpleConfig(CONFIG_DEF, props); 26 | headers = config.getList(HEADERS_CONFIG) 27 | .stream() 28 | .map(s -> s.split(":", 2)) 29 | .collect(Collectors.toMap(ss -> ss[0], ss -> ss[1])); 30 | } 31 | 32 | @Override 33 | public R apply(R record) { 34 | for (Map.Entry header : headers.entrySet()) { 35 | record.headers().add(header.getKey(), header.getValue(), Schema.STRING_SCHEMA); 36 | } 37 | return record; 38 | } 39 | 40 | @Override 41 | public void close() { 42 | } 43 | 44 | @Override 45 | public ConfigDef config() { 46 | return CONFIG_DEF; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/selector/SimpleTopicSelectorConfig.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.selector; 2 | 3 | 4 | import com.tm.kafka.connect.rest.VersionUtil; 5 | import org.apache.kafka.common.config.AbstractConfig; 6 | import org.apache.kafka.common.config.ConfigDef; 7 | import org.apache.kafka.common.config.ConfigDef.Importance; 8 | import org.apache.kafka.common.config.ConfigDef.Type; 9 | 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; 15 | 16 | 17 | public class SimpleTopicSelectorConfig extends AbstractConfig { 18 | 19 | public static final String TOPIC_LIST_CONFIG = "rest.source.destination.topics"; 20 | private static final String TOPIC_LIST_DOC = "The list of destination topics for the REST source connector."; 21 | private static final String TOPIC_LIST_DISPLAY = "Source destination topics"; 22 | 23 | 24 | protected SimpleTopicSelectorConfig(ConfigDef config, Map parsedConfig) { 25 | super(config, parsedConfig); 26 | } 27 | 28 | public SimpleTopicSelectorConfig(Map parsedConfig) { 29 | this(conf(), parsedConfig); 30 | } 31 | 32 | public static ConfigDef conf() { 33 | String group = "REST_HTTP"; 34 | int orderInGroup = 0; 35 | return new ConfigDef() 36 | .define(TOPIC_LIST_CONFIG, 37 | Type.LIST, 38 | NO_DEFAULT_VALUE, 39 | Importance.HIGH, 40 | TOPIC_LIST_DOC, 41 | group, 42 | ++orderInGroup, 43 | ConfigDef.Width.SHORT, 44 | TOPIC_LIST_DISPLAY) 45 | ; 46 | } 47 | 48 | public List getTopics() { 49 | return this.getList(TOPIC_LIST_CONFIG); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /kafka-connect-transform-velocity-eval/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | kafka-connect-rest 7 | com.tm.kafka 8 | 1.0.3 9 | 10 | 4.0.0 11 | 12 | kafka-connect-transform-velocity-eval 13 | 14 | 15 | 16 | 17 | org.apache.velocity 18 | velocity-engine-core 19 | 20 | 21 | 22 | com.fasterxml.jackson.core 23 | jackson-databind 24 | 25 | 26 | 27 | org.apache.kafka 28 | connect-api 29 | 30 | 31 | 32 | org.apache.kafka 33 | kafka-clients 34 | 35 | 36 | 37 | org.apache.kafka 38 | connect-transforms 39 | 40 | 41 | 42 | org.slf4j 43 | slf4j-api 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | org.apache.maven.plugins 53 | maven-shade-plugin 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /examples/spring/config/source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RestSourceConnectorSpring", 3 | "config": { 4 | "producer.compression.type": "snappy", 5 | "connector.class": "com.tm.kafka.connect.rest.RestSourceConnector", 6 | "tasks.max": "1", 7 | "rest.source.poll.interval.ms": "10000", 8 | "rest.source.method": "GET", 9 | "rest.source.url": "http://webservice:8080/greeting", 10 | "rest.source.headers": "Content-Type:application/json,Accept:application/json", 11 | "rest.source.topic.selector": "com.tm.kafka.connect.rest.selector.SimpleTopicSelector", 12 | "rest.source.destination.topics": "restSourceDestinationTopic", 13 | "rest.source.headers": "", 14 | "transforms": "addHeaders,fromJson1,velocityEval,fromJson2", 15 | "transforms.addHeaders.type": "org.apache.kafka.connect.transforms.AddHeaders", 16 | "transforms.addHeaders.headers": "X-test-header-1:Value-1, X-test-header-2:Value-2", 17 | "transforms.fromJson1.type": "org.apache.kafka.connect.transforms.FromJson$Value", 18 | "transforms.fromJson1.message.jar": "jar:file:///jars/greeting-1.0-SNAPSHOT.jar!/", 19 | "transforms.fromJson1.message.class": "hello.Greeting", 20 | "transforms.velocityEval.type": "org.apache.kafka.connect.transforms.VelocityEval$Value", 21 | "transforms.velocityEval.template": "{\"id\":$value.id,\"content\":\"$value.content\",\"topic\":\"$topic\",\"timestamp\":$timestamp,\"add1\":\"$k\",\"add2\":\"$l\",\"add3\":\"$n.kk\"}", 22 | "transforms.velocityEval.context": "{\"k\":\"v\", \"l\":2, \"n\":{\"kk\":\"vv\"}}", 23 | "transforms.fromJson2.type": "org.apache.kafka.connect.transforms.FromJson$Value", 24 | "transforms.fromJson2.message.jar": "jar:file:///jars/greeting-1.0-SNAPSHOT.jar!/", 25 | "transforms.fromJson2.message.class": "hello.Greeting" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/util/StringToMapTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.util; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | public class StringToMapTest { 11 | 12 | @Test 13 | public void testUpdate() { 14 | Map converted = StringToMap.update("test.foo.bar", "value"); 15 | 16 | Map level3 = new HashMap<>(); 17 | level3.put("bar", "value"); 18 | Map level2 = new HashMap<>(); 19 | level2.put("foo", level3); 20 | Map expected = new HashMap<>(); 21 | expected.put("test", level2); 22 | 23 | assertEquals(expected, converted); 24 | } 25 | 26 | 27 | @Test 28 | public void testExtract() { 29 | Map level3 = new HashMap<>(); 30 | level3.put("bar", "value"); 31 | Map level2 = new HashMap<>(); 32 | level2.put("foo", level3); 33 | Map map = new HashMap<>(); 34 | map.put("test", level2); 35 | 36 | String extracted = StringToMap.extract("test.foo.bar", map); 37 | 38 | assertEquals("value", extracted); 39 | } 40 | 41 | @Test 42 | public void testRemove() { 43 | Map level3 = new HashMap<>(); 44 | level3.put("bar", "value"); 45 | Map level2 = new HashMap<>(); 46 | level2.put("foo", level3); 47 | Map map = new HashMap<>(); 48 | map.put("test", level2); 49 | 50 | StringToMap.remove("test.foo.bar", map); 51 | 52 | Map level2Expected = new HashMap<>(); 53 | level2Expected.put("foo", new HashMap<>()); 54 | Map mapExpected = new HashMap<>(); 55 | mapExpected.put("test", level2Expected); 56 | 57 | assertEquals(mapExpected, map); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/ConstantPayloadGenerator.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import org.apache.kafka.common.Configurable; 7 | 8 | import java.util.Collections; 9 | import java.util.Map; 10 | 11 | import static java.lang.System.currentTimeMillis; 12 | 13 | 14 | /** 15 | * This is a payload generator that always returns the same payload. 16 | * The constant payload is defined in the configuration. 17 | */ 18 | public class ConstantPayloadGenerator implements PayloadGenerator, Configurable { 19 | 20 | private String requestBody; 21 | private Map requestParameters; 22 | private Map requestHeaders; 23 | 24 | @Override 25 | public void configure(Map props) { 26 | final ConstantPayloadGeneratorConfig config = new ConstantPayloadGeneratorConfig(props); 27 | 28 | requestBody = config.getRequestBody(); 29 | requestParameters = config.getRequestParameters(); 30 | requestHeaders = config.getRequestHeaders(); 31 | } 32 | 33 | @Override 34 | public boolean update(Request request, Response response) { 35 | // False = Wait for the next poll cycle before calling again. 36 | return false; 37 | } 38 | 39 | @Override 40 | public String getRequestBody() { 41 | return requestBody; 42 | } 43 | 44 | @Override 45 | public Map getRequestParameters() { 46 | return requestParameters; 47 | } 48 | 49 | @Override 50 | public Map getRequestHeaders() { 51 | return requestHeaders; 52 | } 53 | 54 | @Override 55 | public Map getOffsets() { 56 | return Collections.singletonMap("timestamp", currentTimeMillis()); 57 | } 58 | 59 | @Override 60 | public void setOffsets(Map offsets) { 61 | // do nothing. 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/RestSinkConnectorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | import static org.junit.Assert.assertNotNull; 10 | import static org.junit.Assert.assertTrue; 11 | 12 | public class RestSinkConnectorConfigTest { 13 | @Test 14 | public void doc() { 15 | System.out.println(RestSinkConnectorConfig.conf().toRst()); 16 | } 17 | 18 | @Test 19 | public void testInit() { 20 | Map props = new HashMap<>(); 21 | 22 | props.put("rest.sink.method", "POST"); 23 | props.put("rest.sink.headers", "Accept:application/json, Content-Type:application/json"); 24 | props.put("rest.sink.url", "http://test.foobar"); 25 | 26 | props.put("rest.http.max.retries", "3"); 27 | props.put("rest.http.codes.whitelist", "^200$"); 28 | props.put("rest.http.codes.blacklist", "^500$"); 29 | props.put("rest.sink.retry.backoff.ms", "15000"); 30 | 31 | props.put("rest.sink.payload.converter.class", "com.tm.kafka.connect.rest.converter.sink.SinkStringPayloadConverter"); 32 | props.put("rest.http.executor.class", "com.tm.kafka.connect.rest.http.executor.OkHttpRequestExecutor"); 33 | 34 | RestSinkConnectorConfig config = new RestSinkConnectorConfig(props); 35 | 36 | Map expectedHeaders = new HashMap<>(); 37 | expectedHeaders.put("Content-Type", "application/json"); 38 | expectedHeaders.put("Accept", "application/json"); 39 | 40 | assertEquals("POST", config.getMethod()); 41 | assertEquals(expectedHeaders, config.getRequestHeaders()); 42 | assertEquals(expectedHeaders, config.getRequestHeaders()); 43 | assertEquals("http://test.foobar", config.getUrl()); 44 | assertEquals(3, config.getMaxRetries()); 45 | assertTrue(15000 == config.getRetryBackoff()); 46 | 47 | assertNotNull(config.getRequestExecutor()); 48 | assertNotNull(config.getResponseHandler()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/PayloadGenerator.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | 7 | import java.util.Map; 8 | 9 | 10 | /** 11 | * Get the request that should be made next. 12 | * This next request might be made immediately or on next poll. 13 | *

14 | * This is currently only used by sources, 15 | * but could be relevant to sinks if something more than the kafka message needed to be sent. 16 | *

17 | * Note: This is a Service Provider Interface (SPI) 18 | * All implementations should be listed in 19 | * META-INF/services/com.tm.kafka.connect.rest.http.payload.PayloadGenerator 20 | */ 21 | public interface PayloadGenerator { 22 | 23 | /** 24 | * Update the generator with the request/response that just happened. 25 | * 26 | * @param request The request just made 27 | * @param response The response just received 28 | * @return True if another call should be made immediately, false otherwise. 29 | */ 30 | boolean update(Request request, Response response); 31 | 32 | /** 33 | * Get the HTTP request body that should be sent with the next request. 34 | * This is not used for GET requests. 35 | * 36 | * @return The body content to be sent to the REST service. 37 | */ 38 | String getRequestBody(); 39 | 40 | /** 41 | * Get the HTTP request parameters that should be sent with the next request. 42 | * 43 | * @return The parameters to be sent to the REST service. 44 | */ 45 | Map getRequestParameters(); 46 | 47 | /** 48 | * Get the HTTP request headers that should be sent with the next request. 49 | * 50 | * @return The headers to be sent to the REST service. 51 | */ 52 | Map getRequestHeaders(); 53 | 54 | /** 55 | * Get the input stream offsets for the current payload. 56 | * 57 | * @return The offsets. 58 | */ 59 | Map getOffsets(); 60 | 61 | /** 62 | * Set the input stream offsets. 63 | * 64 | * @param offsets The offsets. 65 | */ 66 | void setOffsets(Map offsets); 67 | } 68 | 69 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework 7 | gs-rest-service 8 | 0.1.0 9 | 10 | 11 | org.springframework.boot 12 | spring-boot-starter-parent 13 | 2.1.4.RELEASE 14 | 15 | 16 | 17 | 18 | org.springframework.boot 19 | spring-boot-starter-web 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter-test 24 | test 25 | 26 | 27 | com.jayway.jsonpath 28 | json-path 29 | test 30 | 31 | 32 | org.projectlombok 33 | lombok 34 | 1.18.6 35 | provided 36 | 37 | 38 | 39 | 40 | 1.8 41 | 42 | 43 | 44 | 45 | 46 | 47 | org.springframework.boot 48 | spring-boot-maven-plugin 49 | 50 | 51 | 52 | 53 | 54 | 55 | spring-releases 56 | https://repo.spring.io/libs-release 57 | 58 | 59 | 60 | 61 | spring-releases 62 | https://repo.spring.io/libs-release 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/handler/DefaultResponseHandler.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.handler; 2 | 3 | import com.tm.kafka.connect.rest.ExecutionContext; 4 | import com.tm.kafka.connect.rest.http.Response; 5 | import com.tm.kafka.connect.rest.metrics.Metrics; 6 | import org.apache.kafka.connect.errors.RetriableException; 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.regex.Matcher; 11 | import java.util.regex.Pattern; 12 | 13 | public class DefaultResponseHandler implements ResponseHandler { 14 | 15 | private final Pattern allowedCodes; 16 | private final Pattern forbiddenCodes; 17 | 18 | public DefaultResponseHandler() { 19 | this(null, null); 20 | } 21 | 22 | public DefaultResponseHandler(String whitelist, String blacklist) { 23 | this.allowedCodes = createPattern(whitelist); 24 | this.forbiddenCodes = createPattern(blacklist); 25 | } 26 | 27 | @Override 28 | public List handle(Response response, ExecutionContext ctx) { 29 | 30 | String code = String.valueOf(response.getStatusCode()); 31 | 32 | Metrics.increaseCounter(translateHttpCodeToMetricName(code), ctx); 33 | 34 | if (allowedCodes != null) { 35 | checkCodeIsAllowed(code); 36 | } 37 | 38 | if (forbiddenCodes != null) { 39 | checkCodeIsForbidden(code); 40 | } 41 | 42 | ArrayList records = new ArrayList<>(); 43 | records.add(response.getPayload()); 44 | return records; 45 | } 46 | 47 | private String translateHttpCodeToMetricName(String code) { 48 | return String.format("response_code_%sXX", code.substring(0, 1)); 49 | } 50 | 51 | private void checkCodeIsAllowed(String code) { 52 | Matcher allowed = allowedCodes.matcher(code); 53 | if (!allowed.find()) { 54 | throw new RetriableException("HTTP Response code is not whitelisted " + code); 55 | } 56 | } 57 | 58 | private void checkCodeIsForbidden(String code) { 59 | Matcher forbidden = forbiddenCodes.matcher(code); 60 | if (forbidden.find()) { 61 | throw new RetriableException("HTTP Response code is in blacklist " + code); 62 | } 63 | } 64 | 65 | private Pattern createPattern(String regex) { 66 | if (regex == null || regex.trim().isEmpty()) { 67 | return null; 68 | } 69 | return Pattern.compile(regex); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/util/StringToMap.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.util; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | public class StringToMap { 7 | 8 | public static Map update(String path, String update) { 9 | HashMap map = new HashMap<>(); 10 | update(path.split("\\."), 0, map, update); 11 | return map; 12 | } 13 | 14 | public static void update(String path, String update, Map map) { 15 | update(path.split("\\."), 0, map, update); 16 | } 17 | 18 | private static void update(String[] strHierarchy, int idx, Map map, String update) { 19 | String key = strHierarchy[idx]; 20 | 21 | if (idx == strHierarchy.length - 1) { 22 | map.put(key, update); 23 | return; 24 | } 25 | 26 | Map embedded; 27 | if (map.containsKey(key) && map.get(key) instanceof Map) { 28 | embedded = (Map) map.get(key); 29 | } else { 30 | embedded = new HashMap<>(); 31 | map.put(key, embedded); 32 | } 33 | 34 | update(strHierarchy, ++idx, embedded, update); 35 | } 36 | 37 | public static String extract(String path, Map map) { 38 | return extract(path.split("\\."), 0, map); 39 | } 40 | 41 | private static String extract(String[] strHierarchy, int idx, Map map) { 42 | String key = strHierarchy[idx]; 43 | 44 | if (idx == strHierarchy.length - 1) { 45 | return map.getOrDefault(key, "").toString(); 46 | } 47 | 48 | if (map.containsKey(key) && map.get(key) instanceof Map) { 49 | Map embedded = (Map) map.get(key); 50 | return extract(strHierarchy, ++idx, embedded); 51 | } else { 52 | return ""; 53 | } 54 | } 55 | 56 | public static void remove(String path, Map map) { 57 | remove(path.split("\\."), 0, map); 58 | } 59 | 60 | private static void remove(String[] strHierarchy, int idx, Map map) { 61 | String key = strHierarchy[idx]; 62 | 63 | if (idx == strHierarchy.length - 1) { 64 | map.remove(key); 65 | } 66 | 67 | if (map.containsKey(key) && map.get(key) instanceof Map) { 68 | Map embedded = (Map) map.get(key); 69 | remove(strHierarchy, ++idx, embedded); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/VelocityTemplateEngine.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import org.apache.velocity.app.Velocity; 5 | import org.apache.velocity.context.AbstractContext; 6 | 7 | import java.io.StringWriter; 8 | 9 | 10 | /** 11 | * This template engine uses MessageFormat to convert a template into a series of outputs, based on a series of context 12 | * entries. 13 | */ 14 | public class VelocityTemplateEngine implements TemplateEngine { 15 | 16 | private VelocityContextAdapter contextAdapter = new VelocityContextAdapter(); 17 | 18 | 19 | public VelocityTemplateEngine() { 20 | Velocity.init(); 21 | } 22 | 23 | /** 24 | * Get a particular interpretation of the template based on the values in the given context. 25 | * 26 | * @param context The source from which values are taken. 27 | * @return A completed template where all appliccable placeholders have been replaced with values. 28 | */ 29 | @Override 30 | public String renderTemplate(String template, ValueProvider context) { 31 | StringWriter writer = new StringWriter(); 32 | contextAdapter.setValueProvider(context); 33 | Velocity.evaluate(contextAdapter, writer, "", template); 34 | return writer.toString(); 35 | } 36 | 37 | 38 | /** 39 | * This is an adapter that allows a ValueProvider to be used as a Velocity Context object. 40 | */ 41 | private static class VelocityContextAdapter extends AbstractContext { 42 | 43 | private ValueProvider context; 44 | 45 | 46 | public void setValueProvider(ValueProvider context) { 47 | this.context = context; 48 | } 49 | 50 | @Override 51 | public Object internalGet(String key) { 52 | return context.lookupValue(key); 53 | } 54 | 55 | @Override 56 | public Object internalPut(String s, Object o) { 57 | throw new UnsupportedOperationException("An attempt was made to alter to a read-only context"); 58 | } 59 | 60 | @Override 61 | public boolean internalContainsKey(String key) { 62 | return context.lookupValue(key) != null; 63 | } 64 | 65 | @Override 66 | public String[] internalGetKeys() { 67 | return new String[0]; 68 | } 69 | 70 | @Override 71 | public Object internalRemove(String s) { 72 | throw new UnsupportedOperationException("An attempt was made to alter to a read-only context"); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/AbstractValueProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import org.junit.Test; 7 | 8 | import java.util.Collections; 9 | import java.util.Map; 10 | 11 | import static org.hamcrest.Matchers.*; 12 | import static org.junit.Assert.*; 13 | import static org.mockito.Mockito.mock; 14 | import static org.mockito.Mockito.when; 15 | 16 | 17 | public class AbstractValueProviderTest { 18 | 19 | Request request = mock(Request.class); 20 | Response response = mock(Response.class); 21 | 22 | AbstractValueProvider provider = new TestValueProvider(); 23 | 24 | @Test 25 | public void update() { 26 | when(response.getPayload()).thenReturn("OK"); 27 | provider.update(request, response); 28 | Map parameters = provider.getParameters(); 29 | assertThat(parameters, hasEntry("body", "OK")); 30 | assertThat(parameters.keySet(), hasSize(1)); 31 | } 32 | 33 | @Test 34 | public void lookupValue_alreadyFound() { 35 | assertThat(provider.lookupValue("parameter"), equalTo("value")); 36 | } 37 | 38 | @Test 39 | public void lookupValue_canBeFound() { 40 | assertThat(provider.lookupValue("test"), equalTo("yeah")); 41 | } 42 | 43 | @Test 44 | public void lookupValue_cannotBeFound() { 45 | assertThat(provider.lookupValue("xxx"), nullValue()); 46 | } 47 | 48 | 49 | @Test 50 | public void getParameters() { 51 | assertThat(provider.getParameters(), hasEntry("parameter", "value")); 52 | } 53 | 54 | @Test 55 | public void setParameters() { 56 | provider.setParameters(Collections.singletonMap("new-param", "val")); 57 | Map parameters = provider.getParameters(); 58 | assertThat(parameters, hasEntry("new-param", "val")); 59 | assertThat(parameters.keySet(), hasSize(1)); 60 | } 61 | 62 | 63 | private static class TestValueProvider extends AbstractValueProvider { 64 | 65 | public TestValueProvider() { 66 | parameterMap.put("parameter", "value"); 67 | } 68 | 69 | @Override 70 | protected void extractValues(Request request, Response response) { 71 | parameterMap.put("body", response.getPayload()); 72 | } 73 | 74 | @Override 75 | protected String getValue(String key) { 76 | return "test".equals(key) ? "yeah" : null; 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /kafka-connect-transform-from-json/kafka-connect-transform-from-json-plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | kafka-connect-transform-from-json 7 | com.tm.kafka 8 | 1.0.3 9 | 10 | 4.0.0 11 | 12 | kafka-connect-transform-from-json-plugin 13 | 14 | 15 | 16 | 17 | com.tm.kafka 18 | kafka-connect-transform-from-json-avro 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-clients 24 | 25 | 26 | 27 | org.apache.kafka 28 | connect-api 29 | 30 | 31 | 32 | org.apache.kafka 33 | connect-transforms 34 | 35 | 36 | 37 | io.confluent 38 | kafka-connect-avro-converter 39 | 40 | 41 | 42 | com.fasterxml.jackson.core 43 | jackson-databind 44 | 45 | 46 | 47 | org.slf4j 48 | slf4j-api 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.apache.maven.plugins 59 | maven-jar-plugin 60 | 61 | 62 | 63 | org.apache.maven.plugins 64 | maven-compiler-plugin 65 | 66 | 67 | 68 | org.apache.maven.plugins 69 | maven-shade-plugin 70 | 71 | 72 | 73 | 74 | 75 | 76 | src/main/resources 77 | true 78 | 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /examples/gcf/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | 4 | zookeeper: 5 | hostname: zookeeeper 6 | image: confluentinc/cp-zookeeper:4.1.1 7 | environment: 8 | ZOOKEEPER_CLIENT_PORT: 2181 9 | zk_id: "1" 10 | 11 | kafka: 12 | hostname: kafka 13 | image: confluentinc/cp-kafka:4.1.1 14 | links: 15 | - zookeeper 16 | ports: 17 | - "9092:9092" 18 | environment: 19 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 20 | KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://:9092" 21 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 22 | 23 | schema_registry: 24 | hostname: schema_registry 25 | image: confluentinc/cp-schema-registry:4.1.1 26 | links: 27 | - kafka 28 | - zookeeper 29 | ports: 30 | - "8081:8081" 31 | environment: 32 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" 33 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 34 | 35 | connect: 36 | image: confluentinc/cp-kafka-connect:4.1.1 37 | hostname: connect 38 | depends_on: 39 | - webservice 40 | - zookeeper 41 | - kafka 42 | - schema_registry 43 | ports: 44 | - "8083:8083" 45 | environment: 46 | CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092' 47 | CONNECT_REST_ADVERTISED_HOST_NAME: connect 48 | CONNECT_REST_PORT: 8083 49 | CONNECT_GROUP_ID: compose-connect-group 50 | CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs 51 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 52 | CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 53 | CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets 54 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 55 | CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status 56 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 57 | CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter 58 | CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' 59 | CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter 60 | # CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter 61 | CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' 62 | CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter 63 | CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter 64 | CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181' 65 | CONNECT_PLUGIN_PATH: /jars 66 | volumes: 67 | - ../../target/kafka-connect-rest-1.0-SNAPSHOT-package/share/java/kafka-connect-rest:/jars 68 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/Request.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http; 2 | 3 | import java.util.Collections; 4 | import java.util.Map; 5 | 6 | public class Request { 7 | 8 | private String url; 9 | private String body; 10 | private Map parameters; 11 | private Map headers; 12 | private String method; 13 | 14 | 15 | public Request(String url, String method, String body, Map parameters, Map headers) { 16 | this.url = url; 17 | this.method = method; 18 | this.body = body; 19 | this.parameters = parameters; 20 | this.headers = headers; 21 | } 22 | 23 | public String getUrl() { 24 | return url; 25 | } 26 | 27 | public String getBody() { 28 | return body; 29 | } 30 | 31 | public Map getParameters() { 32 | return parameters; 33 | } 34 | 35 | public Map getHeaders() { 36 | return headers; 37 | } 38 | 39 | public String getMethod() { 40 | return method; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return "Request{" + 46 | "method='" + method + '\'' + 47 | ", url='" + url + '\'' + 48 | ", parameters=" + parameters + 49 | ", headers=" + headers + 50 | ", body='" + body + '\'' + 51 | '}'; 52 | } 53 | 54 | public static class RequestFactory { 55 | 56 | private String url; 57 | private String method; 58 | private Map headers; 59 | 60 | public RequestFactory(String url, String method) { 61 | this.url = url; 62 | this.method = method; 63 | } 64 | 65 | public Request createRequest(String body, Map parameters, Map headers) { 66 | return new Request(url, method, body, parameters, headers); 67 | } 68 | 69 | public Request createRequest(String payload, Map headers) { 70 | // TODO - This is a bit of a hack. How the value is sent to the REST endpoint should be explicitly configured 71 | // It is possible that you may still want to send it as a request parameter for a POST request, 72 | // and the name(s) of the parameter(s) being sent should be determined by config or the payload its self. 73 | if("GET".equalsIgnoreCase(method)) { 74 | return new Request(url, method, null, Collections.singletonMap("value", payload), headers); 75 | } else { 76 | return new Request(url, method, payload, Collections.emptyMap(), headers); 77 | } 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /examples/spring/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | 4 | webservice: 5 | hostname: webservice 6 | image: java 7 | command: java -jar /app/gs-rest-service-0.1.0.jar 8 | ports: 9 | - "8080:8080" 10 | volumes: 11 | - ./gs-rest-service/target:/app 12 | 13 | zookeeper: 14 | hostname: zookeeper 15 | image: confluentinc/cp-zookeeper:5.0.0 16 | environment: 17 | ZOOKEEPER_CLIENT_PORT: 2181 18 | zk_id: "1" 19 | 20 | kafka: 21 | hostname: kafka 22 | image: confluentinc/cp-kafka:5.0.0 23 | links: 24 | - zookeeper 25 | ports: 26 | - "9092:9092" 27 | environment: 28 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 29 | KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://:9092" 30 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 31 | 32 | schema_registry: 33 | hostname: schema_registry 34 | image: confluentinc/cp-schema-registry:5.0.0 35 | links: 36 | - kafka 37 | - zookeeper 38 | ports: 39 | - "8081:8081" 40 | environment: 41 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" 42 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 43 | 44 | connect: 45 | image: confluentinc/cp-kafka-connect:5.0.0 46 | hostname: connect 47 | depends_on: 48 | - webservice 49 | - zookeeper 50 | - kafka 51 | - schema_registry 52 | ports: 53 | - "8083:8083" 54 | environment: 55 | CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092' 56 | CONNECT_REST_ADVERTISED_HOST_NAME: connect 57 | CONNECT_REST_PORT: 8083 58 | CONNECT_GROUP_ID: compose-connect-group 59 | CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs 60 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 61 | CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 62 | CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets 63 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 64 | CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status 65 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 66 | CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter 67 | CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' 68 | CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter 69 | # CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter 70 | CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' 71 | CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter 72 | CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter 73 | CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181' 74 | CONNECT_PRODUCER_COMPRESSION_TYPE: "snappy" 75 | CONNECT_PLUGIN_PATH: /jars 76 | volumes: 77 | - ./jars:/jars 78 | - ./config:/config 79 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/src/main/java/hello/GreetingController.java: -------------------------------------------------------------------------------- 1 | package hello; 2 | 3 | 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.springframework.http.HttpHeaders; 6 | import org.springframework.web.bind.annotation.*; 7 | 8 | import java.util.Arrays; 9 | import java.util.concurrent.atomic.AtomicLong; 10 | 11 | import static hello.Calculation.Expression.Operation.*; 12 | 13 | 14 | @RestController 15 | @Slf4j 16 | public class GreetingController { 17 | private static final String template = "Hello, %s!"; 18 | private final AtomicLong counter = new AtomicLong(); 19 | 20 | @RequestMapping("/greeting") 21 | public Greeting greeting(@RequestParam(value = "name", defaultValue = "World") String name) { 22 | long id = counter.incrementAndGet(); 23 | log.info("Request /greeting: '{}'", id); 24 | Greeting greeting = new Greeting(); 25 | greeting.setId(id); 26 | greeting.setContent(String.format(template, name)); 27 | return greeting; 28 | } 29 | 30 | @RequestMapping("/reverse") 31 | public Greeting reverse(@RequestParam(value = "caps", required = false) Capitalisation caps, 32 | @RequestBody Greeting greetIn) { 33 | StringBuilder sb = new StringBuilder(greetIn.getContent()); 34 | String reverse = sb.reverse().toString(); 35 | Greeting greetOut = new Greeting(); 36 | if (caps == null) { 37 | greetOut.setContent(reverse); 38 | } else { 39 | switch (caps) { 40 | case UPPER: 41 | greetOut.setContent(reverse.toUpperCase()); 42 | break; 43 | case LOWER: 44 | greetOut.setContent(reverse.toLowerCase()); 45 | break; 46 | default: 47 | greetOut.setContent(reverse); 48 | } 49 | } 50 | return greetOut; 51 | } 52 | 53 | @RequestMapping(value = "/sum-plain", produces = "text/plain") 54 | public String sumPlain(@RequestParam(value = "val1", defaultValue = "1") double val1, 55 | @RequestParam(value = "val2", defaultValue = "1") double val2) { 56 | return String.format("%f + %f = %f", val1, val2, (val1 + val2)); 57 | } 58 | 59 | @RequestMapping(value = "/sum-xml", produces = "text/xml") 60 | public Calculation sumXml(@RequestParam(value = "val1", defaultValue = "1") double val1, 61 | @RequestParam(value = "val2", defaultValue = "1") double val2) { 62 | return new Calculation(SUM, Arrays.asList(val1, val2), (val1 + val2)); 63 | } 64 | 65 | @PostMapping(value = "/count") 66 | public void count(@RequestBody Object request, @RequestHeader HttpHeaders headers) { 67 | log.info("Request /count: '{}', {}", request, headers); 68 | } 69 | 70 | public static enum Capitalisation { 71 | UPPER, 72 | LOWER 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/RestSinkTaskTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import com.tm.kafka.connect.rest.http.Request; 4 | import com.tm.kafka.connect.rest.http.Response; 5 | import com.tm.kafka.connect.rest.http.executor.RequestExecutor; 6 | import com.tm.kafka.connect.rest.http.handler.ResponseHandler; 7 | import org.apache.kafka.connect.errors.RetriableException; 8 | import org.apache.kafka.connect.sink.SinkRecord; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.mockito.InjectMocks; 13 | import org.mockito.Mock; 14 | import org.mockito.junit.MockitoJUnitRunner; 15 | 16 | import java.util.Arrays; 17 | 18 | import static org.mockito.ArgumentMatchers.any; 19 | import static org.mockito.Mockito.mock; 20 | import static org.mockito.Mockito.times; 21 | import static org.mockito.Mockito.verify; 22 | import static org.mockito.Mockito.when; 23 | 24 | @RunWith(MockitoJUnitRunner.class) 25 | public class RestSinkTaskTest { 26 | 27 | @Mock 28 | Request.RequestFactory requestFactory; 29 | 30 | @Mock 31 | RequestExecutor executor; 32 | 33 | @Mock 34 | ResponseHandler responseHandler; 35 | 36 | @Mock 37 | SinkRecord sinkRecord; 38 | 39 | @Mock 40 | RestSinkConnectorConfig config; 41 | 42 | @InjectMocks 43 | RestSinkTask subject; 44 | 45 | @Before 46 | public void setUp() throws Exception { 47 | when(requestFactory.createRequest(any(), any())).thenReturn(mock(Request.class)); 48 | when(executor.execute(any())).thenReturn(mock(Response.class)); 49 | } 50 | 51 | @Test 52 | public void shouldRetryInfinitelyWhenMaxRetriesIsNegative() throws Exception { 53 | when(executor.execute(any())) 54 | .thenThrow(new RetriableException("Test")) 55 | .thenThrow(new RetriableException("Test")) 56 | .thenThrow(new RetriableException("Test")) 57 | .thenThrow(new RetriableException("Test")) 58 | .thenThrow(new RetriableException("Test")) 59 | .thenReturn(mock(Response.class)); 60 | 61 | subject.setMaxRetries(-1); 62 | subject.put(Arrays.asList(sinkRecord)); 63 | 64 | verify(executor, times(6)).execute(any()); 65 | } 66 | 67 | @Test 68 | public void shouldRetryOnErrorMaxRetriesTimes() throws Exception { 69 | when(executor.execute(any())) 70 | .thenThrow(new RetriableException("Test")) 71 | .thenThrow(new RetriableException("Test")) 72 | .thenThrow(new RetriableException("Test")) 73 | .thenThrow(new RetriableException("Test")) 74 | .thenThrow(new RetriableException("Test")) 75 | .thenThrow(new RetriableException("Test")) 76 | .thenThrow(new RetriableException("Test")) 77 | .thenReturn(mock(Response.class)); 78 | 79 | subject.setMaxRetries(3); 80 | subject.put(Arrays.asList(sinkRecord)); 81 | 82 | verify(executor, times(4)).execute(any()); 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/src/test/java/hello/GreetingControllerTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package hello; 17 | 18 | import org.junit.Test; 19 | import org.junit.runner.RunWith; 20 | import org.springframework.beans.factory.annotation.Autowired; 21 | import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; 22 | import org.springframework.boot.test.context.SpringBootTest; 23 | import org.springframework.http.MediaType; 24 | import org.springframework.test.context.junit4.SpringRunner; 25 | import org.springframework.test.web.servlet.MockMvc; 26 | 27 | import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; 28 | import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; 29 | import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; 30 | import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; 31 | import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; 32 | 33 | @RunWith(SpringRunner.class) 34 | @SpringBootTest 35 | @AutoConfigureMockMvc 36 | public class GreetingControllerTests { 37 | 38 | @Autowired 39 | private MockMvc mockMvc; 40 | 41 | @Test 42 | public void noParamGreetingShouldReturnDefaultMessage() throws Exception { 43 | 44 | this.mockMvc.perform(get("/greeting")).andDo(print()).andExpect(status().isOk()) 45 | .andExpect(jsonPath("$.content").value("Hello, World!")); 46 | 47 | this.mockMvc.perform(get("/greeting").param("name", "Spring Community")) 48 | .andDo(print()).andExpect(status().isOk()) 49 | .andExpect(jsonPath("$.content").value("Hello, Spring Community!")); 50 | 51 | this.mockMvc.perform(post("/count") 52 | .contentType(MediaType.APPLICATION_JSON) 53 | .content("{\"id\":1,\"content\":\"Hello, World 1!\"}")) 54 | //.param("name", "Spring Community")) 55 | .andDo(print()).andExpect(status().isOk()); 56 | 57 | this.mockMvc.perform(get("/greeting").param("name", "Spring Community")) 58 | .andDo(print()).andExpect(status().isOk()) 59 | .andExpect(jsonPath("$.content").value("Hello, Spring Community!")); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/AbstractValueProvider.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | 7 | import java.util.Collections; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | 12 | /** 13 | * This abstract class is a sensible base for value provider implementations. 14 | * It tracks the parameter set in use and ensures that the same parameter will only be looked up once per update 15 | * meaning it will have the same value throughout a payload. 16 | */ 17 | public abstract class AbstractValueProvider implements ValueProvider { 18 | 19 | protected Map parameterMap = new HashMap<>(); 20 | 21 | 22 | /** 23 | * Extract the values that will be used by the template engine from the last request-response cycle. 24 | * It is not necessary for parameters used by the value provider to come from the request or response, 25 | * and in some cases this method may do nothing. 26 | * 27 | * @param request The last request made. 28 | * @param response The last response received. 29 | */ 30 | protected abstract void extractValues(Request request, Response response); 31 | 32 | /** 33 | * Get the value for a given key. 34 | * This method is called if the key cannot be found in the current set of cached parameters. 35 | * 36 | * @param key the key to lookup 37 | * @return the value or null if the key is undefined. 38 | */ 39 | protected abstract String getValue(String key); 40 | 41 | /** 42 | * Update the parameter values based on the last request and response. 43 | * 44 | * @param request The last request made. 45 | * @param response The last response received. 46 | */ 47 | @Override 48 | public void update(Request request, Response response) { 49 | parameterMap.clear(); 50 | extractValues(request, response); 51 | } 52 | 53 | /** 54 | * Returns the value of the given key. 55 | * 56 | * @return The defined value or null if the key is undefined. 57 | */ 58 | @Override 59 | public String lookupValue(String key) { 60 | String value = parameterMap.getOrDefault(key, getValue(key)); 61 | parameterMap.put(key, value); 62 | return value; 63 | } 64 | 65 | /** 66 | * Get the key-value pairs that have been requested since the last update. 67 | * 68 | * @return The parameter map. 69 | */ 70 | @Override 71 | public Map getParameters() { 72 | return Collections.unmodifiableMap(parameterMap); 73 | } 74 | 75 | /** 76 | * Set the map of keys to values that will be used to generate the next template. 77 | * Note that the update method will overwrite these mappings. 78 | * This method would normally be used to set initial state. 79 | * 80 | * @param params The parameter map. 81 | */ 82 | @Override 83 | public void setParameters(Map params) { 84 | parameterMap.clear(); 85 | params.forEach((k, v) -> parameterMap.put(k, v.toString())); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | kafka-connect-rest 9 | com.tm.kafka 10 | 1.0.3 11 | 12 | 13 | kafka-connect-rest-plugin 14 | 15 | 16 | 17 | 18 | org.apache.kafka 19 | connect-api 20 | 21 | 22 | 23 | org.apache.kafka 24 | kafka-clients 25 | 26 | 27 | 28 | org.apache.kafka 29 | connect-transforms 30 | 31 | 32 | 33 | com.fasterxml.jackson.core 34 | jackson-databind 35 | 36 | 37 | 38 | org.apache.velocity 39 | velocity-engine-core 40 | 41 | 42 | 43 | com.squareup.okhttp3 44 | okhttp 45 | 46 | 47 | 48 | io.dropwizard.metrics 49 | metrics-core 50 | 51 | 52 | 53 | io.dropwizard.metrics 54 | metrics-jmx 55 | 56 | 57 | 58 | org.slf4j 59 | slf4j-api 60 | 61 | 62 | 63 | junit 64 | junit 65 | 66 | 67 | 68 | org.hamcrest 69 | hamcrest-library 70 | 71 | 72 | 73 | org.mockito 74 | mockito-core 75 | 76 | 77 | 78 | com.github.tomakehurst 79 | wiremock 80 | 81 | 82 | 83 | org.powermock 84 | powermock-core 85 | 86 | 87 | org.powermock 88 | powermock-module-junit4 89 | 90 | 91 | org.powermock 92 | powermock-api-mockito2 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | org.apache.maven.plugins 102 | maven-shade-plugin 103 | 104 | 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/ConstantPayloadGeneratorTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload; 2 | 3 | 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | 7 | import java.util.Collections; 8 | import java.util.Map; 9 | import java.util.stream.Collectors; 10 | import java.util.stream.Stream; 11 | 12 | import static org.hamcrest.Matchers.*; 13 | import static org.junit.Assert.assertThat; 14 | 15 | 16 | public class ConstantPayloadGeneratorTest { 17 | 18 | private static final Map CONFIG_PROPS = Stream.of(new String[][] { 19 | { ConstantPayloadGeneratorConfig.REQUEST_BODY_CONFIG, "{\"query\": \"select * from known_stars\"" }, 20 | { ConstantPayloadGeneratorConfig.REQUEST_HEADERS_CONFIG, "Content-Type:application/json, Accept:application/json" }, 21 | { ConstantPayloadGeneratorConfig.REQUEST_PARAMETER_NAMES_CONFIG, "priority, paged" }, 22 | { String.format(ConstantPayloadGeneratorConfig.REQUEST_PARAMETER_VALUE_CONFIG, "priority"), "MAXIMUM" }, 23 | { String.format(ConstantPayloadGeneratorConfig.REQUEST_PARAMETER_VALUE_CONFIG, "paged"), "FALSE" }, 24 | }).collect(Collectors.toMap(d -> d[0], d -> d[1])); 25 | 26 | private ConstantPayloadGenerator generator; 27 | 28 | 29 | @Before 30 | public void before() { 31 | generator = new ConstantPayloadGenerator(); 32 | } 33 | 34 | 35 | @Test 36 | public void testUpdate() { 37 | generator.configure(CONFIG_PROPS); 38 | assertThat(generator.update(null, null), equalTo(false)); 39 | } 40 | 41 | @Test 42 | public void testGetRequestBody() { 43 | generator.configure(CONFIG_PROPS); 44 | assertThat(generator.getRequestBody(), equalTo("{\"query\": \"select * from known_stars\"")); 45 | } 46 | 47 | @Test 48 | public void testGetRequestBody_configUndefined() { 49 | generator.configure(Collections.emptyMap()); 50 | assertThat(generator.getRequestBody(), equalTo("")); 51 | } 52 | 53 | @Test 54 | public void testGetRequestParameters() { 55 | generator.configure(CONFIG_PROPS); 56 | assertThat(generator.getRequestParameters(), allOf(hasEntry("priority", "MAXIMUM"), hasEntry("paged", "FALSE"))); 57 | } 58 | 59 | @Test 60 | public void testGetRequestParameters_configUndefined() { 61 | generator.configure(Collections.emptyMap()); 62 | assertThat(generator.getRequestParameters(), not(hasKey(anything()))); 63 | } 64 | 65 | @Test 66 | public void testGetRequestHeaders() { 67 | generator.configure(CONFIG_PROPS); 68 | assertThat(generator.getRequestHeaders(), allOf( 69 | hasEntry("Content-Type", "application/json"), hasEntry("Accept", "application/json"))); 70 | } 71 | 72 | @Test 73 | public void testGetRequestHeaders_configUndefined() { 74 | generator.configure(Collections.emptyMap()); 75 | assertThat(generator.getRequestHeaders(), not(hasKey(anything()))); 76 | } 77 | 78 | @Test 79 | public void testGetOffsets() { 80 | generator.configure(CONFIG_PROPS); 81 | assertThat(generator.getOffsets(), hasEntry(equalTo("timestamp"), instanceOf(Long.class))); 82 | } 83 | 84 | @Test 85 | public void testSetOffsets() { 86 | generator.configure(CONFIG_PROPS); 87 | generator.setOffsets(null); 88 | assertThat(generator.getOffsets(), notNullValue()); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/TemplatedPayloadGenerator.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import com.tm.kafka.connect.rest.http.payload.PayloadGenerator; 7 | import org.apache.kafka.common.Configurable; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Arrays; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | 16 | /** 17 | * This is a payload generator that attempts to fill in the placeholders in a template using values from a 18 | * ValueProvider instance. 19 | */ 20 | public class TemplatedPayloadGenerator implements PayloadGenerator, Configurable { 21 | 22 | private static Logger log = LoggerFactory.getLogger(RegexResponseValueProvider.class); 23 | 24 | private String requestBodyTemplate; 25 | private Map requestParameterTemplates; 26 | private Map requestParameterValues; 27 | private String requestBodyValue; 28 | private Map requestHeaderTemplates; 29 | private Map requestHeaderValues; 30 | private ValueProvider valueProvider; 31 | private TemplateEngine templateEngine; 32 | 33 | 34 | @Override 35 | public void configure(Map props) { 36 | final TemplatedPayloadGeneratorConfig config = new TemplatedPayloadGeneratorConfig(props); 37 | 38 | requestBodyTemplate = config.getRequestBodyTemplate(); 39 | requestParameterTemplates = config.getRequestParameterTemplates(); 40 | requestHeaderTemplates = config.getRequestHeaderTemplates(); 41 | 42 | requestParameterValues = new HashMap<>(requestParameterTemplates.size()); 43 | requestHeaderValues = new HashMap<>(requestHeaderTemplates.size()); 44 | 45 | valueProvider = config.getValueProvider(); 46 | templateEngine = config.getTemplateEngine(); 47 | 48 | populateValues(); 49 | } 50 | 51 | @Override 52 | public boolean update(Request request, Response response) { 53 | valueProvider.update(request,response); 54 | 55 | populateValues(); 56 | 57 | // False = Wait for the next poll cycle before calling again. 58 | return false; 59 | } 60 | 61 | @Override 62 | public String getRequestBody() { 63 | return requestBodyValue; 64 | } 65 | 66 | @Override 67 | public Map getRequestParameters() { 68 | return requestParameterValues; 69 | } 70 | 71 | @Override 72 | public Map getRequestHeaders() { 73 | return requestHeaderValues; 74 | } 75 | 76 | @Override 77 | public Map getOffsets() { 78 | return valueProvider.getParameters(); 79 | } 80 | 81 | @Override 82 | public void setOffsets(Map offsets) { 83 | valueProvider.setParameters(offsets); 84 | populateValues(); 85 | } 86 | 87 | private void populateValues() { 88 | requestBodyValue = templateEngine.renderTemplate(requestBodyTemplate, valueProvider); 89 | requestParameterTemplates 90 | .forEach((k, v) -> requestParameterValues.put(k, templateEngine.renderTemplate(v, valueProvider))); 91 | requestHeaderTemplates 92 | .forEach((k, v) -> requestHeaderValues.put(k, templateEngine.renderTemplate(v, valueProvider))); 93 | 94 | log.info("Body to be sent: {}", requestBodyValue); 95 | log.info("Parameters to be sent: {}", Arrays.toString(requestParameterValues.entrySet().toArray())); 96 | log.info("Headers to be sent: {}", Arrays.toString(requestHeaderValues.entrySet().toArray())); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/RegexResponseValueProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import org.junit.Test; 7 | 8 | import java.util.Collections; 9 | 10 | import static org.hamcrest.Matchers.*; 11 | import static org.junit.Assert.assertThat; 12 | import static org.mockito.Mockito.mock; 13 | import static org.mockito.Mockito.when; 14 | 15 | 16 | public class RegexResponseValueProviderTest { 17 | 18 | Request request = mock(Request.class); 19 | Response response = mock(Response.class); 20 | 21 | RegexResponseValueProvider provider = new RegexResponseValueProvider(); 22 | 23 | @Test 24 | public void extractValuesTest_wholeRegexFoundOnce() { 25 | provider.setRegexes(Collections.singletonMap("name", ".+")); 26 | when(response.getPayload()).thenReturn("Hello Big Ears"); 27 | provider.extractValues(request, response); 28 | assertThat(provider.getParameters(), hasEntry("name", "Hello Big Ears")); 29 | } 30 | 31 | @Test 32 | public void extractValuesTest_wholeRegexFoundMultiple() { 33 | provider.setRegexes(Collections.singletonMap("name", "\\w+")); 34 | when(response.getPayload()).thenReturn("Hello Big Ears"); 35 | provider.extractValues(request, response); 36 | assertThat(provider.getParameters(), hasEntry("name", "Hello,Big,Ears")); 37 | } 38 | 39 | @Test 40 | public void extractValuesTest_singleGroupFoundOnce() { 41 | provider.setRegexes(Collections.singletonMap("name", "Hello (.+)")); 42 | when(response.getPayload()).thenReturn("Hello Big Ears"); 43 | provider.extractValues(request, response); 44 | assertThat(provider.getParameters(), hasEntry("name", "Big Ears")); 45 | } 46 | 47 | @Test 48 | public void extractValuesTest_singleGroupFoundMultiple() { 49 | provider.setRegexes(Collections.singletonMap("name", "Hello (\\w+)")); 50 | when(response.getPayload()).thenReturn("Hello Noddy Hello Noddy"); 51 | provider.extractValues(request, response); 52 | assertThat(provider.getParameters(), hasEntry("name", "Noddy,Noddy")); 53 | } 54 | 55 | @Test 56 | public void extractValuesTest_multipleGroupsFound() { 57 | provider.setRegexes(Collections.singletonMap("name", "Hello (\\w+) (\\w+)")); 58 | when(response.getPayload()).thenReturn("Hello Big Ears"); 59 | provider.extractValues(request, response); 60 | assertThat(provider.getParameters(), hasEntry("name", "Big,Ears")); 61 | } 62 | 63 | @Test 64 | public void extractValuesTest_valueNotFound() { 65 | provider.setRegexes(Collections.singletonMap("name", "Hello (.*)")); 66 | when(response.getPayload()).thenReturn("Hi Big Ears"); 67 | provider.extractValues(request, response); 68 | assertThat(provider.getParameters(), hasEntry("name", null)); 69 | } 70 | 71 | @Test 72 | public void lookupValueTest_extracted() { 73 | provider.setRegexes(Collections.singletonMap("name", "Hello (.*)")); 74 | when(response.getPayload()).thenReturn("Hello Big Ears"); 75 | provider.extractValues(request, response); 76 | assertThat(provider.lookupValue("name"), equalTo("Big Ears")); 77 | } 78 | 79 | @Test 80 | public void lookupValueTest_fromEnvironment() { 81 | System.setProperty("test", "yeah"); 82 | assertThat(provider.lookupValue("test"), equalTo("yeah")); 83 | } 84 | 85 | @Test 86 | public void lookupValueTest_notDefined() { 87 | System.clearProperty("test"); 88 | assertThat(provider.lookupValue("test"), nullValue()); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/XPathResponseValueProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | import java.util.Collections; 10 | 11 | import static org.hamcrest.Matchers.*; 12 | import static org.junit.Assert.assertThat; 13 | import static org.mockito.Mockito.mock; 14 | import static org.mockito.Mockito.when; 15 | 16 | 17 | public class XPathResponseValueProviderTest { 18 | 19 | Request request = mock(Request.class); 20 | Response response = mock(Response.class); 21 | 22 | XPathResponseValueProvider provider = new XPathResponseValueProvider(); 23 | 24 | @Before 25 | public void before() { 26 | provider.configure(Collections.emptyMap()); 27 | } 28 | 29 | @Test 30 | public void extractValuesTest_oneMatch() { 31 | provider.setExpressions(Collections.singletonMap("name", "/greeting/name")); 32 | when(response.getPayload()).thenReturn("HelloBig Ears"); 33 | provider.extractValues(request, response); 34 | assertThat(provider.getParameters(), hasEntry("name", "Big Ears")); 35 | } 36 | 37 | @Test 38 | public void extractValuesTest_multipleMatches() { 39 | provider.setExpressions(Collections.singletonMap("name", "/greeting/name")); 40 | when(response.getPayload()).thenReturn("HelloBig EarsNoddy"); 41 | provider.extractValues(request, response); 42 | assertThat(provider.getParameters(), hasEntry("name", "Big Ears,Noddy")); 43 | } 44 | 45 | @Test 46 | public void extractValuesTest_noMatch() { 47 | provider.setExpressions(Collections.singletonMap("name", "/greeting/title")); 48 | when(response.getPayload()).thenReturn("HelloBig Ears"); 49 | provider.extractValues(request, response); 50 | assertThat(provider.getParameters(), hasEntry("name", null)); 51 | } 52 | 53 | @Test 54 | public void extractValuesTest_illegalXPath() { 55 | provider.setExpressions(Collections.singletonMap("name", "/[/]title")); 56 | when(response.getPayload()).thenReturn("HelloBig Ears"); 57 | provider.extractValues(request, response); 58 | assertThat(provider.getParameters(), not(hasKey(anything()))); 59 | } 60 | 61 | @Test 62 | public void extractValuesTest_illegalXML() { 63 | provider.setExpressions(Collections.singletonMap("name", "/greeting/title")); 64 | when(response.getPayload()).thenReturn("<"); 65 | provider.extractValues(request, response); 66 | assertThat(provider.getParameters(), not(hasKey(anything()))); 67 | } 68 | 69 | @Test 70 | public void lookupValueTest_extracted() { 71 | provider.setExpressions(Collections.singletonMap("name", "/greeting/name")); 72 | when(response.getPayload()).thenReturn("HelloBig Ears"); 73 | provider.extractValues(request, response); 74 | assertThat(provider.lookupValue("name"), equalTo("Big Ears")); 75 | } 76 | 77 | @Test 78 | public void lookupValueTest_fromEnvironment() { 79 | System.setProperty("test", "yeah"); 80 | assertThat(provider.lookupValue("test"), equalTo("yeah")); 81 | } 82 | 83 | @Test 84 | public void lookupValueTest_notDefined() { 85 | System.clearProperty("test"); 86 | assertThat(provider.lookupValue("test"), nullValue()); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/RegexResponseValueProviderConfig.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import org.apache.kafka.common.config.AbstractConfig; 5 | import org.apache.kafka.common.config.ConfigDef; 6 | import org.apache.kafka.common.config.ConfigDef.Importance; 7 | import org.apache.kafka.common.config.ConfigDef.Type; 8 | 9 | import java.util.Collections; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | 15 | public class RegexResponseValueProviderConfig extends AbstractConfig { 16 | 17 | public static final String RESPONSE_VAR_NAMES_CONFIG = "rest.source.response.var.names"; 18 | private static final String RESPONSE_VAR_NAMES_DOC = "A list of variable names to be used for substitution into the " + 19 | "HTTP request template. A regex must be defined for each variable and will be run against the HTTP response to " + 20 | "extract values for the next HTTP request."; 21 | private static final String RESPONSE_VAR_NAMES_DISPLAY = "Template variables for REST source connector."; 22 | private static final List RESPONSE_VAR_NAMES_DEFAULT = Collections.EMPTY_LIST; 23 | 24 | public static final String RESPONSE_VAR_REGEX_CONFIG = "rest.source.response.var.%s.regex"; 25 | private static final String RESPONSE_VAR_REGEX_DOC = "The regex used to extract the %s variable from the HTTP response. " + 26 | "This parameter can then be used in the next templated HTTP request."; 27 | private static final String RESPONSE_VAR_REGEX_DISPLAY = "Regex for %s variable for REST source connector."; 28 | private static final Object RESPONSE_VAR_REGEX_DEFAULT = ConfigDef.NO_DEFAULT_VALUE; 29 | 30 | 31 | private final Map responseVariableRegexs; 32 | 33 | 34 | protected RegexResponseValueProviderConfig(ConfigDef config, Map unparsedConfig) { 35 | super(config, unparsedConfig); 36 | 37 | List variableNames = getResponseVariableNames(); 38 | responseVariableRegexs = new HashMap<>(variableNames.size()); 39 | variableNames.forEach(key -> responseVariableRegexs.put(key, getString(String.format(RESPONSE_VAR_REGEX_CONFIG, key)))); 40 | } 41 | 42 | public RegexResponseValueProviderConfig(Map unparsedConfig) { 43 | this(conf(unparsedConfig), unparsedConfig); 44 | } 45 | 46 | public static ConfigDef conf(Map unparsedConfig) { 47 | String group = "REST_HTTP"; 48 | int orderInGroup = 0; 49 | ConfigDef config = new ConfigDef() 50 | .define(RESPONSE_VAR_NAMES_CONFIG, 51 | Type.LIST, 52 | RESPONSE_VAR_NAMES_DEFAULT, 53 | Importance.LOW, 54 | RESPONSE_VAR_NAMES_DOC, 55 | group, 56 | ++orderInGroup, 57 | ConfigDef.Width.SHORT, 58 | RESPONSE_VAR_NAMES_DISPLAY) 59 | ; 60 | 61 | // This is a bit hacky and there may be a better way of doing it, but I don't know it. 62 | // We need to create config items dynamically, based on the parameter names, 63 | // so we need a 2 pass parse of the config. 64 | List varNames = (List) config.parse(unparsedConfig).get(RESPONSE_VAR_NAMES_CONFIG); 65 | 66 | for(String varName : varNames) { 67 | config.define(String.format(RESPONSE_VAR_REGEX_CONFIG, varName), 68 | Type.STRING, 69 | RESPONSE_VAR_REGEX_DEFAULT, 70 | Importance.HIGH, 71 | String.format(RESPONSE_VAR_REGEX_DOC, varName), 72 | group, 73 | ++orderInGroup, 74 | ConfigDef.Width.SHORT, 75 | String.format(RESPONSE_VAR_REGEX_DISPLAY, varName)); 76 | } 77 | 78 | return(config); 79 | } 80 | 81 | public List getResponseVariableNames() { 82 | return this.getList(RESPONSE_VAR_NAMES_CONFIG); 83 | } 84 | 85 | public Map getResponseVariableRegexs() { 86 | return responseVariableRegexs; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kafka Connect REST connector 2 | === 3 | 4 | Building and running Spring example in docker 5 | --- 6 | Build the project and copy the jars to the example 7 | 8 | mvn clean install && \ 9 | cd examples/spring/gs-rest-service && \ 10 | mvn clean install && \ 11 | cd .. && \ 12 | cp ../../kafka-connect-rest-plugin/target/kafka-connect-rest-plugin-*-shaded.jar jars/ && \ 13 | cp ../../kafka-connect-transform-from-json/kafka-connect-transform-from-json-plugin/target/kafka-connect-transform-from-json-plugin-*-shaded.jar jars/ && \ 14 | cp ../../kafka-connect-transform-add-headers/target/kafka-connect-transform-add-headers-*-shaded.jar jars/ && \ 15 | cp ../../kafka-connect-transform-velocity-eval/target/kafka-connect-transform-velocity-eval-*-shaded.jar jars/ 16 | 17 | Bring up the docker containers 18 | 19 | docker-compose up -d 20 | 21 | Create the destination topic 22 | 23 | docker exec -it spring_connect_1 bash -c \ 24 | "kafka-topics --zookeeper zookeeper \ 25 | --topic restSourceDestinationTopic --create \ 26 | --replication-factor 1 --partitions 1" 27 | 28 | Configure the sink and source connectors 29 | 30 | curl -X POST \ 31 | -H 'Host: connect.example.com' \ 32 | -H 'Accept: application/json' \ 33 | -H 'Content-Type: application/json' \ 34 | http://localhost:8083/connectors -d @config/sink.json 35 | 36 | curl -X POST \ 37 | -H 'Host: connect.example.com' \ 38 | -H 'Accept: application/json' \ 39 | -H 'Content-Type: application/json' \ 40 | http://localhost:8083/connectors -d @config/source.json 41 | 42 | View the contents of the destination topic 43 | 44 | docker exec -it spring_connect_1 bash -c \ 45 | "kafka-avro-console-consumer --bootstrap-server kafka:9092 \ 46 | --topic restSourceDestinationTopic --from-beginning \ 47 | --property schema.registry.url=http://schema_registry:8081/" 48 | 49 | View the webserver logs 50 | 51 | docker logs -f spring_webservice_1 52 | 53 | Shutdown the docker containers 54 | 55 | docker-compose down 56 | cd ../.. 57 | 58 | #### If you don't want to use Avro 59 | 60 | Change CONNECT_VALUE_CONVERTER in the docker-compose.yml 61 | to org.apache.kafka.connect.storage.StringConverter if you don't want to use Avro. 62 | 63 | docker exec -it spring_connect_1 bash -c \ 64 | "kafka-console-consumer --bootstrap-server kafka:9092 \ 65 | --topic restSourceDestinationTopic --from-beginning" 66 | 67 | Building and running Google Cloud Function example in docker (currently untested) 68 | --- 69 | 70 | You will need gcloud installed and a GCP project with payments enabled. 71 | 72 | mvn clean install 73 | cd examples/gcf 74 | 75 | Replace '\' and '\' in rest.source.url in config/source.json. 76 | 77 | "rest.source.url": "https://\-\.cloudfunctions.net/hello", 78 | 79 | gcloud beta functions deploy hello --trigger-http 80 | 81 | curl -X POST http://https://-.cloudfunctions.net/hello -d 'name=Kafka Connect' 82 | 83 | docker-compose up -d 84 | 85 | docker exec -it gcf_connect_1 bash -c \ 86 | "kafka-topics --zookeeper zookeeper \ 87 | --topic restSourceDestinationTopic --create \ 88 | --replication-factor 1 --partitions 1" 89 | 90 | curl -X POST \ 91 | -H 'Host: connect.example.com' \ 92 | -H 'Accept: application/json' \ 93 | -H 'Content-Type: application/json' \ 94 | http://localhost:8083/connectors -d @config/source.json 95 | 96 | docker exec -it spring_connect_1 bash -c \ 97 | "kafka-avro-console-consumer --bootstrap-server kafka:9092 \ 98 | --topic restSourceDestinationTopic --from-beginning \ 99 | --property schema.registry.url=http://schema_registry:8081/" 100 | 101 | docker-compose down 102 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/XPathResponseValueProviderConfig.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import org.apache.kafka.common.config.AbstractConfig; 5 | import org.apache.kafka.common.config.ConfigDef; 6 | import org.apache.kafka.common.config.ConfigDef.Importance; 7 | import org.apache.kafka.common.config.ConfigDef.Type; 8 | 9 | import java.util.Collections; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | 15 | public class XPathResponseValueProviderConfig extends AbstractConfig { 16 | 17 | public static final String RESPONSE_VAR_NAMES_CONFIG = "rest.source.response.var.names"; 18 | private static final String RESPONSE_VAR_NAMES_DOC = "A list of variable names to be used for substitution into the " + 19 | "HTTP request template. An XPath expression must be defined for each variable and will be run against the HTTP " + 20 | "response to extract values for the next HTTP request."; 21 | private static final String RESPONSE_VAR_NAMES_DISPLAY = "Template variables for REST source connector."; 22 | private static final List RESPONSE_VAR_NAMES_DEFAULT = Collections.EMPTY_LIST; 23 | 24 | public static final String RESPONSE_VAR_XPATH_CONFIG = "rest.source.response.var.%s.xpath"; 25 | private static final String RESPONSE_VAR_XPATH_DOC = "The XPath expression used to extract the %s variable from the " + 26 | "HTTP response. This parameter can then be used in the next templated HTTP request."; 27 | private static final String RESPONSE_VAR_XPATH_DISPLAY = "XPath for %s variable for REST source connector."; 28 | private static final Object RESPONSE_VAR_XPATH_DEFAULT = ConfigDef.NO_DEFAULT_VALUE; 29 | 30 | 31 | private final Map responseVariableXPaths; 32 | 33 | 34 | protected XPathResponseValueProviderConfig(ConfigDef config, Map unparsedConfig) { 35 | super(config, unparsedConfig); 36 | 37 | List variableNames = getResponseVariableNames(); 38 | responseVariableXPaths = new HashMap<>(variableNames.size()); 39 | variableNames.forEach(key -> responseVariableXPaths.put(key, getString(String.format(RESPONSE_VAR_XPATH_CONFIG, key)))); 40 | } 41 | 42 | public XPathResponseValueProviderConfig(Map unparsedConfig) { 43 | this(conf(unparsedConfig), unparsedConfig); 44 | } 45 | 46 | public static ConfigDef conf(Map unparsedConfig) { 47 | String group = "REST_HTTP"; 48 | int orderInGroup = 0; 49 | ConfigDef config = new ConfigDef() 50 | .define(RESPONSE_VAR_NAMES_CONFIG, 51 | Type.LIST, 52 | RESPONSE_VAR_NAMES_DEFAULT, 53 | Importance.LOW, 54 | RESPONSE_VAR_NAMES_DOC, 55 | group, 56 | ++orderInGroup, 57 | ConfigDef.Width.SHORT, 58 | RESPONSE_VAR_NAMES_DISPLAY) 59 | ; 60 | 61 | // This is a bit hacky and there may be a better way of doing it, but I don't know it. 62 | // We need to create config items dynamically, based on the parameter names, 63 | // so we need a 2 pass parse of the config. 64 | List varNames = (List) config.parse(unparsedConfig).get(RESPONSE_VAR_NAMES_CONFIG); 65 | 66 | for(String varName : varNames) { 67 | config.define(String.format(RESPONSE_VAR_XPATH_CONFIG, varName), 68 | Type.STRING, 69 | RESPONSE_VAR_XPATH_DEFAULT, 70 | Importance.HIGH, 71 | String.format(RESPONSE_VAR_XPATH_DOC, varName), 72 | group, 73 | ++orderInGroup, 74 | ConfigDef.Width.SHORT, 75 | String.format(RESPONSE_VAR_XPATH_DISPLAY, varName)); 76 | } 77 | 78 | return(config); 79 | } 80 | 81 | public List getResponseVariableNames() { 82 | return this.getList(RESPONSE_VAR_NAMES_CONFIG); 83 | } 84 | 85 | public Map getResponseVariableXPaths() { 86 | return responseVariableXPaths; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/executor/OkHttpRequestExecutor.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.executor; 2 | 3 | 4 | import okhttp3.ConnectionPool; 5 | import okhttp3.Headers; 6 | import okhttp3.MediaType; 7 | import okhttp3.OkHttpClient; 8 | import okhttp3.RequestBody; 9 | import org.apache.kafka.common.Configurable; 10 | import org.apache.kafka.connect.errors.RetriableException; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.io.IOException; 15 | import java.io.UnsupportedEncodingException; 16 | import java.net.URLEncoder; 17 | import java.util.Map; 18 | import java.util.concurrent.TimeUnit; 19 | import java.util.stream.Collectors; 20 | import java.util.stream.Stream; 21 | 22 | 23 | public class OkHttpRequestExecutor implements RequestExecutor, Configurable { 24 | 25 | private static Logger log = LoggerFactory.getLogger(OkHttpRequestExecutor.class); 26 | 27 | private OkHttpClient client; 28 | 29 | 30 | @Override 31 | public void configure(Map props) { 32 | final OkHttpRequestExecutorConfig config = new OkHttpRequestExecutorConfig(props); 33 | 34 | client = new OkHttpClient.Builder() 35 | .connectionPool(new ConnectionPool(config.getMaxIdleConnections(), config.getKeepAliveDuration(), TimeUnit.MILLISECONDS)) 36 | .connectTimeout(config.getConnectionTimeout(), TimeUnit.MILLISECONDS) 37 | .readTimeout(config.getReadTimeout(), TimeUnit.MILLISECONDS) 38 | .build(); 39 | } 40 | 41 | @Override 42 | public com.tm.kafka.connect.rest.http.Response execute(com.tm.kafka.connect.rest.http.Request request) throws IOException { 43 | okhttp3.Request.Builder builder = new okhttp3.Request.Builder() 44 | .url(createUrl(request.getUrl(), request.getParameters())) 45 | .headers(Headers.of(headersToArray(request.getHeaders()))); 46 | 47 | if ("GET".equalsIgnoreCase(request.getMethod())) { 48 | builder.get(); 49 | } else { 50 | builder.method(request.getMethod(), RequestBody.create( 51 | MediaType.parse(request.getHeaders().getOrDefault("Content-Type", "")), 52 | request.getBody()) 53 | ); 54 | } 55 | 56 | okhttp3.Request okRequest = builder.build(); 57 | log.trace("Making request to: " + request); 58 | 59 | try (okhttp3.Response okResponse = client.newCall(okRequest).execute()) { 60 | 61 | return new com.tm.kafka.connect.rest.http.Response( 62 | okResponse.code(), 63 | okResponse.headers().toMultimap(), 64 | okResponse.body() != null ? okResponse.body().string() : null 65 | ); 66 | } catch (IOException e) { 67 | throw new RetriableException(e.getMessage(), e); 68 | } 69 | } 70 | 71 | private String createUrl(String url, Map parameters) { 72 | if (parameters == null || parameters.isEmpty()) { 73 | return url; 74 | } 75 | 76 | String format = url.endsWith("?") ? "%s&%s" : "%s?%s"; 77 | return String.format(format, url, parametersToString(parameters)); 78 | } 79 | 80 | private String parametersToString(final Map parameters) { 81 | return parameters.entrySet().stream() 82 | .map(e -> parameterToString(e.getKey(), e.getValue())) 83 | .collect(Collectors.joining("&")); 84 | } 85 | 86 | private String parameterToString(final String key, final String value) { 87 | try { 88 | return key.trim() + "=" + URLEncoder.encode(value.trim(), "UTF-8"); 89 | } catch (UnsupportedEncodingException ex) { 90 | // This should never happen! 91 | log.warn("Unable to encode URL parameter as UTF-8 (UTF-8 not supported)"); 92 | return key.trim() + "=" + value.trim(); 93 | } 94 | } 95 | 96 | private String[] headersToArray(final Map headers) { 97 | return headers.entrySet() 98 | .stream() 99 | .flatMap(e -> Stream.of(e.getKey(), e.getValue())) 100 | .toArray(String[]::new); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /kafka-connect-transform-from-json/kafka-connect-transform-from-json-plugin/src/main/java/org/apache/kafka/connect/transforms/FromJson.java: -------------------------------------------------------------------------------- 1 | package org.apache.kafka.connect.transforms; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import hello.Greeting; 5 | import io.confluent.connect.avro.AvroData; 6 | import io.confluent.connect.avro.AvroDataConfig; 7 | import org.apache.kafka.common.config.ConfigDef; 8 | import org.apache.kafka.common.config.ConfigException; 9 | import org.apache.kafka.connect.connector.ConnectRecord; 10 | import org.apache.kafka.connect.data.Schema; 11 | import org.apache.kafka.connect.data.SchemaAndValue; 12 | import org.apache.kafka.connect.errors.DataException; 13 | import org.apache.kafka.connect.transforms.util.SimpleConfig; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.io.IOException; 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | public abstract class FromJson> implements Transformation { 22 | 23 | private static Logger log = LoggerFactory.getLogger(FromJson.class); 24 | 25 | private static final String CLASS_CONFIG = "message.class"; 26 | private static final String CLASS_DOC = "Java class for the JSON object."; 27 | 28 | private static final ConfigDef CONFIG_DEF = new ConfigDef() 29 | .define(CLASS_CONFIG, ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, CLASS_DOC); 30 | 31 | private final ObjectMapper mapper = new ObjectMapper(); 32 | private Map configs = new HashMap<>(); 33 | private AvroData avroData = new AvroData(new AvroDataConfig(configs)); 34 | private Class clazz; 35 | 36 | @Override 37 | public void configure(Map props) { 38 | final SimpleConfig config = new SimpleConfig(CONFIG_DEF, props); 39 | try { 40 | clazz = Class.forName(config.getString(CLASS_CONFIG)); 41 | } catch (ClassNotFoundException e) { 42 | throw new ConfigException(CLASS_CONFIG, e); 43 | } 44 | } 45 | 46 | @Override 47 | public R apply(R record) { 48 | try { 49 | Object updatedValue = mapper.readValue((String) operatingValue(record), clazz); 50 | SchemaAndValue s = avroData.toConnectData(Greeting.getClassSchema(), updatedValue); 51 | return newRecord(record, s.schema(), s.value()); 52 | } catch (IOException e) { 53 | throw new DataException("", e); 54 | } 55 | } 56 | 57 | @Override 58 | public void close() { 59 | } 60 | 61 | @Override 62 | public ConfigDef config() { 63 | return CONFIG_DEF; 64 | } 65 | 66 | protected abstract Schema operatingSchema(R record); 67 | 68 | protected abstract Object operatingValue(R record); 69 | 70 | protected abstract R newRecord(R record, Schema updatedSchema, Object updatedValue); 71 | 72 | public static class Key> extends FromJson { 73 | @Override 74 | protected Schema operatingSchema(R record) { 75 | return record.keySchema(); 76 | } 77 | 78 | @Override 79 | protected Object operatingValue(R record) { 80 | return record.key(); 81 | } 82 | 83 | @Override 84 | protected R newRecord(R record, Schema updatedSchema, Object updatedValue) { 85 | return record.newRecord(record.topic(), record.kafkaPartition(), 86 | updatedSchema, updatedValue, record.valueSchema(), record.value(), record.timestamp()); 87 | } 88 | } 89 | 90 | public static class Value> extends FromJson { 91 | @Override 92 | protected Schema operatingSchema(R record) { 93 | return record.valueSchema(); 94 | } 95 | 96 | @Override 97 | protected Object operatingValue(R record) { 98 | return record.value(); 99 | } 100 | 101 | @Override 102 | protected R newRecord(R record, Schema updatedSchema, Object updatedValue) { 103 | return record.newRecord(record.topic(), record.kafkaPartition(), 104 | record.keySchema(), record.key(), updatedSchema, updatedValue, record.timestamp()); 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/RestSinkTask.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import com.tm.kafka.connect.rest.http.Request; 4 | import com.tm.kafka.connect.rest.http.Response; 5 | import com.tm.kafka.connect.rest.http.executor.RequestExecutor; 6 | import com.tm.kafka.connect.rest.http.handler.ResponseHandler; 7 | import org.apache.kafka.connect.errors.RetriableException; 8 | import org.apache.kafka.connect.header.Header; 9 | import org.apache.kafka.connect.sink.SinkRecord; 10 | import org.apache.kafka.connect.sink.SinkTask; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.Collection; 15 | import java.util.Map; 16 | 17 | import static com.tm.kafka.connect.rest.metrics.Metrics.RETRIABLE_ERROR_METRIC; 18 | import static com.tm.kafka.connect.rest.metrics.Metrics.UNRETRIABLE_ERROR_METRIC; 19 | import static com.tm.kafka.connect.rest.metrics.Metrics.increaseCounter; 20 | 21 | public class RestSinkTask extends SinkTask { 22 | 23 | private static Logger log = LoggerFactory.getLogger(RestSinkTask.class); 24 | 25 | private Long retryBackoff; 26 | private Integer maxRetries; 27 | private Request.RequestFactory requestFactory; 28 | private Map headers; 29 | private RequestExecutor executor; 30 | private ResponseHandler responseHandler; 31 | private String taskName = ""; 32 | 33 | @Override 34 | public void start(Map map) { 35 | RestSinkConnectorConfig connectorConfig = new RestSinkConnectorConfig(map); 36 | taskName = map.getOrDefault("name", "unknown"); 37 | requestFactory = new Request.RequestFactory(connectorConfig.getUrl(), connectorConfig.getMethod()); 38 | headers = connectorConfig.getRequestHeaders(); 39 | retryBackoff = connectorConfig.getRetryBackoff(); 40 | maxRetries = connectorConfig.getMaxRetries(); 41 | responseHandler = connectorConfig.getResponseHandler(); 42 | executor = connectorConfig.getRequestExecutor(); 43 | } 44 | 45 | @Override 46 | public void put(Collection records) { 47 | for (SinkRecord record : records) { 48 | ExecutionContext ctx = ExecutionContext.create(taskName); 49 | int retries = maxRetries; 50 | while (maxRetries < 0 || retries-- >= 0) { 51 | try { 52 | String payload = (String) record.value(); 53 | Request request = requestFactory.createRequest(payload, headers); 54 | 55 | Map headers = request.getHeaders(); 56 | if (record.headers() != null) { 57 | for (Header header : record.headers()) { 58 | headers.put(header.key(), String.valueOf(header.value())); 59 | } 60 | } 61 | if (log.isTraceEnabled()) { 62 | log.info("Request to: {}, Offset: {}", request.getUrl(), record.kafkaOffset()); 63 | } 64 | 65 | Response response = executor.execute(request); 66 | 67 | if (log.isTraceEnabled()) { 68 | log.info("Response: {}, Request: {}", response, request); 69 | } 70 | 71 | responseHandler.handle(response, ctx); 72 | 73 | break; 74 | } catch (RetriableException e) { 75 | log.error("HTTP call failed", e); 76 | increaseCounter(RETRIABLE_ERROR_METRIC, ctx); 77 | try { 78 | Thread.sleep(retryBackoff); 79 | log.error("Retrying"); 80 | } catch (Exception ignored) { 81 | // Ignored 82 | } 83 | } catch (Exception e) { 84 | log.error("HTTP call execution failed " + e.getMessage(), e); 85 | increaseCounter(UNRETRIABLE_ERROR_METRIC, ctx); 86 | break; 87 | } 88 | } 89 | } 90 | } 91 | 92 | @Override 93 | public void stop() { 94 | log.debug("Stopping sink task, setting client to null"); 95 | } 96 | 97 | @Override 98 | public String version() { 99 | return VersionUtil.getVersion(); 100 | } 101 | 102 | void setRetryBackoff(long backoff) { 103 | this.retryBackoff = backoff; 104 | } 105 | 106 | void setMaxRetries(int retries) { 107 | this.maxRetries = retries; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/RegexResponseValueProvider.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import org.apache.kafka.common.Configurable; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | import java.util.regex.Matcher; 13 | import java.util.regex.Pattern; 14 | 15 | 16 | /** 17 | * Lookup values used to populate dynamic payloads. 18 | * These values will be substituted into the payload template. 19 | * 20 | * This implementation uses Regular Expressions to extract values from the HTTP response, 21 | * and if not found looks them up in the System properties and then in environment variables. 22 | */ 23 | public class RegexResponseValueProvider extends EnvironmentValueProvider implements Configurable { 24 | 25 | private static Logger log = LoggerFactory.getLogger(RegexResponseValueProvider.class); 26 | 27 | public static final String MULTI_VALUE_SEPARATOR = ","; 28 | 29 | private Map patterns; 30 | 31 | 32 | /** 33 | * Configure this instance after creation. 34 | * 35 | * @param props The configuration properties 36 | */ 37 | @Override 38 | public void configure(Map props) { 39 | final RegexResponseValueProviderConfig config = new RegexResponseValueProviderConfig(props); 40 | setRegexes(config.getResponseVariableRegexs()); 41 | } 42 | 43 | /** 44 | * Extract values from the response using the regular expressions 45 | * 46 | * @param request The last request made. 47 | * @param response The last response received. 48 | */ 49 | @Override 50 | protected void extractValues(Request request, Response response) { 51 | String resp = response.getPayload(); 52 | patterns.forEach((key, pat) -> parameterMap.put(key, extractValue(key, resp, pat))); 53 | } 54 | 55 | /** 56 | * Set the RegExs to be used for value extraction. 57 | * 58 | * @param regexes A map of key names to regular expressions 59 | */ 60 | protected void setRegexes(Map regexes) { 61 | patterns = new HashMap<>(regexes.size()); 62 | parameterMap = new HashMap<>(patterns.size()); 63 | regexes.forEach((k,v) -> patterns.put(k, Pattern.compile(v))); 64 | } 65 | 66 | /** 67 | * Extract the value for a given key. 68 | * Where the RegEx yeilds more than one result a comma seperated list will be returned. 69 | * If the RegEx has one or more groups then their contents will each be added to the returned list. 70 | * If the RegEx has no groups then the match for the entire RegEx will be added. 71 | * If the RegEx matches multiple times then each match will be processed and added to the list (as above). 72 | * 73 | * @param key The name of the key 74 | * @param resp The response to extract a value from 75 | * @param pattern The compiled RegEx used to find the value 76 | * @return Return the value, or null if it wasn't found 77 | */ 78 | private String extractValue(String key, String resp, Pattern pattern) { 79 | Matcher matcher = pattern.matcher(resp); 80 | StringBuilder values = new StringBuilder(); 81 | // Iterate over each place where the regex matches 82 | while(matcher.find()) { 83 | if(values.length() > 0) { 84 | values.append(MULTI_VALUE_SEPARATOR); 85 | } 86 | if(matcher.groupCount() == 0) { 87 | // if the regex has no groups then the whole thing is the value 88 | values.append(matcher.group()); 89 | } else { 90 | // If the regex has one or more groups then append them in order 91 | for(int g = 1; g <= matcher.groupCount(); g++) { 92 | if(g > 1) { 93 | values.append(MULTI_VALUE_SEPARATOR); 94 | } 95 | values.append(matcher.group(g)); 96 | } 97 | } 98 | } 99 | 100 | String value = (values.length() != 0) ? values.toString() : null; 101 | log.info("Variable {} was assigned the value {}", key, value); 102 | return value; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/executor/OkHttpRequestExecutorConfig.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.executor; 2 | 3 | 4 | import org.apache.kafka.common.config.AbstractConfig; 5 | import org.apache.kafka.common.config.ConfigDef; 6 | import org.apache.kafka.common.config.ConfigDef.Importance; 7 | import org.apache.kafka.common.config.ConfigDef.Type; 8 | 9 | import java.util.*; 10 | 11 | 12 | public class OkHttpRequestExecutorConfig extends AbstractConfig { 13 | 14 | public static final String HTTP_CONNECTION_TIMEOUT_CONFIG = "rest.http.connection.connection.timeout"; 15 | private static final String HTTP_CONNECTION_TIMEOUT_DISPLAY = "HTTP connection timeout in milliseconds"; 16 | private static final String HTTP_CONNECTION_TIMEOUT_DOC = "HTTP connection timeout in milliseconds"; 17 | private static final long HTTP_CONNECTION_TIMEOUT_DEFAULT = 2000; 18 | 19 | public static final String HTTP_READ_TIMEOUT_CONFIG = "rest.http.connection.read.timeout"; 20 | private static final String HTTP_READ_TIMEOUT_DISPLAY = "HTTP read timeout in milliseconds"; 21 | private static final String HTTP_READ_TIMEOUT_DOC = "HTTP read timeout in milliseconds"; 22 | private static final long HTTP_READ_TIMEOUT_DEFAULT = 2000; 23 | 24 | public static final String HTTP_KEEP_ALIVE_DURATION_CONFIG = "rest.http.connection.keep.alive.ms"; 25 | private static final String HTTP_KEEP_ALIVE_DURATION_DISPLAY = "Keep alive in milliseconds"; 26 | private static final String HTTP_KEEP_ALIVE_DURATION_DOC = "For how long keep HTTP connection should be keept alive in milliseconds"; 27 | private static final long HTTP_KEEP_ALIVE_DURATION_DEFAULT = 300000; // 5 minutes 28 | 29 | public static final String HTTP_MAX_IDLE_CONNECTION_CONFIG = "rest.http.connection.max.idle"; 30 | private static final String HTTP_MAX_IDLE_CONNECTION_DISPLAY = "Number of idle connections"; 31 | private static final String HTTP_MAX_IDLE_CONNECTION_DOC = "How many idle connections per host can be keept opened"; 32 | private static final int HTTP_MAX_IDLE_CONNECTION_DEFAULT = 5; 33 | 34 | 35 | protected OkHttpRequestExecutorConfig(ConfigDef config, Map parsedConfig) { 36 | super(config, parsedConfig); 37 | } 38 | 39 | public OkHttpRequestExecutorConfig(Map parsedConfig) { 40 | this(conf(), parsedConfig); 41 | } 42 | 43 | public static ConfigDef conf() { 44 | String group = "REST_HTTP"; 45 | int orderInGroup = 0; 46 | return new ConfigDef() 47 | .define(HTTP_CONNECTION_TIMEOUT_CONFIG, 48 | Type.LONG, 49 | HTTP_CONNECTION_TIMEOUT_DEFAULT, 50 | Importance.LOW, 51 | HTTP_CONNECTION_TIMEOUT_DOC, 52 | group, 53 | ++orderInGroup, 54 | ConfigDef.Width.NONE, 55 | HTTP_CONNECTION_TIMEOUT_DISPLAY) 56 | 57 | .define(HTTP_READ_TIMEOUT_CONFIG, 58 | Type.LONG, 59 | HTTP_READ_TIMEOUT_DEFAULT, 60 | Importance.LOW, 61 | HTTP_READ_TIMEOUT_DOC, 62 | group, 63 | ++orderInGroup, 64 | ConfigDef.Width.NONE, 65 | HTTP_READ_TIMEOUT_DISPLAY) 66 | 67 | .define(HTTP_KEEP_ALIVE_DURATION_CONFIG, 68 | Type.LONG, 69 | HTTP_KEEP_ALIVE_DURATION_DEFAULT, 70 | Importance.LOW, 71 | HTTP_KEEP_ALIVE_DURATION_DOC, 72 | group, 73 | ++orderInGroup, 74 | ConfigDef.Width.NONE, 75 | HTTP_KEEP_ALIVE_DURATION_DISPLAY) 76 | 77 | .define(HTTP_MAX_IDLE_CONNECTION_CONFIG, 78 | Type.INT, 79 | HTTP_MAX_IDLE_CONNECTION_DEFAULT, 80 | Importance.LOW, 81 | HTTP_MAX_IDLE_CONNECTION_DOC, 82 | group, 83 | ++orderInGroup, 84 | ConfigDef.Width.NONE, 85 | HTTP_MAX_IDLE_CONNECTION_DISPLAY) 86 | ; 87 | } 88 | 89 | public long getReadTimeout() { 90 | return this.getLong(HTTP_READ_TIMEOUT_CONFIG); 91 | } 92 | 93 | public long getConnectionTimeout() { 94 | return this.getLong(HTTP_CONNECTION_TIMEOUT_CONFIG); 95 | } 96 | 97 | public long getKeepAliveDuration() { 98 | return this.getLong(HTTP_KEEP_ALIVE_DURATION_CONFIG); 99 | } 100 | 101 | public int getMaxIdleConnections() { 102 | return this.getInt(HTTP_MAX_IDLE_CONNECTION_CONFIG); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /kafka-connect-transform-velocity-eval/src/main/java/org/apache/kafka/connect/transforms/VelocityEval.java: -------------------------------------------------------------------------------- 1 | package org.apache.kafka.connect.transforms; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigException; 6 | import org.apache.kafka.connect.connector.ConnectRecord; 7 | import org.apache.kafka.connect.data.Schema; 8 | import org.apache.kafka.connect.transforms.util.SimpleConfig; 9 | import org.apache.velocity.VelocityContext; 10 | import org.apache.velocity.app.Velocity; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.io.IOException; 15 | import java.io.StringWriter; 16 | import java.util.Map; 17 | 18 | public abstract class VelocityEval> implements Transformation { 19 | 20 | private static Logger log = LoggerFactory.getLogger(VelocityEval.class); 21 | 22 | private VelocityContext globalContext; 23 | private String template; 24 | 25 | private ObjectMapper objectMapper = new ObjectMapper(); 26 | 27 | private static final String TEMPLATE_CONFIG = "template"; 28 | private static final String TEMPLATE_DOC = "Velocity template."; 29 | 30 | private static final String CONTEXT_CONFIG = "context"; 31 | private static final String CONTEXT_DOC = "JSON string of key value pairs added to Velocity context. " + 32 | "E.g. '{\"key1\":\"val1\",\"key2\":2}"; 33 | 34 | private static final ConfigDef CONFIG_DEF = new ConfigDef() 35 | .define(CONTEXT_CONFIG, ConfigDef.Type.STRING, ConfigDef.NO_DEFAULT_VALUE, ConfigDef.Importance.MEDIUM, CONTEXT_DOC) 36 | .define(TEMPLATE_CONFIG, ConfigDef.Type.STRING, ConfigDef.NO_DEFAULT_VALUE, ConfigDef.Importance.MEDIUM, TEMPLATE_DOC); 37 | 38 | @Override 39 | public void configure(Map props) { 40 | final SimpleConfig config = new SimpleConfig(CONFIG_DEF, props); 41 | 42 | // Workaround Velocity classloader issue 43 | Thread thread = Thread.currentThread(); 44 | ClassLoader loader = thread.getContextClassLoader(); 45 | thread.setContextClassLoader(this.getClass().getClassLoader()); 46 | try { 47 | Velocity.init(); 48 | } finally { 49 | thread.setContextClassLoader(loader); 50 | } 51 | 52 | globalContext = new VelocityContext(); 53 | try { 54 | @SuppressWarnings("unchecked") 55 | Map map = objectMapper.readValue(config.getString(CONTEXT_CONFIG), Map.class); 56 | for (Map.Entry entry : map.entrySet()) { 57 | globalContext.put(entry.getKey(), entry.getValue()); 58 | } 59 | } catch (IOException e) { 60 | throw new ConfigException("", e); 61 | } 62 | template = config.getString(TEMPLATE_CONFIG); 63 | } 64 | 65 | @Override 66 | public R apply(R record) { 67 | StringWriter sw = new StringWriter(); 68 | 69 | VelocityContext context = new VelocityContext(globalContext); 70 | 71 | context.put("r", record); 72 | context.put("topic", record.topic()); 73 | context.put("partition", record.kafkaPartition()); 74 | context.put("key", record.key()); 75 | context.put("timestamp", record.timestamp() == null ? 0 : record.timestamp()); 76 | context.put("schema", operatingSchema(record)); 77 | context.put("value", operatingValue(record)); 78 | 79 | Velocity.evaluate(context, sw, "", template); 80 | return newRecord(record, Schema.STRING_SCHEMA, sw.toString()); 81 | } 82 | 83 | @Override 84 | public void close() { 85 | } 86 | 87 | @Override 88 | public ConfigDef config() { 89 | return CONFIG_DEF; 90 | } 91 | 92 | protected abstract Schema operatingSchema(R record); 93 | 94 | protected abstract Object operatingValue(R record); 95 | 96 | protected abstract R newRecord(R record, Schema updatedSchema, Object updatedValue); 97 | 98 | public static class Key> extends VelocityEval { 99 | @Override 100 | protected Schema operatingSchema(R record) { 101 | return record.keySchema(); 102 | } 103 | 104 | @Override 105 | protected Object operatingValue(R record) { 106 | return record.key(); 107 | } 108 | 109 | @Override 110 | protected R newRecord(R record, Schema updatedSchema, Object updatedValue) { 111 | return record.newRecord(record.topic(), record.kafkaPartition(), 112 | updatedSchema, updatedValue, record.valueSchema(), record.value(), record.timestamp()); 113 | } 114 | } 115 | 116 | public static class Value> extends VelocityEval { 117 | @Override 118 | protected Schema operatingSchema(R record) { 119 | return record.valueSchema(); 120 | } 121 | 122 | @Override 123 | protected Object operatingValue(R record) { 124 | return record.value(); 125 | } 126 | 127 | @Override 128 | protected R newRecord(R record, Schema updatedSchema, Object updatedValue) { 129 | return record.newRecord(record.topic(), record.kafkaPartition(), 130 | record.keySchema(), record.key(), updatedSchema, updatedValue, record.timestamp()); 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/RestSourceTask.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | import com.tm.kafka.connect.rest.http.Request; 4 | import com.tm.kafka.connect.rest.http.Response; 5 | import com.tm.kafka.connect.rest.http.executor.RequestExecutor; 6 | import com.tm.kafka.connect.rest.http.handler.ResponseHandler; 7 | import com.tm.kafka.connect.rest.http.payload.PayloadGenerator; 8 | import com.tm.kafka.connect.rest.selector.TopicSelector; 9 | import org.apache.kafka.common.config.ConfigException; 10 | import org.apache.kafka.connect.data.Schema; 11 | import org.apache.kafka.connect.data.SchemaBuilder; 12 | import org.apache.kafka.connect.errors.ConnectException; 13 | import org.apache.kafka.connect.source.SourceRecord; 14 | import org.apache.kafka.connect.source.SourceTask; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import java.util.*; 19 | 20 | import static com.tm.kafka.connect.rest.metrics.Metrics.ERROR_METRIC; 21 | import static com.tm.kafka.connect.rest.metrics.Metrics.increaseCounter; 22 | import static java.lang.System.currentTimeMillis; 23 | 24 | public class RestSourceTask extends SourceTask { 25 | 26 | private static Logger log = LoggerFactory.getLogger(RestSourceTask.class); 27 | 28 | private RestSourceConnectorConfig connectorConfig; 29 | private Long pollInterval; 30 | 31 | private Long lastPollTime = 0L; 32 | private String taskName; 33 | private RequestExecutor executor; 34 | private Request.RequestFactory requestFactory; 35 | private PayloadGenerator payloadGenerator; 36 | private ResponseHandler responseHandler; 37 | private TopicSelector topicSelector; 38 | private Map sourcePartition; 39 | private ExecutionContext ctx; 40 | 41 | @Override 42 | public void start(Map properties) { 43 | log.info("Starting REST source task"); 44 | try { 45 | connectorConfig = new RestSourceConnectorConfig(properties); 46 | } catch (ConfigException ex) { 47 | throw new ConnectException("Couldn't start RestSourceTask due to configuration error", ex); 48 | } 49 | 50 | taskName = properties.getOrDefault("name", "unknown"); 51 | ctx = ExecutionContext.create(taskName); 52 | 53 | pollInterval = connectorConfig.getPollInterval(); 54 | String url = connectorConfig.getUrl(); 55 | requestFactory = new Request.RequestFactory(url, connectorConfig.getMethod()); 56 | payloadGenerator = connectorConfig.getPayloadGenerator(); 57 | responseHandler = connectorConfig.getResponseHandler(); 58 | executor = connectorConfig.getRequestExecutor(); 59 | topicSelector = connectorConfig.getTopicSelector(); 60 | 61 | sourcePartition = Collections.singletonMap("URL", url); 62 | Map offsets = context.offsetStorageReader().offset(sourcePartition); 63 | if(offsets != null) { 64 | log.info("Loaded Offsets: " + Arrays.toString(offsets.entrySet().toArray())); 65 | payloadGenerator.setOffsets(offsets); 66 | } 67 | } 68 | 69 | @Override 70 | public List poll() throws InterruptedException { 71 | long millis = pollInterval - (currentTimeMillis() - lastPollTime); 72 | if (millis > 0) { 73 | Thread.sleep(millis); 74 | } 75 | 76 | ArrayList records = new ArrayList<>(); 77 | boolean makeAnotherRequest = true; 78 | 79 | try { 80 | while (makeAnotherRequest) { 81 | Request request = requestFactory.createRequest(payloadGenerator.getRequestBody(), 82 | payloadGenerator.getRequestParameters(), payloadGenerator.getRequestHeaders()); 83 | 84 | if (log.isTraceEnabled()) { 85 | log.trace("{} request to: {} with parameters: {}, headers: {}, and body: {}", request.getMethod(), 86 | request.getUrl(), request.getParameters(), request.getHeaders(), request.getBody()); 87 | } 88 | 89 | Response response = executor.execute(request); 90 | 91 | if (log.isTraceEnabled()) { 92 | log.trace("Response: {}, Request: {}", response, request); 93 | } 94 | 95 | makeAnotherRequest = payloadGenerator.update(request, response); 96 | 97 | for (String responseRecord : responseHandler.handle(response, ctx)) { 98 | SourceRecord sourceRecord = new SourceRecord(sourcePartition, payloadGenerator.getOffsets(), 99 | topicSelector.getTopic(responseRecord), Schema.STRING_SCHEMA, responseRecord); 100 | for (Map.Entry> header : response.getHeaders().entrySet()) { 101 | sourceRecord.headers().add(header.getKey(), header.getValue(), SchemaBuilder.array(Schema.STRING_SCHEMA).build()); 102 | } 103 | records.add(sourceRecord); 104 | } 105 | } 106 | } catch (Exception e) { 107 | log.error("HTTP call execution failed " + e.getMessage(), e); 108 | increaseCounter(ERROR_METRIC, ctx); 109 | } finally { 110 | lastPollTime = currentTimeMillis(); 111 | } 112 | 113 | return records; 114 | } 115 | 116 | @Override 117 | public synchronized void stop() { 118 | log.debug("Stopping source task"); 119 | } 120 | 121 | @Override 122 | public String version() { 123 | return VersionUtil.getVersion(); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/XPathResponseValueProvider.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.http.Request; 5 | import com.tm.kafka.connect.rest.http.Response; 6 | import org.apache.kafka.common.Configurable; 7 | import org.apache.kafka.common.config.ConfigException; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.w3c.dom.DOMException; 11 | import org.w3c.dom.Document; 12 | import org.w3c.dom.Node; 13 | import org.w3c.dom.NodeList; 14 | import org.xml.sax.InputSource; 15 | import org.xml.sax.SAXException; 16 | 17 | import javax.xml.parsers.DocumentBuilder; 18 | import javax.xml.parsers.DocumentBuilderFactory; 19 | import javax.xml.parsers.ParserConfigurationException; 20 | import javax.xml.xpath.*; 21 | import java.io.IOException; 22 | import java.io.StringReader; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | 27 | /** 28 | * Lookup values used to populate dynamic payloads. 29 | * These values will be substituted into the payload template. 30 | * 31 | * This implementation uses XPath to extract values from an XML HTTP response, 32 | * and if not found looks them up in the System properties and then in environment variables. 33 | */ 34 | // TODO - This class doesn't deal properly with XML namespaces 35 | // We should probably have a mapping in the config from namespaces to tag prefixes. 36 | public class XPathResponseValueProvider extends EnvironmentValueProvider implements Configurable { 37 | 38 | private static Logger log = LoggerFactory.getLogger(XPathResponseValueProvider.class); 39 | 40 | public static final String MULTI_VALUE_SEPARATOR = ","; 41 | 42 | private static final XPathFactory X_PATH_FACTORY = XPathFactory.newInstance(); 43 | 44 | private DocumentBuilder docBuilder; 45 | 46 | private Map expressions; 47 | 48 | 49 | /** 50 | * Configure this instance after creation. 51 | * 52 | * @param props The configuration properties 53 | */ 54 | @Override 55 | public void configure(Map props) { 56 | final XPathResponseValueProviderConfig config = new XPathResponseValueProviderConfig(props); 57 | try { 58 | docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); 59 | } catch (ParserConfigurationException ex) { 60 | throw new ConfigException("Failed to create XML Document Builder due to ConfigurationException: " 61 | + ex.getMessage()); 62 | } 63 | setExpressions(config.getResponseVariableXPaths()); 64 | } 65 | 66 | /** 67 | * Extract values from the response using the XPaths 68 | * 69 | * @param request The last request made. 70 | * @param response The last response received. 71 | */ 72 | @Override 73 | protected void extractValues(Request request, Response response) { 74 | String resp = response.getPayload(); 75 | try { 76 | Document respDom = docBuilder.parse(new InputSource(new StringReader(resp))); 77 | expressions.forEach((key, expr) -> parameterMap.put(key, extractValue(key, respDom, expr))); 78 | } catch (SAXException | IOException ex) { 79 | log.error("The XML could not be parsed: " + resp, ex); 80 | } 81 | } 82 | 83 | /** 84 | * Set the XPaths to be used for value extraction. 85 | * 86 | * @param xPaths A map of key names to XPath expressions 87 | */ 88 | protected void setExpressions(Map xPaths) { 89 | expressions = new HashMap<>(xPaths.size()); 90 | parameterMap = new HashMap<>(expressions.size()); 91 | xPaths.forEach(this::addXPath); 92 | } 93 | 94 | /** 95 | * Extract the value for a given key. 96 | * Where the XPath yeilds more than one result a comma seperated list will be returned. 97 | * 98 | * @param key The name of the key 99 | * @param respDom The response to extract a value from 100 | * @param expression The compiled XPath used to find the value 101 | * @return Return the value, or null if it wasn't found 102 | */ 103 | private String extractValue(String key, Document respDom, XPathExpression expression) { 104 | StringBuilder values = new StringBuilder(); 105 | try { 106 | NodeList nodes = (NodeList) expression.evaluate(respDom, XPathConstants.NODESET); 107 | 108 | for(int i = 0; i < nodes.getLength(); i++) { 109 | if(values.length() > 0) { 110 | values.append(MULTI_VALUE_SEPARATOR); 111 | } 112 | Node node = nodes.item(i); 113 | values.append(node.getTextContent()); 114 | } 115 | 116 | String value = (values.length() != 0) ? values.toString() : null; 117 | log.info("Variable {} was assigned the value {}", key, value); 118 | return value; 119 | } catch (XPathExpressionException ex) { 120 | log.error("The XPath expression '" + expression.toString() + "' could not be evaluated against: " + respDom, ex); 121 | return null; 122 | } catch (DOMException ex) { 123 | log.error("The result(s) were too big when XPath expression '" + expression.toString() 124 | + "' was evaluated against: " + respDom, ex); 125 | return null; 126 | } 127 | } 128 | 129 | private void addXPath(String key, String xPath) { 130 | try { 131 | expressions.put(key, X_PATH_FACTORY.newXPath().compile(xPath)); 132 | } catch (XPathExpressionException ex) { 133 | log.error("The XPath expression '" + xPath + "' could not be compiled", ex); 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | set MAVEN_CMD_LINE_ARGS=%* 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | 121 | set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" 122 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 123 | 124 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% 125 | if ERRORLEVEL 1 goto error 126 | goto end 127 | 128 | :error 129 | set ERROR_CODE=1 130 | 131 | :end 132 | @endlocal & set ERROR_CODE=%ERROR_CODE% 133 | 134 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 135 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 136 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 137 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 138 | :skipRcPost 139 | 140 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 141 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 142 | 143 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 144 | 145 | exit /B %ERROR_CODE% -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # Attempt to set APP_HOME 46 | # Resolve links: $0 may be a link 47 | PRG="$0" 48 | # Need this for relative symlinks. 49 | while [ -h "$PRG" ] ; do 50 | ls=`ls -ld "$PRG"` 51 | link=`expr "$ls" : '.*-> \(.*\)$'` 52 | if expr "$link" : '/.*' > /dev/null; then 53 | PRG="$link" 54 | else 55 | PRG=`dirname "$PRG"`"/$link" 56 | fi 57 | done 58 | SAVED="`pwd`" 59 | cd "`dirname \"$PRG\"`/" >/dev/null 60 | APP_HOME="`pwd -P`" 61 | cd "$SAVED" >/dev/null 62 | 63 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 64 | 65 | # Determine the Java command to use to start the JVM. 66 | if [ -n "$JAVA_HOME" ] ; then 67 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 68 | # IBM's JDK on AIX uses strange locations for the executables 69 | JAVACMD="$JAVA_HOME/jre/sh/java" 70 | else 71 | JAVACMD="$JAVA_HOME/bin/java" 72 | fi 73 | if [ ! -x "$JAVACMD" ] ; then 74 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 75 | 76 | Please set the JAVA_HOME variable in your environment to match the 77 | location of your Java installation." 78 | fi 79 | else 80 | JAVACMD="java" 81 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 82 | 83 | Please set the JAVA_HOME variable in your environment to match the 84 | location of your Java installation." 85 | fi 86 | 87 | # Increase the maximum file descriptors if we can. 88 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 89 | MAX_FD_LIMIT=`ulimit -H -n` 90 | if [ $? -eq 0 ] ; then 91 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 92 | MAX_FD="$MAX_FD_LIMIT" 93 | fi 94 | ulimit -n $MAX_FD 95 | if [ $? -ne 0 ] ; then 96 | warn "Could not set maximum file descriptor limit: $MAX_FD" 97 | fi 98 | else 99 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 100 | fi 101 | fi 102 | 103 | # For Darwin, add options to specify how the application appears in the dock 104 | if $darwin; then 105 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 106 | fi 107 | 108 | # For Cygwin, switch paths to Windows format before running java 109 | if $cygwin ; then 110 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 111 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 112 | JAVACMD=`cygpath --unix "$JAVACMD"` 113 | 114 | # We build the pattern for arguments to be converted via cygpath 115 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 116 | SEP="" 117 | for dir in $ROOTDIRSRAW ; do 118 | ROOTDIRS="$ROOTDIRS$SEP$dir" 119 | SEP="|" 120 | done 121 | OURCYGPATTERN="(^($ROOTDIRS))" 122 | # Add a user-defined pattern to the cygpath arguments 123 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 124 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 125 | fi 126 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 127 | i=0 128 | for arg in "$@" ; do 129 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 130 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 131 | 132 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 133 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 134 | else 135 | eval `echo args$i`="\"$arg\"" 136 | fi 137 | i=$((i+1)) 138 | done 139 | case $i in 140 | (0) set -- ;; 141 | (1) set -- "$args0" ;; 142 | (2) set -- "$args0" "$args1" ;; 143 | (3) set -- "$args0" "$args1" "$args2" ;; 144 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 145 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 146 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 147 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 148 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 149 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 150 | esac 151 | fi 152 | 153 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 154 | function splitJvmOpts() { 155 | JVM_OPTS=("$@") 156 | } 157 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 158 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 159 | 160 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 161 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/ConstantPayloadGeneratorConfig.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload; 2 | 3 | 4 | import org.apache.kafka.common.config.AbstractConfig; 5 | import org.apache.kafka.common.config.ConfigDef; 6 | import org.apache.kafka.common.config.ConfigDef.Importance; 7 | import org.apache.kafka.common.config.ConfigDef.Type; 8 | 9 | import java.util.Collections; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.stream.Collectors; 14 | 15 | 16 | public class ConstantPayloadGeneratorConfig extends AbstractConfig { 17 | 18 | public static final String REQUEST_BODY_CONFIG = "rest.source.body"; 19 | private static final String REQUEST_BODY_DOC = "The HTTP request body that will be sent with each REST request. " + 20 | "This parameter is not appliccable to GET requests."; 21 | private static final String REQUEST_BODY_DISPLAY = "HTTP request body for REST source connector."; 22 | private static final String REQUEST_BODY_DEFAULT = ""; 23 | 24 | public static final String REQUEST_PARAMETER_NAMES_CONFIG = "rest.source.param.names"; 25 | private static final String REQUEST_PARAMETER_NAMES_DOC = "The HTTP request parameter names that will be sent with each " + 26 | "REST request. The parameter values should each be defined by a rest.source.param..value entry."; 27 | private static final String REQUEST_PARAMETER_NAMES_DISPLAY = "HTTP request parameter names for REST source connector."; 28 | private static final List REQUEST_PARAMETER_NAMES_DEFAULT = Collections.EMPTY_LIST; 29 | 30 | public static final String REQUEST_PARAMETER_VALUE_CONFIG = "rest.source.param.%s.value"; 31 | private static final String REQUEST_PARAMETER_VALUE_DOC = "Value for %s parameter which will be passed with each " + 32 | "REST request. This value will be URLEncoded before transmission."; 33 | private static final String REQUEST_PARAMETER_VALUE_DISPLAY = "Value for %s parameter for REST source connector."; 34 | private static final Object REQUEST_PARAMETER_VALUE_DEFAULT = ConfigDef.NO_DEFAULT_VALUE; 35 | 36 | public static final String REQUEST_HEADERS_CONFIG = "rest.source.headers"; 37 | private static final String REQUEST_HEADERS_DISPLAY = "The HTTP request headers that will be sent with each REST " + 38 | "request. The headers should be of the form 'key:value'."; 39 | private static final String REQUEST_HEADERS_DOC = "HTTP request headers for REST source connector."; 40 | private static final List REQUEST_HEADERS_DEFAULT = Collections.EMPTY_LIST; 41 | 42 | private final Map requestParameters; 43 | private final Map requestHeaders; 44 | 45 | 46 | protected ConstantPayloadGeneratorConfig(ConfigDef config, Map unparsedConfig) { 47 | super(config, unparsedConfig); 48 | 49 | List paramNames = getRequestParameterNames(); 50 | requestParameters = new HashMap<>(paramNames.size()); 51 | paramNames.forEach(key -> requestParameters.put(key, getString(String.format(REQUEST_PARAMETER_VALUE_CONFIG, key)))); 52 | 53 | requestHeaders = getList(REQUEST_HEADERS_CONFIG).stream() 54 | .map(a -> a.split(":", 2)) 55 | .collect(Collectors.toMap(a -> a[0], a -> a[1])); 56 | } 57 | 58 | public ConstantPayloadGeneratorConfig(Map unparsedConfig) { 59 | this(conf(unparsedConfig), unparsedConfig); 60 | } 61 | 62 | 63 | 64 | public static ConfigDef conf(Map unparsedConfig) { 65 | String group = "REST_HTTP"; 66 | int orderInGroup = 0; 67 | ConfigDef config = new ConfigDef() 68 | .define(REQUEST_BODY_CONFIG, 69 | Type.STRING, 70 | REQUEST_BODY_DEFAULT, 71 | Importance.LOW, 72 | REQUEST_BODY_DOC, 73 | group, 74 | ++orderInGroup, 75 | ConfigDef.Width.LONG, 76 | REQUEST_BODY_DISPLAY) 77 | 78 | .define(REQUEST_PARAMETER_NAMES_CONFIG, 79 | Type.LIST, 80 | REQUEST_PARAMETER_NAMES_DEFAULT, 81 | Importance.HIGH, 82 | REQUEST_PARAMETER_NAMES_DOC, 83 | group, 84 | ++orderInGroup, 85 | ConfigDef.Width.SHORT, 86 | REQUEST_PARAMETER_NAMES_DISPLAY) 87 | 88 | .define(REQUEST_HEADERS_CONFIG, 89 | Type.LIST, 90 | REQUEST_HEADERS_DEFAULT, 91 | Importance.HIGH, 92 | REQUEST_HEADERS_DOC, 93 | group, 94 | ++orderInGroup, 95 | ConfigDef.Width.SHORT, 96 | REQUEST_HEADERS_DISPLAY) 97 | ; 98 | 99 | // This is a bit hacky and there may be a better way of doing it, but I don't know it. 100 | // We need to create config items dynamically, based on the parameter names, 101 | // so we need a 2 pass parse of the config. 102 | List paramNames = (List) config.parse(unparsedConfig).get(REQUEST_PARAMETER_NAMES_CONFIG); 103 | 104 | for(String paramName : paramNames) { 105 | config.define(String.format(REQUEST_PARAMETER_VALUE_CONFIG, paramName), 106 | Type.STRING, 107 | REQUEST_PARAMETER_VALUE_DEFAULT, 108 | Importance.HIGH, 109 | String.format(REQUEST_PARAMETER_VALUE_DOC, paramName), 110 | group, 111 | ++orderInGroup, 112 | ConfigDef.Width.SHORT, 113 | String.format(REQUEST_PARAMETER_VALUE_DISPLAY, paramName)); 114 | } 115 | 116 | return(config); 117 | } 118 | 119 | public String getRequestBody() { 120 | return this.getString(REQUEST_BODY_CONFIG); 121 | } 122 | 123 | public List getRequestParameterNames() { 124 | return this.getList(REQUEST_PARAMETER_NAMES_CONFIG); 125 | } 126 | 127 | public Map getRequestParameters() { 128 | return requestParameters; 129 | } 130 | 131 | public Map getRequestHeaders() { 132 | return requestHeaders; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/test/java/com/tm/kafka/connect/rest/http/payload/templated/TemplatedPayloadGeneratorTest.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.RestSinkConnectorConfig; 5 | import com.tm.kafka.connect.rest.RestSourceConnectorConfig; 6 | import com.tm.kafka.connect.rest.http.Request; 7 | import com.tm.kafka.connect.rest.http.Response; 8 | import com.tm.kafka.connect.rest.http.payload.ConstantPayloadGenerator; 9 | import com.tm.kafka.connect.rest.http.payload.ConstantPayloadGeneratorConfig; 10 | import org.hamcrest.Matchers; 11 | import org.junit.Before; 12 | import org.junit.Test; 13 | 14 | import java.util.Collections; 15 | import java.util.Map; 16 | import java.util.stream.Collectors; 17 | import java.util.stream.Stream; 18 | 19 | import static java.lang.System.currentTimeMillis; 20 | import static org.hamcrest.CoreMatchers.equalTo; 21 | import static org.hamcrest.Matchers.*; 22 | import static org.hamcrest.Matchers.notNullValue; 23 | import static org.junit.Assert.assertThat; 24 | 25 | 26 | public class TemplatedPayloadGeneratorTest { 27 | 28 | private static String REQUEST_BODY_VAL = "{\"query\": \"select * from known_stars where name='${STAR_NAME}'\""; 29 | 30 | private static final Map REQUEST_PARAM_VAL = Stream.of(new String[][] { 31 | { "priority", "MAXIMUM" }, 32 | { "paged", "FALSE" }, 33 | }).collect(Collectors.toMap(d -> d[0], d -> d[1])); 34 | 35 | private static final Map REQUEST_HEADER_VAL = Stream.of(new String[][] { 36 | { "Content-Type", "application/json" }, 37 | { "Accept", "application/json" }, 38 | }).collect(Collectors.toMap(d -> d[0], d -> d[1])); 39 | 40 | private static final Map CONFIG_PROPS = Stream.of(new String[][] { 41 | { TemplatedPayloadGeneratorConfig.REQUEST_BODY_TEMPLATE_CONFIG, REQUEST_BODY_VAL }, 42 | { TemplatedPayloadGeneratorConfig.REQUEST_HEADERS_TEMPLATE_CONFIG, "Content-Type:${CONTENT_TYPE}, Accept:${ACCEPT_TYPE}" }, 43 | { TemplatedPayloadGeneratorConfig.REQUEST_PARAMETER_NAMES_CONFIG, "priority, paged" }, 44 | { String.format(TemplatedPayloadGeneratorConfig.REQUEST_PARAMETER_TEMPLATE_CONFIG, "priority"), "${PRIORITY}" }, 45 | { String.format(TemplatedPayloadGeneratorConfig.REQUEST_PARAMETER_TEMPLATE_CONFIG, "paged"), "${PAGED}" }, 46 | }).collect(Collectors.toMap(d -> d[0], d -> d[1])); 47 | 48 | private static final Request REQUEST = new Request("http://data.stars.org/query", "POST", 49 | REQUEST_BODY_VAL, REQUEST_PARAM_VAL, REQUEST_HEADER_VAL); 50 | 51 | private static final Response RESPONSE = new Response(200, Collections.emptyMap(), "LOTS of data"); 52 | 53 | private TemplatedPayloadGenerator generator; 54 | 55 | @Before 56 | public void before() { 57 | generator = new TemplatedPayloadGenerator(); 58 | } 59 | 60 | 61 | @Test 62 | public void testUpdate() { 63 | generator.configure(CONFIG_PROPS); 64 | assertThat(generator.update(REQUEST, RESPONSE), Matchers.equalTo(false)); 65 | } 66 | 67 | @Test 68 | public void testGetRequestBody() { 69 | System.setProperty("STAR_NAME", "Sol"); 70 | generator.configure(CONFIG_PROPS); 71 | assertThat(generator.getRequestBody(), Matchers.equalTo("{\"query\": \"select * from known_stars where name='Sol'\"")); 72 | } 73 | 74 | @Test 75 | public void testGetRequestBody_configUndefined() { 76 | generator.configure(Collections.emptyMap()); 77 | assertThat(generator.getRequestBody(), Matchers.equalTo("")); 78 | } 79 | 80 | @Test 81 | public void testGetRequestParameters() { 82 | System.setProperty("PRIORITY", "MAXIMUM"); 83 | System.setProperty("PAGED", "FALSE"); 84 | generator.configure(CONFIG_PROPS); 85 | assertThat(generator.getRequestParameters(), allOf( 86 | hasEntry("priority", "MAXIMUM"), 87 | hasEntry("paged", "FALSE"))); 88 | } 89 | 90 | @Test 91 | public void testGetRequestParameters_configUndefined() { 92 | generator.configure(Collections.emptyMap()); 93 | assertThat(generator.getRequestParameters(), not(hasKey(anything()))); 94 | } 95 | 96 | @Test 97 | public void testGetRequestHeaders() { 98 | System.setProperty("CONTENT_TYPE", "application/json"); 99 | System.setProperty("ACCEPT_TYPE", "application/json"); 100 | generator.configure(CONFIG_PROPS); 101 | assertThat(generator.getRequestHeaders(), allOf( 102 | hasEntry("Content-Type", "application/json"), 103 | hasEntry("Accept", "application/json"))); 104 | } 105 | 106 | @Test 107 | public void testGetRequestHeaders_configUndefined() { 108 | generator.configure(Collections.emptyMap()); 109 | assertThat(generator.getRequestHeaders(), not(hasKey(anything()))); 110 | } 111 | 112 | @Test 113 | public void testGetOffsets() { 114 | System.setProperty("STAR_NAME", "Vega"); 115 | System.setProperty("PRIORITY", "MINIMUM"); 116 | System.setProperty("PAGED", "TRUE"); 117 | System.setProperty("CONTENT_TYPE", "text/plain"); 118 | System.setProperty("ACCEPT_TYPE", "text/plain"); 119 | generator.configure(CONFIG_PROPS); 120 | assertThat(generator.getOffsets(), allOf( 121 | hasEntry("STAR_NAME", "Vega"), 122 | hasEntry("PRIORITY", "MINIMUM"), 123 | hasEntry("PAGED", "TRUE"), 124 | hasEntry("CONTENT_TYPE", "text/plain"), 125 | hasEntry("ACCEPT_TYPE", "text/plain"))); 126 | } 127 | 128 | @Test 129 | public void testGetOffsets_configUndefined() { 130 | generator.configure(Collections.emptyMap()); 131 | assertThat(generator.getOffsets(), not(hasKey(anything()))); 132 | } 133 | 134 | @Test 135 | public void testSetOffsets() { 136 | generator.configure(CONFIG_PROPS); 137 | generator.setOffsets(Collections.singletonMap("STAR_NAME", "Pollux")); 138 | assertThat(generator.getOffsets(), hasEntry("STAR_NAME", "Pollux")); 139 | assertThat(generator.getRequestBody(), Matchers.equalTo("{\"query\": \"select * from known_stars where name='Pollux'\"")); 140 | } 141 | 142 | @Test 143 | public void testSetOffsets_configUndefined() { 144 | generator.configure(Collections.emptyMap()); 145 | generator.setOffsets(Collections.singletonMap("STAR_NAME", "Pollux")); 146 | assertThat(generator.getOffsets(), hasEntry("STAR_NAME", "Pollux")); 147 | assertThat(generator.getRequestBody(), Matchers.equalTo("")); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/RestSourceConnectorConfig.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest; 2 | 3 | 4 | import com.tm.kafka.connect.rest.config.*; 5 | import com.tm.kafka.connect.rest.http.executor.RequestExecutor; 6 | import com.tm.kafka.connect.rest.http.handler.DefaultResponseHandler; 7 | import com.tm.kafka.connect.rest.http.handler.ResponseHandler; 8 | import com.tm.kafka.connect.rest.http.payload.ConstantPayloadGenerator; 9 | import com.tm.kafka.connect.rest.http.payload.PayloadGenerator; 10 | import com.tm.kafka.connect.rest.selector.SimpleTopicSelector; 11 | import com.tm.kafka.connect.rest.selector.TopicSelector; 12 | import org.apache.kafka.common.config.AbstractConfig; 13 | import org.apache.kafka.common.config.ConfigDef; 14 | import org.apache.kafka.common.config.ConfigDef.Importance; 15 | import org.apache.kafka.common.config.ConfigDef.Type; 16 | 17 | import java.util.HashMap; 18 | import java.util.Map; 19 | 20 | import static org.apache.kafka.common.config.ConfigDef.NO_DEFAULT_VALUE; 21 | 22 | 23 | public class RestSourceConnectorConfig extends AbstractConfig { 24 | 25 | public static final String SOURCE_POLL_INTERVAL_CONFIG = "rest.source.poll.interval.ms"; 26 | private static final String SOURCE_POLL_INTERVAL_DOC = "How often to poll the source URL."; 27 | private static final String SOURCE_POLL_INTERVAL_DISPLAY = "Polling interval"; 28 | private static final Long SOURCE_POLL_INTERVAL_DEFAULT = 60000L; 29 | 30 | public static final String SOURCE_METHOD_CONFIG = "rest.source.method"; 31 | private static final String SOURCE_METHOD_DOC = "The HTTP method for REST source connector."; 32 | private static final String SOURCE_METHOD_DISPLAY = "Source method"; 33 | private static final String SOURCE_METHOD_DEFAULT = "POST"; 34 | 35 | public static final String SOURCE_URL_CONFIG = "rest.source.url"; 36 | private static final String SOURCE_URL_DOC = "The URL for REST source connector."; 37 | private static final String SOURCE_URL_DISPLAY = "URL for REST source connector."; 38 | 39 | public static final String SOURCE_PAYLOAD_GENERATOR_CONFIG = "rest.source.data.generator"; 40 | private static final String SOURCE_PAYLOAD_GENERATOR_DOC = "The payload generator class which will produce the HTTP " + 41 | "request payload to be sent to the REST endpoint. The payload may be sent as request parameters in the case of a " + 42 | "GET request, or as the request body in the case of POST"; 43 | private static final String SOURCE_PAYLOAD_GENERATOR_DISPLAY = "Payload Generator class for REST source connector."; 44 | private static final Class SOURCE_PAYLOAD_GENERATOR_DEFAULT = ConstantPayloadGenerator.class; 45 | 46 | public static final String SOURCE_TOPIC_SELECTOR_CONFIG = "rest.source.topic.selector"; 47 | private static final String SOURCE_TOPIC_SELECTOR_DOC = "The topic selector class for REST source connector."; 48 | private static final String SOURCE_TOPIC_SELECTOR_DISPLAY = "Topic selector class for REST source connector."; 49 | private static final Class SOURCE_TOPIC_SELECTOR_DEFAULT = SimpleTopicSelector.class; 50 | 51 | public static final String SOURCE_REQUEST_EXECUTOR_CONFIG = "rest.http.executor.class"; 52 | private static final String SOURCE_REQUEST_EXECUTOR_DISPLAY = "HTTP request executor"; 53 | private static final String SOURCE_REQUEST_EXECUTOR_DOC = "HTTP request executor. Default is OkHttpRequestExecutor"; 54 | private static final String SOURCE_REQUEST_EXECUTOR_DEFAULT = "com.tm.kafka.connect.rest.http.executor.OkHttpRequestExecutor"; 55 | 56 | 57 | private final TopicSelector topicSelector; 58 | private final PayloadGenerator payloadGenerator; 59 | private RequestExecutor requestExecutor; 60 | 61 | 62 | protected RestSourceConnectorConfig(ConfigDef config, Map parsedConfig) { 63 | super(config, parsedConfig); 64 | topicSelector = this.getConfiguredInstance(SOURCE_TOPIC_SELECTOR_CONFIG, TopicSelector.class); 65 | requestExecutor = this.getConfiguredInstance(SOURCE_REQUEST_EXECUTOR_CONFIG, RequestExecutor.class); 66 | payloadGenerator = this.getConfiguredInstance(SOURCE_PAYLOAD_GENERATOR_CONFIG, PayloadGenerator.class); 67 | } 68 | 69 | public RestSourceConnectorConfig(Map parsedConfig) { 70 | this(conf(), parsedConfig); 71 | } 72 | 73 | public static ConfigDef conf() { 74 | String group = "REST"; 75 | int orderInGroup = 0; 76 | return new ConfigDef() 77 | .define(SOURCE_POLL_INTERVAL_CONFIG, 78 | Type.LONG, 79 | SOURCE_POLL_INTERVAL_DEFAULT, 80 | Importance.LOW, 81 | SOURCE_POLL_INTERVAL_DOC, 82 | group, 83 | ++orderInGroup, 84 | ConfigDef.Width.SHORT, 85 | SOURCE_POLL_INTERVAL_DISPLAY) 86 | 87 | .define(SOURCE_METHOD_CONFIG, 88 | Type.STRING, 89 | SOURCE_METHOD_DEFAULT, 90 | new MethodValidator(), 91 | Importance.HIGH, 92 | SOURCE_METHOD_DOC, 93 | group, 94 | ++orderInGroup, 95 | ConfigDef.Width.SHORT, 96 | SOURCE_METHOD_DISPLAY, 97 | new MethodRecommender()) 98 | 99 | .define(SOURCE_URL_CONFIG, 100 | Type.STRING, 101 | NO_DEFAULT_VALUE, 102 | Importance.HIGH, 103 | SOURCE_URL_DOC, 104 | group, 105 | ++orderInGroup, 106 | ConfigDef.Width.SHORT, 107 | SOURCE_URL_DISPLAY) 108 | 109 | .define(SOURCE_PAYLOAD_GENERATOR_CONFIG, 110 | Type.CLASS, 111 | SOURCE_PAYLOAD_GENERATOR_DEFAULT, 112 | new InstanceOfValidator(PayloadGenerator.class), 113 | Importance.HIGH, 114 | SOURCE_PAYLOAD_GENERATOR_DOC, 115 | group, 116 | ++orderInGroup, 117 | ConfigDef.Width.SHORT, 118 | SOURCE_PAYLOAD_GENERATOR_DISPLAY, 119 | new ServiceProviderInterfaceRecommender<>(PayloadGenerator.class)) 120 | 121 | .define(SOURCE_TOPIC_SELECTOR_CONFIG, 122 | Type.CLASS, 123 | SOURCE_TOPIC_SELECTOR_DEFAULT, 124 | new InstanceOfValidator(TopicSelector.class), 125 | Importance.HIGH, 126 | SOURCE_TOPIC_SELECTOR_DOC, 127 | group, 128 | ++orderInGroup, 129 | ConfigDef.Width.SHORT, 130 | SOURCE_TOPIC_SELECTOR_DISPLAY, 131 | new ServiceProviderInterfaceRecommender<>(TopicSelector.class)) 132 | 133 | .define(SOURCE_REQUEST_EXECUTOR_CONFIG, 134 | Type.CLASS, 135 | SOURCE_REQUEST_EXECUTOR_DEFAULT, 136 | new InstanceOfValidator(RequestExecutor.class), 137 | Importance.LOW, 138 | SOURCE_REQUEST_EXECUTOR_DOC, 139 | group, 140 | ++orderInGroup, 141 | ConfigDef.Width.NONE, 142 | SOURCE_REQUEST_EXECUTOR_DISPLAY, 143 | new ServiceProviderInterfaceRecommender<>(RequestExecutor.class)) 144 | ; 145 | } 146 | 147 | public ResponseHandler getResponseHandler() { 148 | return new DefaultResponseHandler(); 149 | } 150 | 151 | public RequestExecutor getRequestExecutor() { 152 | return requestExecutor; 153 | } 154 | 155 | public long getPollInterval() { 156 | return this.getLong(SOURCE_POLL_INTERVAL_CONFIG); 157 | } 158 | 159 | public String getMethod() { 160 | return this.getString(SOURCE_METHOD_CONFIG); 161 | } 162 | 163 | public String getUrl() { 164 | return this.getString(SOURCE_URL_CONFIG); 165 | } 166 | 167 | public TopicSelector getTopicSelector() { 168 | return topicSelector; 169 | } 170 | 171 | public PayloadGenerator getPayloadGenerator() { 172 | return payloadGenerator; 173 | } 174 | 175 | private static ConfigDef getConfig() { 176 | Map everything = new HashMap<>(conf().configKeys()); 177 | ConfigDef visible = new ConfigDef(); 178 | for (ConfigDef.ConfigKey key : everything.values()) { 179 | visible.define(key); 180 | } 181 | return visible; 182 | } 183 | 184 | public static void main(String[] args) { 185 | System.out.println(VersionUtil.getVersion()); 186 | System.out.println(getConfig().toEnrichedRst()); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /examples/spring/gs-rest-service/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # 58 | # Look for the Apple JDKs first to preserve the existing behaviour, and then look 59 | # for the new JDKs provided by Oracle. 60 | # 61 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then 62 | # 63 | # Apple JDKs 64 | # 65 | export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home 66 | fi 67 | 68 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then 69 | # 70 | # Apple JDKs 71 | # 72 | export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 73 | fi 74 | 75 | if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then 76 | # 77 | # Oracle JDKs 78 | # 79 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 80 | fi 81 | 82 | if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then 83 | # 84 | # Apple JDKs 85 | # 86 | export JAVA_HOME=`/usr/libexec/java_home` 87 | fi 88 | ;; 89 | esac 90 | 91 | if [ -z "$JAVA_HOME" ] ; then 92 | if [ -r /etc/gentoo-release ] ; then 93 | JAVA_HOME=`java-config --jre-home` 94 | fi 95 | fi 96 | 97 | if [ -z "$M2_HOME" ] ; then 98 | ## resolve links - $0 may be a link to maven's home 99 | PRG="$0" 100 | 101 | # need this for relative symlinks 102 | while [ -h "$PRG" ] ; do 103 | ls=`ls -ld "$PRG"` 104 | link=`expr "$ls" : '.*-> \(.*\)$'` 105 | if expr "$link" : '/.*' > /dev/null; then 106 | PRG="$link" 107 | else 108 | PRG="`dirname "$PRG"`/$link" 109 | fi 110 | done 111 | 112 | saveddir=`pwd` 113 | 114 | M2_HOME=`dirname "$PRG"`/.. 115 | 116 | # make it fully qualified 117 | M2_HOME=`cd "$M2_HOME" && pwd` 118 | 119 | cd "$saveddir" 120 | # echo Using m2 at $M2_HOME 121 | fi 122 | 123 | # For Cygwin, ensure paths are in UNIX format before anything is touched 124 | if $cygwin ; then 125 | [ -n "$M2_HOME" ] && 126 | M2_HOME=`cygpath --unix "$M2_HOME"` 127 | [ -n "$JAVA_HOME" ] && 128 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 129 | [ -n "$CLASSPATH" ] && 130 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 131 | fi 132 | 133 | # For Migwn, ensure paths are in UNIX format before anything is touched 134 | if $mingw ; then 135 | [ -n "$M2_HOME" ] && 136 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 137 | [ -n "$JAVA_HOME" ] && 138 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 139 | # TODO classpath? 140 | fi 141 | 142 | if [ -z "$JAVA_HOME" ]; then 143 | javaExecutable="`which javac`" 144 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 145 | # readlink(1) is not available as standard on Solaris 10. 146 | readLink=`which readlink` 147 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 148 | if $darwin ; then 149 | javaHome="`dirname \"$javaExecutable\"`" 150 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 151 | else 152 | javaExecutable="`readlink -f \"$javaExecutable\"`" 153 | fi 154 | javaHome="`dirname \"$javaExecutable\"`" 155 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 156 | JAVA_HOME="$javaHome" 157 | export JAVA_HOME 158 | fi 159 | fi 160 | fi 161 | 162 | if [ -z "$JAVACMD" ] ; then 163 | if [ -n "$JAVA_HOME" ] ; then 164 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 165 | # IBM's JDK on AIX uses strange locations for the executables 166 | JAVACMD="$JAVA_HOME/jre/sh/java" 167 | else 168 | JAVACMD="$JAVA_HOME/bin/java" 169 | fi 170 | else 171 | JAVACMD="`which java`" 172 | fi 173 | fi 174 | 175 | if [ ! -x "$JAVACMD" ] ; then 176 | echo "Error: JAVA_HOME is not defined correctly." >&2 177 | echo " We cannot execute $JAVACMD" >&2 178 | exit 1 179 | fi 180 | 181 | if [ -z "$JAVA_HOME" ] ; then 182 | echo "Warning: JAVA_HOME environment variable is not set." 183 | fi 184 | 185 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 186 | 187 | # For Cygwin, switch paths to Windows format before running java 188 | if $cygwin; then 189 | [ -n "$M2_HOME" ] && 190 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 191 | [ -n "$JAVA_HOME" ] && 192 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 193 | [ -n "$CLASSPATH" ] && 194 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 195 | fi 196 | 197 | # traverses directory structure from process work directory to filesystem root 198 | # first directory with .mvn subdirectory is considered project base directory 199 | find_maven_basedir() { 200 | local basedir=$(pwd) 201 | local wdir=$(pwd) 202 | while [ "$wdir" != '/' ] ; do 203 | if [ -d "$wdir"/.mvn ] ; then 204 | basedir=$wdir 205 | break 206 | fi 207 | wdir=$(cd "$wdir/.."; pwd) 208 | done 209 | echo "${basedir}" 210 | } 211 | 212 | # concatenates all lines of a file 213 | concat_lines() { 214 | if [ -f "$1" ]; then 215 | echo "$(tr -s '\n' ' ' < "$1")" 216 | fi 217 | } 218 | 219 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} 220 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 221 | 222 | # Provide a "standardized" way to retrieve the CLI args that will 223 | # work with both Windows and non-Windows executions. 224 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 225 | export MAVEN_CMD_LINE_ARGS 226 | 227 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 228 | 229 | exec "$JAVACMD" \ 230 | $MAVEN_OPTS \ 231 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 232 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 233 | ${WRAPPER_LAUNCHER} "$@" 234 | -------------------------------------------------------------------------------- /kafka-connect-rest-plugin/src/main/java/com/tm/kafka/connect/rest/http/payload/templated/TemplatedPayloadGeneratorConfig.java: -------------------------------------------------------------------------------- 1 | package com.tm.kafka.connect.rest.http.payload.templated; 2 | 3 | 4 | import com.tm.kafka.connect.rest.config.InstanceOfValidator; 5 | import com.tm.kafka.connect.rest.config.ServiceProviderInterfaceRecommender; 6 | import org.apache.kafka.common.config.AbstractConfig; 7 | import org.apache.kafka.common.config.ConfigDef; 8 | import org.apache.kafka.common.config.ConfigDef.Importance; 9 | import org.apache.kafka.common.config.ConfigDef.Type; 10 | 11 | import java.util.Collections; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.stream.Collectors; 16 | 17 | 18 | public class TemplatedPayloadGeneratorConfig extends AbstractConfig { 19 | 20 | public static final String REQUEST_BODY_TEMPLATE_CONFIG = "rest.source.body.template"; 21 | private static final String REQUEST_BODY_TEMPLATE_DOC = "The template used to generate the HTTP request body that will be " + 22 | "sent with each REST request. This parameter is not appliccable to GET requests."; 23 | private static final String REQUEST_BODY_TEMPLATE_DISPLAY = "Request body template for REST source connector."; 24 | private static final String REQUEST_BODY_TEMPLATE_DEFAULT = ""; 25 | 26 | public static final String REQUEST_PARAMETER_NAMES_CONFIG = "rest.source.param.names"; 27 | private static final String REQUEST_PARAMETER_NAMES_DOC = "The HTTP request parameter names that will be sent with each " + 28 | "REST request. The parameter values should each be defined by a rest.source.param..value entry."; 29 | private static final String REQUEST_PARAMETER_NAMES_DISPLAY = "HTTP request parameter names for REST source connector."; 30 | private static final List REQUEST_PARAMETER_NAMES_DEFAULT = Collections.EMPTY_LIST; 31 | 32 | public static final String REQUEST_PARAMETER_TEMPLATE_CONFIG = "rest.source.param.%s.template"; 33 | private static final String REQUEST_PARAMETER_TEMPLATE_DOC = "Template used to generate the %s parameter which will " + 34 | "be passed with each REST request. This value will be URLEncoded before transmission."; 35 | private static final String REQUEST_PARAMETER_TEMPLATE_DISPLAY = "Template for %s parameter for REST source connector."; 36 | private static final Object REQUEST_PARAMETER_TEMPLATE_DEFAULT = ConfigDef.NO_DEFAULT_VALUE; 37 | 38 | public static final String REQUEST_HEADERS_TEMPLATE_CONFIG = "rest.source.headers.template"; 39 | private static final String REQUEST_HEADERS_TEMPLATE_DISPLAY = "The Templates used to generate HTTP request headers " + 40 | "that will be sent with each REST request. The headers should be of the form 'key:value'."; 41 | private static final String REQUEST_HEADERS_TEMPLATE_DOC = "Request headers template for REST source connector."; 42 | private static final List REQUEST_HEADERS_TEMPLATE_DEFAULT = Collections.EMPTY_LIST; 43 | 44 | public static final String VALUE_PROVIDER_CONFIG = "rest.source.payload.value.provider"; 45 | private static final String VALUE_PROVIDER_DOC = "The class that will provide the values to be substituted into " + 46 | "the payload template by the payload generator."; 47 | private static final String VALUE_PROVIDER_DISPLAY = "Payload Value Provider class for REST source connector."; 48 | private static final Class VALUE_PROVIDER_DEFAULT = 49 | EnvironmentValueProvider.class; 50 | 51 | public static final String TEMPLATE_ENGINE_CONFIG = "rest.source.payload.template.engine"; 52 | private static final String TEMPLATE_ENGINE_DOC = "The template engine that will process the template and " + 53 | "substitute values to generate an actual payload string."; 54 | private static final String TEMPLATE_ENGINE_DISPLAY = "Payload Template Engine class for REST source connector."; 55 | private static final Class TEMPLATE_ENGINE_DEFAULT = 56 | VelocityTemplateEngine.class; 57 | 58 | private final Map requestParameterTemplates; 59 | private final Map requestHeaderTemplates; 60 | private final ValueProvider valueProvider; 61 | private final TemplateEngine templateEngine; 62 | 63 | 64 | protected TemplatedPayloadGeneratorConfig(ConfigDef config, Map unparsedConfig) { 65 | super(config, unparsedConfig); 66 | 67 | valueProvider = this.getConfiguredInstance(VALUE_PROVIDER_CONFIG, ValueProvider.class); 68 | templateEngine = this.getConfiguredInstance(TEMPLATE_ENGINE_CONFIG, TemplateEngine.class); 69 | 70 | List paramNames = getRequestParameterNames(); 71 | requestParameterTemplates = new HashMap<>(paramNames.size()); 72 | paramNames.forEach(key -> requestParameterTemplates.put(key, getString(String.format(REQUEST_PARAMETER_TEMPLATE_CONFIG, key)))); 73 | 74 | requestHeaderTemplates = getList(REQUEST_HEADERS_TEMPLATE_CONFIG).stream() 75 | .map(a -> a.split(":", 2)) 76 | .collect(Collectors.toMap(a -> a[0], a -> a[1])); 77 | } 78 | 79 | public TemplatedPayloadGeneratorConfig(Map unparsedConfig) { 80 | this(conf(unparsedConfig), unparsedConfig); 81 | } 82 | 83 | public static ConfigDef conf(Map unparsedConfig) { 84 | String group = "REST_HTTP"; 85 | int orderInGroup = 0; 86 | ConfigDef config = new ConfigDef() 87 | .define(VALUE_PROVIDER_CONFIG, 88 | Type.CLASS, 89 | VALUE_PROVIDER_DEFAULT, 90 | new InstanceOfValidator(ValueProvider.class), 91 | Importance.HIGH, 92 | VALUE_PROVIDER_DOC, 93 | group, 94 | ++orderInGroup, 95 | ConfigDef.Width.SHORT, 96 | VALUE_PROVIDER_DISPLAY, 97 | new ServiceProviderInterfaceRecommender(ValueProvider.class)) 98 | 99 | .define(TEMPLATE_ENGINE_CONFIG, 100 | Type.CLASS, 101 | TEMPLATE_ENGINE_DEFAULT, 102 | new InstanceOfValidator(TemplateEngine.class), 103 | Importance.HIGH, 104 | TEMPLATE_ENGINE_DOC, 105 | group, 106 | ++orderInGroup, 107 | ConfigDef.Width.SHORT, 108 | TEMPLATE_ENGINE_DISPLAY, 109 | new ServiceProviderInterfaceRecommender(TemplateEngine.class)) 110 | 111 | .define(REQUEST_BODY_TEMPLATE_CONFIG, 112 | Type.STRING, 113 | REQUEST_BODY_TEMPLATE_DEFAULT, 114 | Importance.LOW, 115 | REQUEST_BODY_TEMPLATE_DOC, 116 | group, 117 | ++orderInGroup, 118 | ConfigDef.Width.LONG, 119 | REQUEST_BODY_TEMPLATE_DISPLAY) 120 | 121 | .define(REQUEST_PARAMETER_NAMES_CONFIG, 122 | Type.LIST, 123 | REQUEST_PARAMETER_NAMES_DEFAULT, 124 | Importance.LOW, 125 | REQUEST_PARAMETER_NAMES_DOC, 126 | group, 127 | ++orderInGroup, 128 | ConfigDef.Width.SHORT, 129 | REQUEST_PARAMETER_NAMES_DISPLAY) 130 | 131 | .define(REQUEST_HEADERS_TEMPLATE_CONFIG, 132 | Type.LIST, 133 | REQUEST_HEADERS_TEMPLATE_DEFAULT, 134 | Importance.LOW, 135 | REQUEST_HEADERS_TEMPLATE_DOC, 136 | group, 137 | ++orderInGroup, 138 | ConfigDef.Width.SHORT, 139 | REQUEST_HEADERS_TEMPLATE_DISPLAY) 140 | ; 141 | 142 | // This is a bit hacky and there may be a better way of doing it, but I don't know it. 143 | // We need to create config items dynamically, based on the parameter names, 144 | // so we need a 2 pass parse of the config. 145 | List paramNames = (List) config.parse(unparsedConfig).get(REQUEST_PARAMETER_NAMES_CONFIG); 146 | 147 | for(String paramName : paramNames) { 148 | config.define(String.format(REQUEST_PARAMETER_TEMPLATE_CONFIG, paramName), 149 | Type.STRING, 150 | REQUEST_PARAMETER_TEMPLATE_DEFAULT, 151 | Importance.HIGH, 152 | String.format(REQUEST_PARAMETER_TEMPLATE_DOC, paramName), 153 | group, 154 | ++orderInGroup, 155 | ConfigDef.Width.SHORT, 156 | String.format(REQUEST_PARAMETER_TEMPLATE_DISPLAY, paramName)); 157 | } 158 | 159 | return(config); 160 | } 161 | 162 | public String getRequestBodyTemplate() { 163 | return this.getString(REQUEST_BODY_TEMPLATE_CONFIG); 164 | } 165 | 166 | public List getRequestParameterNames() { 167 | return this.getList(REQUEST_PARAMETER_NAMES_CONFIG); 168 | } 169 | 170 | public Map getRequestParameterTemplates() { 171 | return requestParameterTemplates; 172 | } 173 | 174 | public Map getRequestHeaderTemplates() { 175 | return requestHeaderTemplates; 176 | } 177 | 178 | public ValueProvider getValueProvider() { 179 | return valueProvider; 180 | } 181 | 182 | public TemplateEngine getTemplateEngine() { 183 | return templateEngine; 184 | } 185 | } 186 | --------------------------------------------------------------------------------