taskSettings = new LinkedHashMap<>(this.settings);
78 | taskSettings.put(TwitterSourceConnectorConfig.FILTER_KEYWORDS_CONF, Joiner.on(',').join(keywords));
79 | taskConfigs.add(taskSettings);
80 | }
81 |
82 | return taskConfigs;
83 | }
84 |
85 | @Override
86 | public void stop() {
87 |
88 | }
89 |
90 | @Override
91 | public ConfigDef config() {
92 | return TwitterSourceConnectorConfig.conf();
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/twitter/TwitterSourceConnectorConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.twitter;
17 |
18 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder;
19 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils;
20 | import com.google.common.primitives.Longs;
21 | import org.apache.kafka.common.config.AbstractConfig;
22 | import org.apache.kafka.common.config.ConfigDef;
23 | import org.apache.kafka.common.config.ConfigDef.Importance;
24 | import org.apache.kafka.common.config.ConfigDef.Type;
25 | import org.apache.kafka.common.config.ConfigException;
26 | import twitter4j.conf.Configuration;
27 | import twitter4j.conf.PropertyConfiguration;
28 |
29 | import java.util.Collections;
30 | import java.util.List;
31 | import java.util.Map;
32 | import java.util.Properties;
33 | import java.util.Set;
34 | import java.util.stream.Collectors;
35 |
36 |
37 | public class TwitterSourceConnectorConfig extends AbstractConfig {
38 |
39 | public static final String TWITTER_DEBUG_CONF = "twitter.debug";
40 | public static final String TWITTER_OAUTH_CONSUMER_KEY_CONF = "twitter.oauth.consumerKey";
41 | public static final String TWITTER_OAUTH_SECRET_KEY_CONF = "twitter.oauth.consumerSecret";
42 | public static final String TWITTER_OAUTH_ACCESS_TOKEN_CONF = "twitter.oauth.accessToken";
43 | public static final String TWITTER_OAUTH_ACCESS_TOKEN_SECRET_CONF = "twitter.oauth.accessTokenSecret";
44 | public static final String FILTER_KEYWORDS_CONF = "filter.keywords";
45 | public static final String FILTER_USER_IDS_CONF = "filter.userIds";
46 | public static final String KAFKA_STATUS_TOPIC_CONF = "kafka.status.topic";
47 | public static final String KAFKA_STATUS_TOPIC_DOC = "Kafka topic to write the statuses to.";
48 | public static final String PROCESS_DELETES_CONF = "process.deletes";
49 | public static final String PROCESS_DELETES_DOC = "Should this connector process deletes.";
50 | public static final String QUEUE_EMPTY_MS_CONF = "queue.empty.ms";
51 | public static final String QUEUE_BATCH_SIZE_CONF = "queue.batch.size";
52 | private static final String TWITTER_DEBUG_DOC = "Flag to enable debug logging for the twitter api.";
53 | private static final String TWITTER_OAUTH_CONSUMER_KEY_DOC = "OAuth consumer key";
54 | private static final String TWITTER_OAUTH_SECRET_KEY_DOC = "OAuth consumer secret";
55 | private static final String TWITTER_OAUTH_ACCESS_TOKEN_DOC = "OAuth access token";
56 | private static final String TWITTER_OAUTH_ACCESS_TOKEN_SECRET_DOC = "OAuth access token secret";
57 | private static final String FILTER_KEYWORDS_DOC = "Twitter keywords to filter for.";
58 | private static final String FILTER_USER_IDS_DOC = "Twitter user IDs to follow.";
59 | public static final String QUEUE_EMPTY_MS_DOC = "The amount of time to wait if there are no records in the queue.";
60 | public static final String QUEUE_BATCH_SIZE_DOC = "The number of records to return in a single batch.";
61 |
62 |
63 | public final String topic;
64 | public final boolean twitterDebug;
65 | public final boolean processDeletes;
66 | public final Set filterKeywords;
67 | public final Set filterUserIds;
68 | public final int queueEmptyMs;
69 | public final int queueBatchSize;
70 |
71 |
72 | public TwitterSourceConnectorConfig(Map parsedConfig) {
73 | super(conf(), parsedConfig);
74 | this.topic = this.getString(KAFKA_STATUS_TOPIC_CONF);
75 | this.twitterDebug = this.getBoolean(TWITTER_DEBUG_CONF);
76 | this.processDeletes = this.getBoolean(PROCESS_DELETES_CONF);
77 | this.filterKeywords = ConfigUtils.getSet(this, FILTER_KEYWORDS_CONF);
78 | this.filterUserIds = ConfigUtils.getSet(this, FILTER_USER_IDS_CONF)
79 | .stream()
80 | .map(Long::parseLong)
81 | .collect(Collectors.toSet());
82 | this.queueBatchSize = getInt(QUEUE_BATCH_SIZE_CONF);
83 | this.queueEmptyMs = getInt(QUEUE_EMPTY_MS_CONF);
84 | }
85 |
86 | static class UserIdValidator implements ConfigDef.Validator {
87 | @Override
88 | public void ensureValid(String key, Object o) {
89 | if (o instanceof List) {
90 | List userIds = (List) o;
91 | for (String userId : userIds) {
92 | if (null == Longs.tryParse(userId)) {
93 | throw new ConfigException(key, userId, "Could not parse to long.");
94 | }
95 | }
96 | }
97 | }
98 | }
99 |
100 | static final ConfigDef.Validator USERID_VALIDATOR = new UserIdValidator();
101 |
102 | public static ConfigDef conf() {
103 | return new ConfigDef()
104 | .define(TWITTER_DEBUG_CONF, Type.BOOLEAN, false, Importance.LOW, TWITTER_DEBUG_DOC)
105 | .define(TWITTER_OAUTH_CONSUMER_KEY_CONF, Type.PASSWORD, Importance.HIGH, TWITTER_OAUTH_CONSUMER_KEY_DOC)
106 | .define(TWITTER_OAUTH_SECRET_KEY_CONF, Type.PASSWORD, Importance.HIGH, TWITTER_OAUTH_SECRET_KEY_DOC)
107 | .define(TWITTER_OAUTH_ACCESS_TOKEN_CONF, Type.PASSWORD, Importance.HIGH, TWITTER_OAUTH_ACCESS_TOKEN_DOC)
108 | .define(TWITTER_OAUTH_ACCESS_TOKEN_SECRET_CONF, Type.PASSWORD, Importance.HIGH, TWITTER_OAUTH_ACCESS_TOKEN_SECRET_DOC)
109 | .define(FILTER_KEYWORDS_CONF, Type.LIST, Importance.HIGH, FILTER_KEYWORDS_DOC)
110 | .define(
111 | ConfigKeyBuilder.of(FILTER_USER_IDS_CONF, Type.LIST)
112 | .importance(Importance.HIGH)
113 | .documentation(FILTER_USER_IDS_DOC)
114 | .defaultValue(Collections.emptyList())
115 | .validator(USERID_VALIDATOR)
116 | .build()
117 | )
118 | .define(KAFKA_STATUS_TOPIC_CONF, Type.STRING, Importance.HIGH, KAFKA_STATUS_TOPIC_DOC)
119 | .define(PROCESS_DELETES_CONF, Type.BOOLEAN, Importance.HIGH, PROCESS_DELETES_DOC)
120 | .define(
121 | ConfigKeyBuilder.of(QUEUE_EMPTY_MS_CONF, Type.INT)
122 | .importance(Importance.LOW)
123 | .documentation(QUEUE_EMPTY_MS_DOC)
124 | .defaultValue(100)
125 | .validator(ConfigDef.Range.atLeast(10))
126 | .build()
127 | )
128 | .define(
129 | ConfigKeyBuilder.of(QUEUE_BATCH_SIZE_CONF, Type.INT)
130 | .importance(Importance.LOW)
131 | .documentation(QUEUE_BATCH_SIZE_DOC)
132 | .defaultValue(100)
133 | .validator(ConfigDef.Range.atLeast(1))
134 | .build()
135 | );
136 | }
137 |
138 |
139 | public Configuration configuration() {
140 | Properties properties = new Properties();
141 | /*
142 | Grab all of the key/values that have a key that starts with twitter. This will strip 'twitter.' from beginning of
143 | each key. This aligns with what the twitter4j framework is expecting.
144 | */
145 | properties.putAll(this.originalsWithPrefix("twitter."));
146 | return new PropertyConfiguration(properties);
147 | }
148 | }
149 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/twitter/TwitterSourceTask.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.twitter;
17 |
18 | import com.github.jcustenborder.kafka.connect.utils.VersionUtil;
19 | import com.github.jcustenborder.kafka.connect.utils.data.SourceRecordDeque;
20 | import com.github.jcustenborder.kafka.connect.utils.data.SourceRecordDequeBuilder;
21 | import com.google.common.base.Joiner;
22 | import com.google.common.collect.ImmutableMap;
23 | import org.apache.kafka.connect.data.Struct;
24 | import org.apache.kafka.connect.source.SourceRecord;
25 | import org.apache.kafka.connect.source.SourceTask;
26 | import org.slf4j.Logger;
27 | import org.slf4j.LoggerFactory;
28 | import twitter4j.FilterQuery;
29 | import twitter4j.StallWarning;
30 | import twitter4j.Status;
31 | import twitter4j.StatusDeletionNotice;
32 | import twitter4j.StatusListener;
33 | import twitter4j.TwitterStream;
34 | import twitter4j.TwitterStreamFactory;
35 |
36 | import java.util.List;
37 | import java.util.Map;
38 |
39 | public class TwitterSourceTask extends SourceTask implements StatusListener {
40 | static final Logger log = LoggerFactory.getLogger(TwitterSourceTask.class);
41 | SourceRecordDeque messageQueue;
42 |
43 | TwitterStream twitterStream;
44 | TwitterSourceConnectorConfig config;
45 |
46 | @Override
47 | public String version() {
48 | return VersionUtil.version(this.getClass());
49 | }
50 |
51 | @Override
52 | public void start(Map map) {
53 | this.config = new TwitterSourceConnectorConfig(map);
54 | this.messageQueue = SourceRecordDequeBuilder.of()
55 | .emptyWaitMs(this.config.queueEmptyMs)
56 | .batchSize(this.config.queueBatchSize)
57 | .build();
58 |
59 | TwitterStreamFactory twitterStreamFactory = new TwitterStreamFactory(this.config.configuration());
60 | this.twitterStream = twitterStreamFactory.getInstance();
61 | String[] keywords = this.config.filterKeywords.toArray(new String[0]);
62 | if (log.isInfoEnabled()) {
63 | log.info("Setting up filters. Keywords = {}", Joiner.on(", ").join(keywords));
64 | }
65 |
66 | FilterQuery filterQuery = new FilterQuery();
67 | filterQuery.track(keywords);
68 | if (!this.config.filterUserIds.isEmpty()) {
69 | long[] userIds = this.config.filterUserIds.stream().mapToLong(Long::valueOf).toArray();
70 | log.info("Setting up filters. userIds = {}", Joiner.on(", ").join(this.config.filterUserIds));
71 | filterQuery.follow(userIds);
72 | }
73 |
74 | if (log.isInfoEnabled()) {
75 | log.info("Starting the twitter stream.");
76 | }
77 | twitterStream.addListener(this);
78 | twitterStream.filter(filterQuery);
79 | }
80 |
81 | @Override
82 | public List poll() throws InterruptedException {
83 | return this.messageQueue.getBatch();
84 | }
85 |
86 | @Override
87 | public void stop() {
88 | if (log.isInfoEnabled()) {
89 | log.info("Shutting down twitter stream.");
90 | }
91 | twitterStream.shutdown();
92 | }
93 |
94 | @Override
95 | public void onStatus(Status status) {
96 | try {
97 | Struct keyStruct = new Struct(StatusConverter.STATUS_SCHEMA_KEY);
98 | Struct valueStruct = new Struct(StatusConverter.STATUS_SCHEMA);
99 |
100 | StatusConverter.convertKey(status, keyStruct);
101 | StatusConverter.convert(status, valueStruct);
102 |
103 | Map sourcePartition = ImmutableMap.of();
104 | Map sourceOffset = ImmutableMap.of();
105 |
106 | SourceRecord record = new SourceRecord(sourcePartition, sourceOffset, this.config.topic, StatusConverter.STATUS_SCHEMA_KEY, keyStruct, StatusConverter.STATUS_SCHEMA, valueStruct);
107 | this.messageQueue.add(record);
108 | } catch (Exception ex) {
109 | if (log.isErrorEnabled()) {
110 | log.error("Exception thrown", ex);
111 | }
112 | }
113 | }
114 |
115 | @Override
116 | public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
117 | if (!this.config.processDeletes) {
118 | return;
119 | }
120 |
121 | try {
122 | Struct keyStruct = new Struct(StatusConverter.SCHEMA_STATUS_DELETION_NOTICE_KEY);
123 |
124 | StatusConverter.convertKey(statusDeletionNotice, keyStruct);
125 |
126 | Map sourcePartition = ImmutableMap.of();
127 | Map sourceOffset = ImmutableMap.of();
128 |
129 | SourceRecord record = new SourceRecord(sourcePartition, sourceOffset, this.config.topic, StatusConverter.SCHEMA_STATUS_DELETION_NOTICE_KEY, keyStruct, null, null);
130 | this.messageQueue.add(record);
131 | } catch (Exception ex) {
132 | if (log.isErrorEnabled()) {
133 | log.error("Exception thrown", ex);
134 | }
135 | }
136 | }
137 |
138 | @Override
139 | public void onTrackLimitationNotice(int i) {
140 |
141 | }
142 |
143 | @Override
144 | public void onScrubGeo(long l, long l1) {
145 |
146 | }
147 |
148 | @Override
149 | public void onStallWarning(StallWarning stallWarning) {
150 | if (log.isWarnEnabled()) {
151 | log.warn("code = '{}' percentFull = '{}' - {}", stallWarning.getCode(), stallWarning.getPercentFull(), stallWarning.getMessage());
152 | }
153 | }
154 |
155 | @Override
156 | public void onException(Exception e) {
157 | if (log.isErrorEnabled()) {
158 | log.error("onException", e);
159 | }
160 | }
161 | }
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/twitter/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | @Title("Twitter")
17 | @Introduction("The Twitter plugin is used to pull data from Twitter and write it to Kafka.")
18 | @PluginOwner("jcustenborder")
19 | @PluginName("kafka-connect-twitter")
20 | package com.github.jcustenborder.kafka.connect.twitter;
21 |
22 | import com.github.jcustenborder.kafka.connect.utils.config.Introduction;
23 | import com.github.jcustenborder.kafka.connect.utils.config.PluginName;
24 | import com.github.jcustenborder.kafka.connect.utils.config.PluginOwner;
25 | import com.github.jcustenborder.kafka.connect.utils.config.Title;
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/twitter/DocumentationTest.java:
--------------------------------------------------------------------------------
1 | package com.github.jcustenborder.kafka.connect.twitter;
2 |
3 | import com.github.jcustenborder.kafka.connect.utils.BaseDocumentationTest;
4 | import org.apache.kafka.connect.data.Schema;
5 |
6 | import java.lang.reflect.Field;
7 | import java.lang.reflect.Modifier;
8 | import java.util.Arrays;
9 | import java.util.List;
10 | import java.util.stream.Collectors;
11 |
12 | public class DocumentationTest extends BaseDocumentationTest {
13 | static Schema schema(Field field) {
14 | try {
15 | return (Schema) field.get(null);
16 | } catch (IllegalAccessException e) {
17 | throw new IllegalStateException(e);
18 | }
19 | }
20 |
21 | @Override
22 | protected List schemas() {
23 | List schemas = Arrays.stream(StatusConverter.class.getFields())
24 | .filter(field -> Modifier.isFinal(field.getModifiers()))
25 | .filter(field -> Modifier.isStatic(field.getModifiers()))
26 | .filter(field -> Schema.class.equals(field.getType()))
27 | .map(DocumentationTest::schema)
28 | .collect(Collectors.toList());
29 | return schemas;
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/twitter/SchemaGeneratorTest.java:
--------------------------------------------------------------------------------
1 | package com.github.jcustenborder.kafka.connect.twitter;
2 |
3 | import com.google.common.base.CaseFormat;
4 | import org.junit.jupiter.api.Test;
5 | import org.reflections.Reflections;
6 | import org.reflections.util.ClasspathHelper;
7 | import org.reflections.util.ConfigurationBuilder;
8 | import twitter4j.MediaEntity;
9 | import twitter4j.TweetEntity;
10 |
11 | import java.lang.reflect.Method;
12 | import java.lang.reflect.Modifier;
13 | import java.util.ArrayList;
14 | import java.util.Comparator;
15 | import java.util.HashSet;
16 | import java.util.List;
17 | import java.util.Map;
18 | import java.util.Set;
19 | import java.util.stream.Collectors;
20 |
21 | public class SchemaGeneratorTest {
22 |
23 | List> list(Reflections reflections, Class cls) {
24 | List> classes = reflections.getSubTypesOf(cls)
25 | .stream()
26 | .filter(aClass -> Modifier.isInterface(aClass.getModifiers()))
27 | .collect(Collectors.toList());
28 | classes.sort(Comparator.comparing(Class::getName));
29 | return classes;
30 | }
31 |
32 | String schema(Class cls) {
33 | String result;
34 |
35 | if (String.class.equals(cls)) {
36 | result = "SchemaBuilder.string().optional().doc(\"\").build()";
37 | } else if (int.class.equals(cls)) {
38 | result = "SchemaBuilder.int32().optional().doc(\"\").build()";
39 | } else if (long.class.equals(cls)) {
40 | result = "SchemaBuilder.int64().optional().doc(\"\").build()";
41 | } else if (cls.isArray()) {
42 | String childSchema = schema(cls.getComponentType());
43 | result = String.format("SchemaBuilder.array(%s).optional().doc(\"\").build()", childSchema);
44 | } else if (Map.class.isAssignableFrom(cls)) {
45 | result = "SchemaBuilder.map(Schema.STRING_SCHEMA, SCHEMA_MEDIA_ENTITY_SIZE)";
46 |
47 | } else {
48 | result = "SCHEMA_" + CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, cls.getSimpleName()).replace('$', '_');
49 | }
50 |
51 |
52 | return result;
53 | }
54 |
55 | void processClass(Class cls, StringBuilder builder) {
56 |
57 | final String schemaConstantName;
58 | final String schemaName;
59 | final String typeName;
60 | final String convertMethodName;
61 |
62 | if (null == cls.getDeclaringClass()) {
63 | schemaConstantName = "SCHEMA_" + CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, cls.getSimpleName());
64 | schemaName = String.format("com.github.jcustenborder.kafka.connect.twitter.%s", cls.getSimpleName());
65 | typeName = cls.getSimpleName();
66 | convertMethodName = String.format("convert%s", cls.getSimpleName());
67 | } else {
68 | schemaConstantName = "SCHEMA_" + CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, cls.getDeclaringClass().getSimpleName() + cls.getSimpleName());
69 | typeName = String.format("%s.%s", cls.getDeclaringClass().getSimpleName(), cls.getSimpleName());
70 | schemaName = String.format("com.github.jcustenborder.kafka.connect.twitter.%s.%s", cls.getSimpleName(), cls.getDeclaringClass().getSimpleName());
71 | convertMethodName = String.format("convert%s%s", cls.getDeclaringClass().getSimpleName(), cls.getSimpleName());
72 | }
73 |
74 |
75 | builder.append(String.format("public static final Schema %s =SchemaBuilder.struct()\n", schemaConstantName));
76 | builder.append(String.format(" .name(\"%s\")\n", schemaName));
77 | builder.append(" .doc(\"\")\n");
78 |
79 | Set methods = new HashSet<>();
80 | for (Method method : cls.getMethods()) {
81 | String methodName = method.getName().replace("get", "");
82 | if (!methods.add(methodName)) {
83 | continue;
84 | }
85 | String expectedSchema = schema(method.getReturnType());
86 | builder.append(String.format(" .field(\"%s\", %s)\n", methodName, expectedSchema));
87 | }
88 | builder.append(" .build();\n\n");
89 |
90 | methods.clear();
91 | String variableName = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_CAMEL, cls.getSimpleName());
92 | builder.append(String.format("static Struct %s(%s %s) {\n", convertMethodName, typeName, variableName));
93 | builder.append(String.format(" return new Struct(%s)", schemaConstantName));
94 | for (Method method : cls.getMethods()) {
95 | String methodName = method.getName().replace("get", "");
96 | if (!methods.add(methodName)) {
97 | continue;
98 | }
99 | builder.append(String.format("\n .put(\"%s\", %s.%s())", methodName, variableName, method.getName()));
100 | }
101 | builder.append(";\n }\n");
102 |
103 | builder.append("\n");
104 | builder.append(String.format("public static List convert(%s[] items) {\n", typeName));
105 | builder.append(" List result = new ArrayList<>();\n");
106 | builder.append(" if(null==items) {\n");
107 | builder.append(" return result;\n");
108 | builder.append(" }\n");
109 | builder.append(String.format(" for(%s item: items) {\n", typeName));
110 | builder.append(String.format(" Struct struct = %s(item);\n", convertMethodName));
111 | builder.append(" result.add(struct);\n");
112 | builder.append(" }\n");
113 | builder.append(" return result;\n");
114 | builder.append("}\n");
115 |
116 | // }
117 | // public static List convert(UserMentionEntity[] userMentionEntities) {
118 | // List result = new ArrayList<>();
119 | // if(null==userMentionEntities) {
120 | // return result;
121 | // }
122 | // for(UserMentionEntity item: userMentionEntities) {
123 | // Struct struct = convertUserMentionEntity(item);
124 | // result.add(struct);
125 | // }
126 | // return result;
127 | // }
128 |
129 |
130 | }
131 |
132 | @Test
133 | public void tweetEntities() {
134 | Reflections reflections = new Reflections(new ConfigurationBuilder()
135 | .setUrls(ClasspathHelper.forJavaClassPath())
136 | .forPackages(TweetEntity.class.getPackage().getName())
137 | );
138 |
139 | List> allClasses = new ArrayList<>();
140 | List> classes = list(reflections, TweetEntity.class);
141 | allClasses.add(MediaEntity.Variant.class);
142 | allClasses.add(MediaEntity.Size.class);
143 | allClasses.addAll(classes);
144 |
145 |
146 | for (Class cls : allClasses) {
147 | StringBuilder builder = new StringBuilder();
148 | processClass(cls, builder);
149 |
150 | System.out.println(builder);
151 | }
152 |
153 |
154 | }
155 |
156 |
157 | }
158 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/twitter/StatusConverterTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.twitter;
17 |
18 | import org.apache.kafka.connect.data.Struct;
19 | import org.junit.jupiter.api.Test;
20 | import twitter4j.GeoLocation;
21 | import twitter4j.Place;
22 | import twitter4j.Status;
23 | import twitter4j.StatusDeletionNotice;
24 | import twitter4j.User;
25 |
26 | import java.util.ArrayList;
27 | import java.util.Date;
28 | import java.util.List;
29 |
30 | import static org.junit.jupiter.api.Assertions.assertEquals;
31 | import static org.junit.jupiter.api.Assertions.assertNotNull;
32 | import static org.mockito.Mockito.mock;
33 | import static org.mockito.Mockito.when;
34 |
35 | public class StatusConverterTest {
36 |
37 | public static GeoLocation mockGeoLocation() {
38 | return new GeoLocation(30.2672D, 97.7431D);
39 | }
40 |
41 | public static Place mockPlace() {
42 | Place place = mock(Place.class);
43 | when(place.getName()).thenReturn("Example place");
44 | when(place.getStreetAddress()).thenReturn("123 Example St");
45 | when(place.getCountryCode()).thenReturn("US");
46 | when(place.getId()).thenReturn("asdfaisdfasd");
47 | when(place.getCountry()).thenReturn("United States");
48 | when(place.getPlaceType()).thenReturn("ADF");
49 | when(place.getURL()).thenReturn("http://www.example.com/");
50 | when(place.getFullName()).thenReturn("Example place");
51 | return place;
52 | }
53 |
54 | public static Status mockStatus() {
55 | Status status = mock(Status.class);
56 | User user = mockUser();
57 | GeoLocation geoLocation = mockGeoLocation();
58 | Place place = mockPlace();
59 |
60 | when(status.getCreatedAt()).thenReturn(new Date(1471667709998L));
61 | when(status.getId()).thenReturn(9823452L);
62 | when(status.getText()).thenReturn("This is a twit");
63 | when(status.getSource()).thenReturn("foo");
64 | when(status.isTruncated()).thenReturn(false);
65 | when(status.getInReplyToStatusId()).thenReturn(2345234L);
66 | when(status.getInReplyToUserId()).thenReturn(8756786L);
67 | when(status.getInReplyToScreenName()).thenReturn("foo");
68 | when(status.getGeoLocation()).thenReturn(geoLocation);
69 | when(status.getPlace()).thenReturn(place);
70 | when(status.isFavorited()).thenReturn(true);
71 | when(status.isRetweeted()).thenReturn(false);
72 | when(status.getFavoriteCount()).thenReturn(1234);
73 | when(status.getUser()).thenReturn(user);
74 | when(status.isRetweet()).thenReturn(false);
75 | when(status.getContributors()).thenReturn(new long[]{431234L, 986789678L});
76 | when(status.getRetweetCount()).thenReturn(1234);
77 | when(status.isRetweetedByMe()).thenReturn(false);
78 | when(status.getCurrentUserRetweetId()).thenReturn(653456345L);
79 | when(status.isPossiblySensitive()).thenReturn(false);
80 | when(status.getLang()).thenReturn("en-US");
81 | when(status.getWithheldInCountries()).thenReturn(new String[]{"CN"});
82 |
83 | return status;
84 | }
85 |
86 | public static User mockUser() {
87 | User user = mock(User.class);
88 |
89 | when(user.getId()).thenReturn(1234L);
90 | when(user.getName()).thenReturn("Example User");
91 | when(user.getScreenName()).thenReturn("example");
92 | when(user.getLocation()).thenReturn("Austin, TX");
93 | when(user.getDescription()).thenReturn("This is a description");
94 | when(user.isContributorsEnabled()).thenReturn(true);
95 | when(user.getProfileImageURL()).thenReturn("http://i.twittercdn.com/profile.jpg");
96 | when(user.getBiggerProfileImageURL()).thenReturn("http://i.twittercdn.com/biggerprofile.jpg");
97 | when(user.getMiniProfileImageURL()).thenReturn("http://i.twittercdn.com/mini.profile.jpg");
98 | when(user.getOriginalProfileImageURL()).thenReturn("http://i.twittercdn.com/original.profile.jpg");
99 | when(user.getProfileImageURLHttps()).thenReturn("https://i.twittercdn.com/profile.jpg");
100 | when(user.getBiggerProfileImageURLHttps()).thenReturn("https://i.twittercdn.com/bigger.profile.jpg");
101 | when(user.getMiniProfileImageURLHttps()).thenReturn("https://i.twittercdn.com/mini.profile.jpg");
102 | when(user.getOriginalProfileImageURLHttps()).thenReturn("https://i.twittercdn.com/original.profile.jpg");
103 | when(user.isDefaultProfileImage()).thenReturn(true);
104 | when(user.getURL()).thenReturn("https://www.twitter.com/example");
105 | when(user.isProtected()).thenReturn(false);
106 | when(user.getFollowersCount()).thenReturn(54245);
107 | when(user.getProfileBackgroundColor()).thenReturn("#ffffff");
108 | when(user.getProfileTextColor()).thenReturn("#000000");
109 | when(user.getProfileLinkColor()).thenReturn("#aaaaaa");
110 | when(user.getProfileSidebarFillColor()).thenReturn("#333333");
111 | when(user.getProfileSidebarBorderColor()).thenReturn("#555555");
112 | when(user.isProfileUseBackgroundImage()).thenReturn(true);
113 | when(user.isDefaultProfile()).thenReturn(true);
114 | when(user.isShowAllInlineMedia()).thenReturn(true);
115 | when(user.getFriendsCount()).thenReturn(452345234);
116 | when(user.getCreatedAt()).thenReturn(new Date(1471665653209L));
117 | when(user.getFavouritesCount()).thenReturn(12341);
118 | when(user.getUtcOffset()).thenReturn(8);
119 | when(user.getTimeZone()).thenReturn("UTC");
120 | when(user.getProfileBackgroundImageURL()).thenReturn("https://i.twittercdn.com/original.background.jpg");
121 | when(user.getProfileBackgroundImageUrlHttps()).thenReturn("https://i.twittercdn.com/original.background.jpg");
122 | when(user.getProfileBannerURL()).thenReturn("https://i.twittercdn.com/original.banner.jpg");
123 | when(user.getProfileBannerRetinaURL()).thenReturn("https://i.twittercdn.com/original.banner.jpg");
124 | when(user.getProfileBannerIPadURL()).thenReturn("https://i.twittercdn.com/original.banner.jpg");
125 | when(user.getProfileBannerIPadRetinaURL()).thenReturn("https://i.twittercdn.com/original.banner.jpg");
126 | when(user.getProfileBannerMobileURL()).thenReturn("https://i.twittercdn.com/original.banner.jpg");
127 | when(user.getProfileBannerMobileRetinaURL()).thenReturn("https://i.twittercdn.com/original.banner.jpg");
128 | when(user.isProfileBackgroundTiled()).thenReturn(false);
129 | when(user.getLang()).thenReturn("en-us");
130 | when(user.getStatusesCount()).thenReturn(543);
131 | when(user.isGeoEnabled()).thenReturn(true);
132 | when(user.isVerified()).thenReturn(true);
133 | when(user.isTranslator()).thenReturn(false);
134 | when(user.getListedCount()).thenReturn(4);
135 | when(user.isFollowRequestSent()).thenReturn(false);
136 | when(user.getWithheldInCountries()).thenReturn(new String[]{"CN"});
137 |
138 |
139 | return user;
140 | }
141 |
142 | public static StatusDeletionNotice mockStatusDeletionNotice() {
143 | StatusDeletionNotice statusDeletionNotice = mock(StatusDeletionNotice.class);
144 | when(statusDeletionNotice.getStatusId()).thenReturn(1234565345L);
145 | when(statusDeletionNotice.getUserId()).thenReturn(6543456354L);
146 | return statusDeletionNotice;
147 | }
148 |
149 | List convert(long[] values) {
150 | List list = new ArrayList<>();
151 | for (Long l : values) {
152 | list.add(l);
153 | }
154 | return list;
155 | }
156 |
157 | List convert(String[] values) {
158 | List list = new ArrayList<>();
159 | for (String l : values) {
160 | list.add(l);
161 | }
162 | return list;
163 | }
164 |
165 | void assertStatus(Status status, Struct struct) {
166 | assertEquals(status.getCreatedAt(), struct.get("CreatedAt"), "CreatedAt does not match.");
167 | assertEquals(status.getId(), struct.get("Id"), "Id does not match.");
168 | assertEquals(status.getText(), struct.get("Text"), "Text does not match.");
169 | assertEquals(status.getSource(), struct.get("Source"), "Source does not match.");
170 | assertEquals(status.isTruncated(), struct.get("Truncated"), "Truncated does not match.");
171 | assertEquals(status.getInReplyToStatusId(), struct.get("InReplyToStatusId"), "InReplyToStatusId does not match.");
172 | assertEquals(status.getInReplyToUserId(), struct.get("InReplyToUserId"), "InReplyToUserId does not match.");
173 | assertEquals(status.getInReplyToScreenName(), struct.get("InReplyToScreenName"), "InReplyToScreenName does not match.");
174 | assertEquals(status.isFavorited(), struct.get("Favorited"), "Favorited does not match.");
175 | assertEquals(status.isRetweeted(), struct.get("Retweeted"), "Retweeted does not match.");
176 | assertEquals(status.getFavoriteCount(), struct.get("FavoriteCount"), "FavoriteCount does not match.");
177 | assertEquals(status.isRetweet(), struct.get("Retweet"), "Retweet does not match.");
178 | assertEquals(status.getRetweetCount(), struct.get("RetweetCount"), "RetweetCount does not match.");
179 | assertEquals(status.isRetweetedByMe(), struct.get("RetweetedByMe"), "RetweetedByMe does not match.");
180 | assertEquals(status.getCurrentUserRetweetId(), struct.get("CurrentUserRetweetId"), "CurrentUserRetweetId does not match.");
181 | assertEquals(status.isPossiblySensitive(), struct.get("PossiblySensitive"), "PossiblySensitive does not match.");
182 | assertEquals(status.getLang(), struct.get("Lang"), "Lang does not match.");
183 |
184 | assertUser(status.getUser(), struct.getStruct("User"));
185 | assertPlace(status.getPlace(), struct.getStruct("Place"));
186 | assertGeoLocation(status.getGeoLocation(), struct.getStruct("GeoLocation"));
187 |
188 | assertEquals(convert(status.getContributors()), struct.getArray("Contributors"), "Contributors does not match.");
189 | assertEquals(convert(status.getWithheldInCountries()), struct.get("WithheldInCountries"), "WithheldInCountries does not match.");
190 | }
191 |
192 | void assertGeoLocation(GeoLocation geoLocation, Struct struct) {
193 | assertEquals(struct.getFloat64("Latitude"), 1, geoLocation.getLatitude());
194 | assertEquals(struct.getFloat64("Longitude"), 1, geoLocation.getLongitude());
195 | }
196 |
197 | void assertPlace(Place place, Struct struct) {
198 | assertEquals(place.getName(), struct.get("Name"), "Name does not match.");
199 | assertEquals(place.getStreetAddress(), struct.get("StreetAddress"), "StreetAddress does not match.");
200 | assertEquals(place.getCountryCode(), struct.get("CountryCode"), "CountryCode does not match.");
201 | assertEquals(place.getId(), struct.get("Id"), "Id does not match.");
202 | assertEquals(place.getCountry(), struct.get("Country"), "Country does not match.");
203 | assertEquals(place.getPlaceType(), struct.get("PlaceType"), "PlaceType does not match.");
204 | assertEquals(place.getURL(), struct.get("URL"), "URL does not match.");
205 | assertEquals(place.getFullName(), struct.get("FullName"), "FullName does not match.");
206 | }
207 |
208 | void assertUser(User user, Struct struct) {
209 | assertNotNull(struct, "struct should not be null.");
210 | assertEquals(user.getId(), struct.get("Id"), "Id does not match.");
211 | assertEquals(user.getName(), struct.get("Name"), "Name does not match.");
212 | assertEquals(user.getScreenName(), struct.get("ScreenName"), "ScreenName does not match.");
213 | assertEquals(user.getLocation(), struct.get("Location"), "Location does not match.");
214 | assertEquals(user.getDescription(), struct.get("Description"), "Description does not match.");
215 | assertEquals(user.isContributorsEnabled(), struct.get("ContributorsEnabled"), "ContributorsEnabled does not match.");
216 | assertEquals(user.getProfileImageURL(), struct.get("ProfileImageURL"), "ProfileImageURL does not match.");
217 | assertEquals(user.getBiggerProfileImageURL(), struct.get("BiggerProfileImageURL"), "BiggerProfileImageURL does not match.");
218 | assertEquals(user.getMiniProfileImageURL(), struct.get("MiniProfileImageURL"), "MiniProfileImageURL does not match.");
219 | assertEquals(user.getOriginalProfileImageURL(), struct.get("OriginalProfileImageURL"), "OriginalProfileImageURL does not match.");
220 | assertEquals(user.getProfileImageURLHttps(), struct.get("ProfileImageURLHttps"), "ProfileImageURLHttps does not match.");
221 | assertEquals(user.getBiggerProfileImageURLHttps(), struct.get("BiggerProfileImageURLHttps"), "BiggerProfileImageURLHttps does not match.");
222 | assertEquals(user.getMiniProfileImageURLHttps(), struct.get("MiniProfileImageURLHttps"), "MiniProfileImageURLHttps does not match.");
223 | assertEquals(user.getOriginalProfileImageURLHttps(), struct.get("OriginalProfileImageURLHttps"), "OriginalProfileImageURLHttps does not match.");
224 | assertEquals(user.isDefaultProfileImage(), struct.get("DefaultProfileImage"), "DefaultProfileImage does not match.");
225 | assertEquals(user.getURL(), struct.get("URL"), "URL does not match.");
226 | assertEquals(user.isProtected(), struct.get("Protected"), "Protected does not match.");
227 | assertEquals(user.getFollowersCount(), struct.get("FollowersCount"), "FollowersCount does not match.");
228 | assertEquals(user.getProfileBackgroundColor(), struct.get("ProfileBackgroundColor"), "ProfileBackgroundColor does not match.");
229 | assertEquals(user.getProfileTextColor(), struct.get("ProfileTextColor"), "ProfileTextColor does not match.");
230 | assertEquals(user.getProfileLinkColor(), struct.get("ProfileLinkColor"), "ProfileLinkColor does not match.");
231 | assertEquals(user.getProfileSidebarFillColor(), struct.get("ProfileSidebarFillColor"), "ProfileSidebarFillColor does not match.");
232 | assertEquals(user.getProfileSidebarBorderColor(), struct.get("ProfileSidebarBorderColor"), "ProfileSidebarBorderColor does not match.");
233 | assertEquals(user.isProfileUseBackgroundImage(), struct.get("ProfileUseBackgroundImage"), "ProfileUseBackgroundImage does not match.");
234 | assertEquals(user.isDefaultProfile(), struct.get("DefaultProfile"), "DefaultProfile does not match.");
235 | assertEquals(user.isShowAllInlineMedia(), struct.get("ShowAllInlineMedia"), "ShowAllInlineMedia does not match.");
236 | assertEquals(user.getFriendsCount(), struct.get("FriendsCount"), "FriendsCount does not match.");
237 | assertEquals(user.getCreatedAt(), struct.get("CreatedAt"), "CreatedAt does not match.");
238 | assertEquals(user.getFavouritesCount(), struct.get("FavouritesCount"), "FavouritesCount does not match.");
239 | assertEquals(user.getUtcOffset(), struct.get("UtcOffset"), "UtcOffset does not match.");
240 | assertEquals(user.getTimeZone(), struct.get("TimeZone"), "TimeZone does not match.");
241 | assertEquals(user.getProfileBackgroundImageURL(), struct.get("ProfileBackgroundImageURL"), "ProfileBackgroundImageURL does not match.");
242 | assertEquals(user.getProfileBackgroundImageUrlHttps(), struct.get("ProfileBackgroundImageUrlHttps"), "ProfileBackgroundImageUrlHttps does not match.");
243 | assertEquals(user.getProfileBannerURL(), struct.get("ProfileBannerURL"), "ProfileBannerURL does not match.");
244 | assertEquals(user.getProfileBannerRetinaURL(), struct.get("ProfileBannerRetinaURL"), "ProfileBannerRetinaURL does not match.");
245 | assertEquals(user.getProfileBannerIPadURL(), struct.get("ProfileBannerIPadURL"), "ProfileBannerIPadURL does not match.");
246 | assertEquals(user.getProfileBannerIPadRetinaURL(), struct.get("ProfileBannerIPadRetinaURL"), "ProfileBannerIPadRetinaURL does not match.");
247 | assertEquals(user.getProfileBannerMobileURL(), struct.get("ProfileBannerMobileURL"), "ProfileBannerMobileURL does not match.");
248 | assertEquals(user.getProfileBannerMobileRetinaURL(), struct.get("ProfileBannerMobileRetinaURL"), "ProfileBannerMobileRetinaURL does not match.");
249 | assertEquals(user.isProfileBackgroundTiled(), struct.get("ProfileBackgroundTiled"), "ProfileBackgroundTiled does not match.");
250 | assertEquals(user.getLang(), struct.get("Lang"), "Lang does not match.");
251 | assertEquals(user.getStatusesCount(), struct.get("StatusesCount"), "StatusesCount does not match.");
252 | assertEquals(user.isGeoEnabled(), struct.get("GeoEnabled"), "GeoEnabled does not match.");
253 | assertEquals(user.isVerified(), struct.get("Verified"), "Verified does not match.");
254 | assertEquals(user.isTranslator(), struct.get("Translator"), "Translator does not match.");
255 | assertEquals(user.getListedCount(), struct.get("ListedCount"), "ListedCount does not match.");
256 | assertEquals(user.isFollowRequestSent(), struct.get("FollowRequestSent"), "FollowRequestSent does not match.");
257 | }
258 |
259 | void assertKey(Status status, Struct struct) {
260 | assertEquals(status.getId(), struct.get("Id"), "Id does not match.");
261 | }
262 |
263 | @Test
264 | public void convertStatus() {
265 | Status status = mockStatus();
266 | Struct struct = new Struct(StatusConverter.STATUS_SCHEMA);
267 | StatusConverter.convert(status, struct);
268 | assertStatus(status, struct);
269 | }
270 |
271 | @Test
272 | public void convertUser() {
273 | User user = mockUser();
274 | Struct struct = new Struct(StatusConverter.USER_SCHEMA);
275 | StatusConverter.convert(user, struct);
276 | assertUser(user, struct);
277 | }
278 |
279 | @Test
280 | public void convertPlace() {
281 | Place place = mockPlace();
282 | Struct struct = new Struct(StatusConverter.PLACE_SCHEMA);
283 | StatusConverter.convert(place, struct);
284 | assertPlace(place, struct);
285 | }
286 |
287 | @Test
288 | public void convertGeoLocation() {
289 | GeoLocation geoLocation = mockGeoLocation();
290 | Struct struct = new Struct(StatusConverter.GEO_LOCATION_SCHEMA);
291 | StatusConverter.convert(geoLocation, struct);
292 | assertGeoLocation(geoLocation, struct);
293 | }
294 |
295 | @Test
296 | public void convertStatusKey() {
297 | Status status = mockStatus();
298 | Struct struct = new Struct(StatusConverter.STATUS_SCHEMA_KEY);
299 | StatusConverter.convertKey(status, struct);
300 | assertKey(status, struct);
301 | }
302 |
303 | void assertStatusDeletionNotice(StatusDeletionNotice statusDeletionNotice, Struct struct) {
304 | assertEquals(statusDeletionNotice.getStatusId(), struct.get("StatusId"), "StatusId does not match.");
305 | assertEquals(statusDeletionNotice.getUserId(), struct.get("UserId"), "UserId does not match.");
306 | }
307 |
308 | void assertStatusDeletionNoticeKey(StatusDeletionNotice statusDeletionNotice, Struct struct) {
309 | assertEquals(statusDeletionNotice.getStatusId(), struct.get("StatusId"), "StatusId does not match.");
310 | }
311 |
312 | @Test
313 | public void convertStatusDeletionNotice() {
314 | StatusDeletionNotice statusDeletionNotice = mockStatusDeletionNotice();
315 | Struct struct = new Struct(StatusConverter.SCHEMA_STATUS_DELETION_NOTICE);
316 | StatusConverter.convert(statusDeletionNotice, struct);
317 | assertStatusDeletionNotice(statusDeletionNotice, struct);
318 | }
319 |
320 | @Test
321 | public void convertKeyStatusDeletionNotice() {
322 | StatusDeletionNotice statusDeletionNotice = mockStatusDeletionNotice();
323 | Struct struct = new Struct(StatusConverter.SCHEMA_STATUS_DELETION_NOTICE_KEY);
324 | StatusConverter.convertKey(statusDeletionNotice, struct);
325 | assertStatusDeletionNoticeKey(statusDeletionNotice, struct);
326 | }
327 | }
328 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/twitter/TwitterSourceConnectorTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.twitter;
17 |
18 |
19 | import com.google.common.base.Joiner;
20 | import com.google.common.collect.ImmutableMap;
21 | import org.junit.jupiter.api.BeforeEach;
22 | import org.junit.jupiter.api.DynamicTest;
23 | import org.junit.jupiter.api.TestFactory;
24 |
25 | import java.util.ArrayList;
26 | import java.util.Arrays;
27 | import java.util.LinkedHashMap;
28 | import java.util.List;
29 | import java.util.Map;
30 | import java.util.stream.Stream;
31 |
32 | import static org.junit.jupiter.api.Assertions.assertEquals;
33 | import static org.junit.jupiter.api.DynamicTest.dynamicTest;
34 |
35 | public class TwitterSourceConnectorTest {
36 |
37 | TwitterSourceConnector connector;
38 | Map defaultSettings;
39 |
40 | @BeforeEach
41 | public void setup() {
42 | this.connector = new TwitterSourceConnector();
43 | this.defaultSettings = new LinkedHashMap<>();
44 | this.defaultSettings.put(TwitterSourceConnectorConfig.TWITTER_OAUTH_ACCESS_TOKEN_CONF, "xxxxxx");
45 | this.defaultSettings.put(TwitterSourceConnectorConfig.TWITTER_OAUTH_SECRET_KEY_CONF, "xxxxxx");
46 | this.defaultSettings.put(TwitterSourceConnectorConfig.TWITTER_OAUTH_CONSUMER_KEY_CONF, "xxxxxx");
47 | this.defaultSettings.put(TwitterSourceConnectorConfig.TWITTER_OAUTH_ACCESS_TOKEN_SECRET_CONF, "xxxxxx");
48 | this.defaultSettings.put(TwitterSourceConnectorConfig.KAFKA_STATUS_TOPIC_CONF, "xxxxxx");
49 | this.defaultSettings.put(TwitterSourceConnectorConfig.PROCESS_DELETES_CONF, "false");
50 |
51 | }
52 |
53 | List> expectedSettings(List... keywords) {
54 | List> result = new ArrayList<>();
55 | for (List keywordSet : keywords) {
56 | Map settings = new LinkedHashMap<>(this.defaultSettings);
57 | settings.put(TwitterSourceConnectorConfig.FILTER_KEYWORDS_CONF, Joiner.on(',').join(keywordSet));
58 | result.add(settings);
59 | }
60 | return result;
61 | }
62 |
63 | @TestFactory
64 | public Stream taskConfigs() {
65 |
66 | Map>> testCases = ImmutableMap.of(
67 | 1, expectedSettings(Arrays.asList("one", "two", "three")),
68 | 2, expectedSettings(Arrays.asList("one", "three"), Arrays.asList("two")),
69 | 3, expectedSettings(Arrays.asList("one"), Arrays.asList("two"), Arrays.asList("three"))
70 | );
71 |
72 | return testCases.entrySet().stream()
73 | .map(e -> dynamicTest(e.getKey().toString(), () -> {
74 | this.defaultSettings.put(TwitterSourceConnectorConfig.FILTER_KEYWORDS_CONF, "one,two,three");
75 | this.connector.start(this.defaultSettings);
76 | List> taskConfigs = this.connector.taskConfigs(e.getKey());
77 | assertEquals(
78 | e.getValue(),
79 | taskConfigs
80 | );
81 | }));
82 |
83 |
84 | }
85 |
86 | }
87 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/twitter/TwitterSourceTaskTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2016 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.twitter;
17 |
18 |
19 | import org.junit.jupiter.api.Test;
20 |
21 | public class TwitterSourceTaskTest {
22 | @Test
23 | public void test() {
24 | // Congrats on a passing test!
25 | }
26 | }
--------------------------------------------------------------------------------
/src/test/resources/logback.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
6 |
7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
--------------------------------------------------------------------------------