├── .classpath ├── .gitignore ├── .project ├── .settings ├── org.eclipse.core.resources.prefs ├── org.eclipse.jdt.core.prefs └── org.eclipse.m2e.core.prefs ├── README.md ├── nb-configuration.xml ├── nbactions.xml ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── hortonworks │ │ ├── ConsumerGroupExample.java │ │ ├── ConsumerTest.java │ │ ├── Field.java │ │ ├── JsonUtils.java │ │ ├── KafkaToHBaseConfig.java │ │ ├── MappingConfig.java │ │ ├── Notifier.java │ │ └── Util.java └── resources │ └── kafka.json └── test └── java └── com └── hortonworks └── JsonUtilsTest.java /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | KafkaHBaseBenchmark 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/main/java=UTF-8 3 | encoding//src/main/resources=UTF-8 4 | encoding//src/test/java=UTF-8 5 | encoding/=UTF-8 6 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 3 | org.eclipse.jdt.core.compiler.compliance=1.7 4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 5 | org.eclipse.jdt.core.compiler.source=1.7 6 | -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KafkaHBaseBenchmark 2 | -------------------------------------------------------------------------------- /nb-configuration.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 9 | 10 | 16 | apache20 17 | 18 | 19 | -------------------------------------------------------------------------------- /nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | run 5 | 6 | jar 7 | 8 | 9 | process-classes 10 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 11 | 12 | 13 | -classpath %classpath com.hortonworks.kafka.ConsumerGroupExample 14 | java 15 | 16 | 17 | 18 | debug 19 | 20 | jar 21 | 22 | 23 | process-classes 24 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 25 | 26 | 27 | -Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath com.hortonworks.kafka.ConsumerGroupExample 28 | java 29 | true 30 | 31 | 32 | 33 | profile 34 | 35 | jar 36 | 37 | 38 | process-classes 39 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 40 | 41 | 42 | -classpath %classpath com.hortonworks.kafka.ConsumerGroupExample 43 | java 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.jet 8 | KafkaHBaseBenchmark 9 | 1.0-SNAPSHOT 10 | jar 11 | 12 | 13 | UTF-8 14 | 1.7 15 | 1.7 16 | 0.8.2.2.3.0.0-2557 17 | 18 | 19 | 20 | 21 | 22 | org.apache.maven.plugins 23 | maven-compiler-plugin 24 | 2.5.1 25 | 26 | 1.7 27 | 1.7 28 | 29 | 30 | 31 | 32 | org.apache.maven.plugins 33 | maven-jar-plugin 34 | 2.6 35 | 36 | 37 | 38 | org.apache.maven.plugins 39 | maven-assembly-plugin 40 | 2.5.5 41 | 42 | 43 | jar-with-dependencies 44 | 45 | 46 | 47 | com.hortonworks.ConsumerGroupExample 48 | 49 | 50 | 51 | 52 | 53 | package 54 | 55 | single 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | HDPReleases 66 | HDP Releases 67 | http://repo.hortonworks.com/content/repositories/public 68 | default 69 | 70 | true 71 | always 72 | warn 73 | 74 | 75 | false 76 | never 77 | fail 78 | 79 | 80 | 81 | 82 | 83 | 84 | org.apache.kafka 85 | kafka-clients 86 | ${kafka.version} 87 | 88 | 89 | org.apache.kafka 90 | kafka_2.10 91 | ${kafka.version} 92 | 93 | 94 | javax.jms 95 | jms 96 | 97 | 98 | com.sun.jdmk 99 | jmxtools 100 | 101 | 102 | com.sun.jmx 103 | jmxri 104 | 105 | 106 | org.slf4j 107 | slf4j-log4j12 108 | 109 | 110 | 111 | 112 | org.apache.hbase 113 | hbase-client 114 | 1.1.1 115 | jar 116 | 117 | 118 | org.apache.hadoop 119 | hadoop-hdfs 120 | 2.7.1 121 | jar 122 | 123 | 124 | org.apache.hadoop 125 | hadoop-common 126 | 2.7.1 127 | jar 128 | 129 | 130 | com.google.code.gson 131 | gson 132 | 2.3.1 133 | 134 | 135 | org.apache.commons 136 | commons-lang3 137 | 3.4 138 | jar 139 | 140 | 141 | KafkaHBaseBenchmark 142 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/ConsumerGroupExample.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 aervits. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.hortonworks; 17 | 18 | /** 19 | * 20 | * @author aervits 21 | * 22 | * https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example 23 | */ 24 | 25 | import java.io.IOException; 26 | import java.util.ArrayList; 27 | import java.util.Collection; 28 | import java.util.HashMap; 29 | import java.util.List; 30 | import java.util.Map; 31 | import java.util.Properties; 32 | import java.util.Set; 33 | import java.util.concurrent.ExecutorService; 34 | import java.util.concurrent.Executors; 35 | import java.util.concurrent.LinkedBlockingQueue; 36 | import java.util.concurrent.TimeUnit; 37 | import java.util.logging.Level; 38 | import java.util.logging.Logger; 39 | 40 | import kafka.consumer.ConsumerConfig; 41 | import kafka.consumer.KafkaStream; 42 | import kafka.javaapi.consumer.ConsumerConnector; 43 | 44 | import org.apache.hadoop.conf.Configuration; 45 | import org.apache.hadoop.hbase.HBaseConfiguration; 46 | import org.apache.hadoop.hbase.TableName; 47 | import org.apache.hadoop.hbase.client.BufferedMutator; 48 | import org.apache.hadoop.hbase.client.BufferedMutatorParams; 49 | import org.apache.hadoop.hbase.client.Connection; 50 | import org.apache.hadoop.hbase.client.ConnectionFactory; 51 | import org.apache.hadoop.hbase.client.Put; 52 | import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; 53 | import org.apache.hadoop.hbase.util.Bytes; 54 | 55 | public class ConsumerGroupExample { 56 | 57 | private int total = 0; 58 | 59 | public synchronized int getTotal() { 60 | return total; 61 | } 62 | 63 | public synchronized void setTotal(int total) { 64 | this.total = total; 65 | } 66 | public synchronized void updateTotal(int newNumber) { 67 | this.total = total +newNumber; 68 | } 69 | 70 | private final ConsumerConnector consumer; 71 | private final String topic; 72 | private ExecutorService executor; 73 | private static final byte[] CF_DEFAULT = Bytes.toBytes("cf"); 74 | private static final byte[] CF_FAMILY = Bytes.toBytes("message"); 75 | private static final Logger LOG = Logger 76 | .getLogger(ConsumerGroupExample.class.getName()); 77 | protected LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); 78 | 79 | private ExecutorService executorConsumer; 80 | 81 | public ConsumerGroupExample(String a_zookeeper, String a_groupId, 82 | String a_topic) { 83 | consumer = kafka.consumer.Consumer 84 | .createJavaConsumerConnector(createConsumerConfig(a_zookeeper, 85 | a_groupId)); 86 | this.topic = a_topic; 87 | } 88 | 89 | public void shutdown() { 90 | if (consumer != null) { 91 | consumer.shutdown(); 92 | } 93 | if (executor != null) { 94 | executor.shutdown(); 95 | } 96 | try { 97 | if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) { 98 | System.out 99 | .println("Timed out waiting for consumer threads to shut down, exiting uncleanly"); 100 | } 101 | } catch (InterruptedException e) { 102 | System.out 103 | .println("Interrupted during shutdown, exiting uncleanly"); 104 | } 105 | } 106 | 107 | public void run(int a_numThreads) { 108 | Map topicCountMap = new HashMap<>(); 109 | topicCountMap.put(topic, a_numThreads); 110 | Map>> consumerMap = consumer 111 | .createMessageStreams(topicCountMap); 112 | System.out.println("consumer map size=" + consumerMap.size()); 113 | Collection>> values = consumerMap 114 | .values(); 115 | 116 | final Configuration hbaseConfig = HBaseConfiguration.create(); 117 | 118 | hbaseConfig 119 | .set("hbase.zookeeper.quorum", 120 | "jetmaster2.jetnetname.artem.com,jetslave5.jetnetname.artem.com,jetslave1.jetnetname.artem.com"); 121 | hbaseConfig.set("hbase.zookeeper.property.clientPort", "2181"); 122 | hbaseConfig.set("zookeeper.znode.parent", "/hbase-unsecure"); 123 | 124 | final long start = System.currentTimeMillis(); 125 | for (List> streams : values) { 126 | // List> streams = 127 | // consumerMap.get(topic); 128 | System.out.println("streams size====" + streams.size()); 129 | 130 | // now launch all the threads 131 | executor = Executors.newFixedThreadPool(a_numThreads); 132 | 133 | executorConsumer = Executors.newFixedThreadPool(a_numThreads); 134 | 135 | // now create an object to consume the messages 136 | // 137 | int threadNumber = 0; 138 | for (final KafkaStream stream : streams) { 139 | 140 | // System.out.println("submiting stream:" + stream.clientId() + 141 | // " threadId= " + threadNumber + " , stream size" + 142 | // stream.size()); 143 | executor.submit(new ConsumerTest(stream, threadNumber, queue, 144 | new Notifier() { 145 | public synchronized void flushMsgs() { 146 | final List messages = new ArrayList<>(); 147 | queue.drainTo(messages); 148 | 149 | executorConsumer.submit(new Runnable(){ 150 | 151 | @Override 152 | public void run() { 153 | write(hbaseConfig, messages); 154 | long end = System.currentTimeMillis(); 155 | end = System.currentTimeMillis(); 156 | updateTotal( messages.size()); 157 | 158 | System.out.println(String.format("Flushing end: %d total= %d" , (end - start) / 1000, getTotal()) ); 159 | } 160 | 161 | }); 162 | 163 | } 164 | @Override 165 | public void shutdown() { 166 | long end = System.currentTimeMillis(); 167 | System.out.println(String.format("SHUTODOWN end: %d total= %d" , (end - start) / 1000, total) ); 168 | 169 | } 170 | })); 171 | threadNumber++; 172 | } 173 | 174 | } 175 | Set keySet = consumerMap.keySet(); 176 | for (String key : keySet) { 177 | System.out.println("key=" + key); 178 | } 179 | 180 | } 181 | 182 | private static ConsumerConfig createConsumerConfig(String a_zookeeper, 183 | String a_groupId) { 184 | Properties props = new Properties(); 185 | props.put("zookeeper.connect", a_zookeeper); 186 | props.put("group.id", a_groupId); 187 | props.put("zookeeper.session.timeout.ms", "400"); 188 | props.put("zookeeper.sync.time.ms", "200"); 189 | props.put("auto.commit.interval.ms", "1000"); 190 | 191 | return new ConsumerConfig(props); 192 | } 193 | 194 | public static void main(String[] args) { 195 | // String zooKeeper = args[0]; 196 | // String groupId = args[1]; 197 | String zooKeeper = "jetslave5.jetnetname.artem.com:2181,jetmaster1.jetnetname.artem.com:2181,jetmaster2.jetnetname.artem.com:2181"; 198 | String groupId = "StormSpoutConsumerGroup"; 199 | String topic = args[0]; 200 | int threads = Integer.parseInt(args[1]); 201 | 202 | // int threads = 2; 203 | 204 | ConsumerGroupExample example = new ConsumerGroupExample(zooKeeper, 205 | groupId, topic); 206 | LOG.info("Starting Consumer"); 207 | example.run(threads); 208 | 209 | // example.shutdown(); 210 | } 211 | 212 | /** 213 | * @return the map 214 | */ 215 | 216 | /** 217 | * @param aMap 218 | * the map to set 219 | */ 220 | public static void write(Configuration config, List messages) { 221 | 222 | TableName tableName = TableName.valueOf("api"); 223 | /** 224 | * a callback invoked when an asynchronous write fails. 225 | */ 226 | final BufferedMutator.ExceptionListener listener = new BufferedMutator.ExceptionListener() { 227 | @Override 228 | public void onException(RetriesExhaustedWithDetailsException e, 229 | BufferedMutator mutator) { 230 | for (int i = 0; i < e.getNumExceptions(); i++) { 231 | LOG.info("Failed to send put " + e.getRow(i) + "."); 232 | } 233 | } 234 | }; 235 | BufferedMutatorParams params = new BufferedMutatorParams(tableName) 236 | .listener(listener); 237 | try (Connection connection = ConnectionFactory.createConnection(config); 238 | final BufferedMutator mutator = connection 239 | .getBufferedMutator(params)) { 240 | List puts = new ArrayList<>(messages.size()); 241 | Put put; 242 | int count=0; 243 | for (Object[] message : messages) { 244 | 245 | String clicks_key = (String) message[0]; 246 | put = new Put(Bytes.toBytes(clicks_key)); 247 | put.addColumn(CF_DEFAULT, CF_FAMILY, Bytes.toBytes(clicks_key)); 248 | puts.add(put); 249 | count++; 250 | } 251 | mutator.mutate(puts); 252 | 253 | } catch (IOException ex) { 254 | LOG.log(Level.SEVERE, ex.getMessage()); 255 | } 256 | } 257 | 258 | } 259 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/ConsumerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 aervits. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.hortonworks; 17 | 18 | import java.io.IOException; 19 | import java.util.concurrent.LinkedBlockingQueue; 20 | import java.util.logging.Level; 21 | import java.util.logging.Logger; 22 | 23 | import kafka.consumer.ConsumerIterator; 24 | import kafka.consumer.KafkaStream; 25 | 26 | /** 27 | * 28 | * @author aervits 29 | */ 30 | public class ConsumerTest extends JsonUtils implements Runnable { 31 | 32 | private static final Logger LOG = Logger.getLogger(ConsumerTest.class.getName()); 33 | 34 | private KafkaStream m_stream; 35 | private int m_threadNumber; 36 | 37 | private Notifier notifier; 38 | private LinkedBlockingQueue m_queue; 39 | 40 | 41 | public ConsumerTest(KafkaStream a_stream, int a_threadNumber, LinkedBlockingQueue queue, Notifier notifier) { 42 | m_threadNumber = a_threadNumber; 43 | m_stream = a_stream; 44 | m_queue = queue; 45 | this.notifier =notifier; 46 | } 47 | 48 | @Override 49 | public void run() { 50 | ConsumerIterator it = m_stream.iterator(); 51 | int count =0; 52 | while (it.hasNext()){ 53 | byte[] message = it.next().message(); 54 | // System.out.println("Thread " + m_threadNumber + ": " + new String(message)); 55 | try { 56 | Object[] fields = createValues(new String(message)); 57 | m_queue.add( fields); 58 | } catch (IOException | ClassNotFoundException ex) { 59 | LOG.log(Level.SEVERE, null, ex); 60 | } 61 | 62 | if(count%10000==0){ 63 | notifier.flushMsgs(); 64 | } 65 | count++; 66 | } 67 | System.out.println("Shutting down Thread: " + m_threadNumber); 68 | notifier.shutdown(); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/Field.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by christoph on 8/24/15. 7 | */ 8 | public class Field implements Serializable { 9 | public String name; 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/JsonUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 aervits. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.hortonworks; 17 | 18 | import java.io.IOException; 19 | import java.util.logging.Level; 20 | import java.util.logging.Logger; 21 | import org.codehaus.jackson.JsonFactory; 22 | import org.codehaus.jackson.JsonNode; 23 | import org.codehaus.jackson.map.ObjectMapper; 24 | 25 | /** 26 | * 27 | * @author aervits 28 | */ 29 | public class JsonUtils { 30 | 31 | private static KafkaToHBaseConfig config; 32 | 33 | public JsonUtils() { 34 | try { 35 | // FOR TEST 36 | // config = Util.getKafkaToHiveConfig("./src/main/resources/kafka.json"); 37 | 38 | config = Util.getKafkaToHiveConfig("./src/main/resources/kafka.json"); 39 | } catch (IOException ex) { 40 | Logger.getLogger(ConsumerTest.class.getName()).log(Level.SEVERE, null, ex); 41 | } 42 | } 43 | 44 | public Object[] createValues(String jsonIn) throws IOException, ClassNotFoundException { 45 | JsonNode rootNode = getJsonNode(jsonIn); 46 | Field[] fields = config.mappingConfig.getCombinedFields(); 47 | Object[] values = new Object[fields.length]; 48 | int index = 0; 49 | for (Field f : fields) { 50 | String o = getValue(rootNode, f); 51 | values[index] = o; 52 | index++; 53 | } 54 | return values; 55 | } 56 | 57 | protected JsonNode getJsonNode(String jsonIn) throws IOException { 58 | JsonFactory factory = new JsonFactory(); 59 | ObjectMapper mapper = new ObjectMapper(factory); 60 | return mapper.readTree(jsonIn); 61 | } 62 | 63 | protected String getValue(JsonNode rootNode, Field f) throws ClassNotFoundException { 64 | JsonNode node = rootNode.get(f.name); 65 | if (node == null) { 66 | throw new RuntimeException(f.name + " cannot be found in json " + rootNode + " please revisit input json or configuration."); 67 | } 68 | if (node.isContainerNode()) { 69 | return node.toString(); 70 | } else { 71 | return node.asText(); 72 | } 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/KafkaToHBaseConfig.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks; 2 | 3 | import com.google.gson.Gson; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * Created by christoph on 8/24/15. 9 | */ 10 | public class KafkaToHBaseConfig implements Serializable{ 11 | 12 | public MappingConfig mappingConfig = new MappingConfig(); 13 | 14 | public String id = ""; 15 | 16 | @Override 17 | public String toString(){ 18 | return new Gson().toJson(this); 19 | } 20 | 21 | public static KafkaToHBaseConfig parse(String config){ 22 | return new Gson().fromJson(config, KafkaToHBaseConfig.class); 23 | } 24 | 25 | public String getId() { 26 | return id; 27 | } 28 | public void setId(String id) 29 | { 30 | this.id=id; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/MappingConfig.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import org.apache.commons.lang3.ArrayUtils; 8 | 9 | /** 10 | * Created by christoph on 8/24/15. 11 | */ 12 | public class MappingConfig implements Serializable { 13 | 14 | public Field[] fields = new Field[0]; 15 | public Field[] partitionFields = new Field[0]; 16 | 17 | public String[] dateFormats; 18 | 19 | 20 | public String[] getCombinedFieldNames() { 21 | String[] r = new String[fields.length + partitionFields.length]; 22 | List l = new ArrayList(); 23 | addFieldNames(fields, l); 24 | addFieldNames(partitionFields, l); 25 | return l.toArray(r); 26 | } 27 | 28 | public Field[] getCombinedFields() { 29 | return ArrayUtils.addAll(fields, partitionFields); 30 | } 31 | 32 | public String[] getFieldNames() { 33 | String[] r = new String[fields.length]; 34 | List l = new ArrayList(); 35 | addFieldNames(fields, l); 36 | return l.toArray(r); 37 | } 38 | 39 | public String[] getPartitionFieldNames() { 40 | String[] r = new String[partitionFields.length]; 41 | List l = new ArrayList(); 42 | addFieldNames(partitionFields, l); 43 | return l.toArray(r); 44 | // return Arrays.asList(fields) 45 | // .stream().map(f -> f.name) 46 | // .collect((Collectors.toList())) 47 | // .toArray(new String[fields.length]); 48 | } 49 | 50 | private void addFieldNames(Field[] fields, List l) { 51 | for(Field f : fields){ 52 | l.add(f.name); 53 | } 54 | } 55 | 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/Notifier.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks; 2 | 3 | public interface Notifier { 4 | public void flushMsgs(); 5 | public void shutdown(); 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/hortonworks/Util.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks; 2 | 3 | import java.io.File; 4 | 5 | import java.io.IOException; 6 | import java.util.List; 7 | import java.util.logging.Logger; 8 | import org.apache.commons.io.FileUtils; 9 | import org.apache.commons.lang.StringUtils; 10 | 11 | /** 12 | * Created by christoph on 8/26/15. 13 | */ 14 | public class Util { 15 | 16 | private static Logger LOG = Logger.getLogger(Util.class.getName()); 17 | 18 | public static KafkaToHBaseConfig getKafkaToHiveConfig(String fileName) throws IOException { 19 | 20 | List configCP = FileUtils.readLines(new File(fileName)); 21 | 22 | // List configCP = IOUtils.readLines(ClassLoader.getSystemResourceAsStream(fileName)); 23 | String configString = StringUtils.join(configCP, "\n"); 24 | LOG.info("using config '" + configString + "'"); 25 | 26 | return KafkaToHBaseConfig.parse(configString); 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /src/main/resources/kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "testId", 3 | "kafkaConfig": { 4 | "sourceTopic": "clicks_new10", 5 | "kafkaSpoutZkHosts": "jetslave5.jetnetname.artem.com:2181,jetmaster1.jetnetname.artem.com:2181,jetmaster2.jetnetname.artem.com:2181", 6 | "kafkaSpoutZkRoot": "/kafka_storm", 7 | "kafkaSpoutConsumerClientId": "StormSpoutClientId", 8 | "kafkaSpoutConsumerGroup": "StormSpoutConsumerGroup", 9 | "kafkaBrokerList": "jetmaster1.jetnetname.artem.com:6667,jetmaster2.jetnetname.artem.com:6667,jetslave4.jetnetname.artem.com:6667,jetslave5.jetnetname.artem.com:6667,jetslave6.jetnetname.artem.com:6667", 10 | "useStartOffsetTimeIfOffsetOutOfRange": true 11 | }, 12 | "mappingConfig": { 13 | "fields": [ 14 | { 15 | "name": "click_key" 16 | }, 17 | { 18 | "name": "client_id" 19 | }, 20 | { 21 | "name": "ordinal" 22 | }, 23 | { 24 | "name": "datetime" 25 | }, 26 | { 27 | "name": "user_agent" 28 | }, 29 | { 30 | "name": "url" 31 | }, 32 | { 33 | "name": "params" 34 | }, 35 | { 36 | "name": "tags" 37 | }, 38 | { 39 | "name": "dec" 40 | }, 41 | { 42 | "name": "json_example" 43 | }, 44 | { 45 | "name": "list_example" 46 | }, 47 | { 48 | "name": "map_example" 49 | }, 50 | { 51 | "name": "struct_example" 52 | }, 53 | { 54 | "name": "source_message" 55 | } 56 | ], 57 | "partitionFields": [ 58 | { 59 | "name": "partition_save_date" 60 | } 61 | ] 62 | } 63 | } -------------------------------------------------------------------------------- /src/test/java/com/hortonworks/JsonUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 aervits. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.hortonworks; 17 | 18 | import org.codehaus.jackson.JsonNode; 19 | import org.junit.After; 20 | import org.junit.AfterClass; 21 | import org.junit.Before; 22 | import org.junit.BeforeClass; 23 | import org.junit.Test; 24 | import static org.junit.Assert.*; 25 | import org.junit.Ignore; 26 | 27 | /** 28 | * 29 | * @author aervits 30 | */ 31 | public class JsonUtilsTest { 32 | 33 | public JsonUtilsTest() { 34 | } 35 | 36 | @BeforeClass 37 | public static void setUpClass() { 38 | } 39 | 40 | @AfterClass 41 | public static void tearDownClass() { 42 | } 43 | 44 | @Before 45 | public void setUp() { 46 | } 47 | 48 | @After 49 | public void tearDown() { 50 | } 51 | 52 | /** 53 | * Test of createValues method, of class JsonUtils. 54 | */ 55 | @Test 56 | public void testCreateValuesClick_key() throws Exception { 57 | System.out.println("createValues"); 58 | String jsonIn = "{\n" + 59 | " \"click_key\": \"1a4817d8-7412-47f0-88a6-3596e9a4e649\",\n" + 60 | " \"client_id\": \"-5534114214170524001\",\n" + 61 | " \"ordinal\": \"9\",\n" + 62 | " \"datetime\": \"Wed Oct 07 18:05:54 UTC 2015\",\n" + 63 | " \"user_agent\": \"mozilla\",\n" + 64 | " \"url\": \"http://jet.com/seach\",\n" + 65 | " \"params\": \"lskdjf,sdlfkjsldf\",\n" + 66 | " \"tags\": \"lskdjf,sdlfkjsldf\",\n" + 67 | " \"dec\": \"0.015939381932791097\",\n" + 68 | " \"json_example\": {\n" + 69 | " \"color\": \"Blue\",\n" + 70 | " \"sport\": \"Soccer\",\n" + 71 | " \"food\": \"Spaghetti\"\n" + 72 | " } ,\n" + 73 | " \"list_example\": \"list\",\n" + 74 | " \"map_example\": \"map\",\n" + 75 | " \"struct_example\": \"struct\",\n" + 76 | " \"source_message\": \"lskdjf,sdlfkjsldf\",\n" + 77 | " \"partition_save_date\": \"2015-10-07\"\n" + 78 | "}"; 79 | JsonUtils instance = new JsonUtils(); 80 | String expResult = "1a4817d8-7412-47f0-88a6-3596e9a4e649"; 81 | Object[] result = instance.createValues(jsonIn); 82 | assertEquals(expResult, result[0]); 83 | } 84 | 85 | /** 86 | * Test of createValues method, of class JsonUtils. 87 | */ 88 | @Test 89 | public void testCreateValuesClick_ID() throws Exception { 90 | System.out.println("createValues"); 91 | String jsonIn = "{\n" + 92 | " \"click_key\": \"1a4817d8-7412-47f0-88a6-3596e9a4e649\",\n" + 93 | " \"client_id\": \"-5534114214170524001\",\n" + 94 | " \"ordinal\": \"9\",\n" + 95 | " \"datetime\": \"Wed Oct 07 18:05:54 UTC 2015\",\n" + 96 | " \"user_agent\": \"mozilla\",\n" + 97 | " \"url\": \"http://jet.com/seach\",\n" + 98 | " \"params\": \"lskdjf,sdlfkjsldf\",\n" + 99 | " \"tags\": \"lskdjf,sdlfkjsldf\",\n" + 100 | " \"dec\": \"0.015939381932791097\",\n" + 101 | " \"json_example\": {\n" + 102 | " \"color\": \"Blue\",\n" + 103 | " \"sport\": \"Soccer\",\n" + 104 | " \"food\": \"Spaghetti\"\n" + 105 | " } ,\n" + 106 | " \"list_example\": \"list\",\n" + 107 | " \"map_example\": \"map\",\n" + 108 | " \"struct_example\": \"struct\",\n" + 109 | " \"source_message\": \"lskdjf,sdlfkjsldf\",\n" + 110 | " \"partition_save_date\": \"2015-10-07\"\n" + 111 | "}"; 112 | JsonUtils instance = new JsonUtils(); 113 | String expResult = "-5534114214170524001"; 114 | Object[] result = instance.createValues(jsonIn); 115 | assertEquals(expResult, result[1]); 116 | } 117 | 118 | /** 119 | * Test of getJsonNode method, of class JsonUtils. 120 | */ 121 | @Ignore 122 | public void testGetJsonNode() throws Exception { 123 | System.out.println("getJsonNode"); 124 | String jsonIn = ""; 125 | JsonUtils instance = new JsonUtils(); 126 | JsonNode expResult = null; 127 | JsonNode result = instance.getJsonNode(jsonIn); 128 | assertEquals(expResult, result); 129 | // TODO review the generated test code and remove the default call to fail. 130 | fail("The test case is a prototype."); 131 | } 132 | 133 | /** 134 | * Test of getValue method, of class JsonUtils. 135 | */ 136 | @Ignore 137 | public void testGetValue() throws Exception { 138 | System.out.println("getValue"); 139 | JsonNode rootNode = null; 140 | Field f = null; 141 | JsonUtils instance = new JsonUtils(); 142 | String expResult = ""; 143 | String result = instance.getValue(rootNode, f); 144 | assertEquals(expResult, result); 145 | // TODO review the generated test code and remove the default call to fail. 146 | fail("The test case is a prototype."); 147 | } 148 | 149 | } 150 | --------------------------------------------------------------------------------