├── .gitignore ├── src ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── storm │ │ └── hbase │ │ ├── trident │ │ ├── mapper │ │ │ ├── TridentHBaseMapper.java │ │ │ └── SimpleTridentHBaseMapper.java │ │ └── state │ │ │ ├── HBaseUpdater.java │ │ │ ├── HBaseStateFactory.java │ │ │ ├── HBaseQuery.java │ │ │ ├── HBaseState.java │ │ │ └── HBaseMapState.java │ │ ├── common │ │ ├── ICounter.java │ │ ├── IColumn.java │ │ ├── Utils.java │ │ ├── HBaseClient.java │ │ └── ColumnList.java │ │ ├── bolt │ │ ├── mapper │ │ │ ├── HBaseMapper.java │ │ │ ├── HBaseValueMapper.java │ │ │ ├── HBaseProjectionCriteria.java │ │ │ └── SimpleHBaseMapper.java │ │ ├── HBaseBolt.java │ │ ├── AbstractHBaseBolt.java │ │ └── HBaseLookupBolt.java │ │ └── security │ │ └── HBaseSecurityUtil.java └── test │ └── java │ └── org │ └── apache │ └── storm │ └── hbase │ ├── trident │ ├── PrintFunction.java │ └── WordCountTrident.java │ └── topology │ ├── WordCounter.java │ ├── WordCountClient.java │ ├── WordCountValueMapper.java │ ├── TotalWordCounter.java │ ├── WordSpout.java │ ├── PersistentWordCount.java │ └── LookupWordCount.java ├── pom.xml ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/trident/mapper/TridentHBaseMapper.java: -------------------------------------------------------------------------------- 1 | package org.apache.storm.hbase.trident.mapper; 2 | 3 | 4 | import backtype.storm.tuple.Tuple; 5 | import org.apache.storm.hbase.common.ColumnList; 6 | import storm.trident.tuple.TridentTuple; 7 | 8 | import java.io.Serializable; 9 | /** 10 | * Maps a storm.trident.tuple.TridentTuple object 11 | * to a row in an HBase table. 12 | */ 13 | public interface TridentHBaseMapper extends Serializable { 14 | 15 | 16 | /** 17 | * Given a tuple, return the HBase rowkey. 18 | * 19 | * @param tuple 20 | * @return 21 | */ 22 | byte[] rowKey(TridentTuple tuple); 23 | 24 | /** 25 | * Given a tuple, return a list of HBase columns to insert. 26 | * 27 | * @param tuple 28 | * @return 29 | */ 30 | ColumnList columns(TridentTuple tuple); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/common/ICounter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.common; 19 | 20 | /** 21 | * Interface definition for classes that support being written to HBase as 22 | * a counter column. 23 | * 24 | */ 25 | public interface ICounter { 26 | byte[] family(); 27 | byte[] qualifier(); 28 | long increment(); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/common/IColumn.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.common; 19 | 20 | /** 21 | * Interface definition for classes that support being written to HBase as 22 | * a regular column. 23 | * 24 | */ 25 | public interface IColumn { 26 | byte[] family(); 27 | byte[] qualifier(); 28 | byte[] value(); 29 | long timestamp(); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/trident/state/HBaseUpdater.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.trident.state; 19 | 20 | import storm.trident.operation.TridentCollector; 21 | import storm.trident.state.BaseStateUpdater; 22 | import storm.trident.tuple.TridentTuple; 23 | 24 | import java.util.List; 25 | 26 | public class HBaseUpdater extends BaseStateUpdater { 27 | 28 | @Override 29 | public void updateState(HBaseState hBaseState, List tuples, TridentCollector collector) { 30 | hBaseState.updateState(tuples, collector); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/trident/PrintFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.trident; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import storm.trident.operation.BaseFunction; 23 | import storm.trident.operation.TridentCollector; 24 | import storm.trident.tuple.TridentTuple; 25 | 26 | import java.util.Random; 27 | 28 | public class PrintFunction extends BaseFunction { 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(PrintFunction.class); 31 | 32 | private static final Random RANDOM = new Random(); 33 | 34 | @Override 35 | public void execute(TridentTuple tuple, TridentCollector tridentCollector) { 36 | if(RANDOM.nextInt(1000) > 995) { 37 | LOG.info(tuple.toString()); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.bolt.mapper; 19 | 20 | 21 | import backtype.storm.tuple.Tuple; 22 | import org.apache.storm.hbase.common.ColumnList; 23 | 24 | import java.io.Serializable; 25 | 26 | /** 27 | * Maps a backtype.storm.tuple.Tuple object 28 | * to a row in an HBase table. 29 | */ 30 | public interface HBaseMapper extends Serializable { 31 | 32 | /** 33 | * Given a tuple, return the HBase rowkey. 34 | * 35 | * @param tuple 36 | * @return 37 | */ 38 | byte[] rowKey(Tuple tuple); 39 | 40 | /** 41 | * Given a tuple, return a list of HBase columns to insert. 42 | * 43 | * @param tuple 44 | * @return 45 | */ 46 | ColumnList columns(Tuple tuple); 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/trident/state/HBaseStateFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.trident.state; 19 | 20 | import backtype.storm.task.IMetricsContext; 21 | import storm.trident.state.State; 22 | import storm.trident.state.StateFactory; 23 | 24 | import java.util.Map; 25 | 26 | public class HBaseStateFactory implements StateFactory { 27 | 28 | private HBaseState.Options options; 29 | 30 | public HBaseStateFactory(HBaseState.Options options) { 31 | this.options = options; 32 | } 33 | 34 | @Override 35 | public State makeState(Map map, IMetricsContext iMetricsContext, int partitionIndex, int numPartitions) { 36 | HBaseState state = new HBaseState(map , partitionIndex, numPartitions, options); 37 | state.prepare(); 38 | return state; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseValueMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.bolt.mapper; 19 | 20 | import backtype.storm.topology.OutputFieldsDeclarer; 21 | import backtype.storm.tuple.Values; 22 | import org.apache.hadoop.hbase.client.Result; 23 | 24 | import java.io.Serializable; 25 | import java.util.List; 26 | 27 | public interface HBaseValueMapper extends Serializable { 28 | /** 29 | * 30 | * @param result HBase lookup result instance. 31 | * @return list of values that should be emitted by the lookup bolt. 32 | * @throws Exception 33 | */ 34 | public List toValues(Result result) throws Exception; 35 | 36 | /** 37 | * declares the output fields for the lookup bolt. 38 | * @param declarer 39 | */ 40 | void declareOutputFields(OutputFieldsDeclarer declarer); 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/trident/state/HBaseQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.trident.state; 19 | 20 | import backtype.storm.tuple.Values; 21 | import storm.trident.operation.TridentCollector; 22 | import storm.trident.state.BaseQueryFunction; 23 | import storm.trident.tuple.TridentTuple; 24 | 25 | import java.util.List; 26 | 27 | public class HBaseQuery extends BaseQueryFunction> { 28 | 29 | @Override 30 | public List> batchRetrieve(HBaseState hBaseState, List tridentTuples) { 31 | return hBaseState.batchRetrieve(tridentTuples); 32 | } 33 | 34 | @Override 35 | public void execute(TridentTuple tuples, List values, TridentCollector tridentCollector) { 36 | for (Values value : values) { 37 | tridentCollector.emit(value); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/topology/WordCounter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.topology; 19 | 20 | import backtype.storm.task.TopologyContext; 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.IBasicBolt; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Tuple; 26 | 27 | import java.util.Map; 28 | 29 | import static backtype.storm.utils.Utils.tuple; 30 | 31 | public class WordCounter implements IBasicBolt { 32 | 33 | 34 | @SuppressWarnings("rawtypes") 35 | public void prepare(Map stormConf, TopologyContext context) { 36 | } 37 | 38 | /* 39 | * Just output the word value with a count of 1. 40 | * The HBaseBolt will handle incrementing the counter. 41 | */ 42 | public void execute(Tuple input, BasicOutputCollector collector) { 43 | collector.emit(tuple(input.getValues().get(0), 1)); 44 | } 45 | 46 | public void cleanup() { 47 | 48 | } 49 | 50 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 51 | declarer.declare(new Fields("word", "count")); 52 | } 53 | 54 | @Override 55 | public Map getComponentConfiguration() { 56 | return null; 57 | } 58 | 59 | } -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.security; 19 | 20 | import org.apache.hadoop.conf.Configuration; 21 | import org.apache.hadoop.hbase.security.UserProvider; 22 | import org.apache.hadoop.security.UserGroupInformation; 23 | 24 | import java.io.IOException; 25 | import java.net.InetAddress; 26 | import java.util.Map; 27 | 28 | /** 29 | * This class provides util methods for storm-hbase connector communicating 30 | * with secured HBase. 31 | */ 32 | public class HBaseSecurityUtil { 33 | public static final String STORM_KEYTAB_FILE_KEY = "storm.keytab.file"; 34 | public static final String STORM_USER_NAME_KEY = "storm.kerberos.principal"; 35 | 36 | public static UserProvider login(Map conf, Configuration hbaseConfig) throws IOException { 37 | UserProvider provider = UserProvider.instantiate(hbaseConfig); 38 | if (UserGroupInformation.isSecurityEnabled()) { 39 | String keytab = (String) conf.get(STORM_KEYTAB_FILE_KEY); 40 | if (keytab != null) { 41 | hbaseConfig.set(STORM_KEYTAB_FILE_KEY, keytab); 42 | } 43 | String userName = (String) conf.get(STORM_USER_NAME_KEY); 44 | if (userName != null) { 45 | hbaseConfig.set(STORM_USER_NAME_KEY, userName); 46 | } 47 | provider.login(STORM_KEYTAB_FILE_KEY, STORM_USER_NAME_KEY, 48 | InetAddress.getLocalHost().getCanonicalHostName()); 49 | } 50 | return provider; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/topology/WordCountClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.topology; 19 | 20 | import org.apache.hadoop.conf.Configuration; 21 | import org.apache.hadoop.hbase.HBaseConfiguration; 22 | import org.apache.hadoop.hbase.client.Get; 23 | import org.apache.hadoop.hbase.client.HTable; 24 | import org.apache.hadoop.hbase.client.Result; 25 | import org.apache.hadoop.hbase.util.Bytes; 26 | 27 | /** 28 | * Connects to the 'WordCount' table and prints counts for each word. 29 | * 30 | * Assumes you have run (or are running) PersistentWordCount 31 | */ 32 | public class WordCountClient { 33 | 34 | public static void main(String[] args) throws Exception { 35 | Configuration config = HBaseConfiguration.create(); 36 | if(args.length > 0){ 37 | config.set("hbase.rootdir", args[0]); 38 | } 39 | 40 | HTable table = new HTable(config, "WordCount"); 41 | 42 | 43 | for (String word : WordSpout.words) { 44 | Get get = new Get(Bytes.toBytes(word)); 45 | Result result = table.get(get); 46 | 47 | byte[] countBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("count")); 48 | byte[] wordBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("word")); 49 | 50 | String wordStr = Bytes.toString(wordBytes); 51 | System.out.println(wordStr); 52 | long count = Bytes.toLong(countBytes); 53 | System.out.println("Word: '" + wordStr + "', Count: " + count); 54 | } 55 | 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/topology/WordCountValueMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.topology; 19 | 20 | 21 | import backtype.storm.topology.OutputFieldsDeclarer; 22 | import backtype.storm.tuple.Fields; 23 | import backtype.storm.tuple.Values; 24 | import org.apache.hadoop.hbase.Cell; 25 | import org.apache.hadoop.hbase.CellUtil; 26 | import org.apache.hadoop.hbase.client.Result; 27 | import org.apache.hadoop.hbase.util.Bytes; 28 | import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper; 29 | 30 | import java.util.ArrayList; 31 | import java.util.List; 32 | 33 | /** 34 | * Takes a Hbase result and returns a value list that has a value instance for each column and corresponding value. 35 | * So if the result from Hbase was 36 | * 
37 |  * WORD, COUNT
38 |  * apple, 10
39 |  * bannana, 20
40 |  *
41 | * 42 | * this will return 43 | * 
44 |  *     [WORD, apple]
45 |  *     [COUNT, 10]
46 |  *     [WORD, banana]
47 |  *     [COUNT, 20]
48 |  *
49 | * 50 | */ 51 | public class WordCountValueMapper implements HBaseValueMapper { 52 | 53 | @Override 54 | public List toValues(Result result) throws Exception { 55 | List values = new ArrayList(); 56 | Cell[] cells = result.rawCells(); 57 | for(Cell cell : cells) { 58 | Values value = new Values (Bytes.toString(CellUtil.cloneQualifier(cell)), Bytes.toLong(CellUtil.cloneValue(cell))); 59 | values.add(value); 60 | } 61 | return values; 62 | } 63 | 64 | @Override 65 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 66 | declarer.declare(new Fields("columnName","columnValue")); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/common/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.common; 19 | 20 | import org.apache.hadoop.hbase.util.Bytes; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.math.BigDecimal; 25 | 26 | public class Utils { 27 | private static final Logger LOG = LoggerFactory.getLogger(Utils.class); 28 | 29 | private Utils(){} 30 | 31 | public static long toLong(Object obj){ 32 | long l = 0; 33 | if(obj != null){ 34 | if(obj instanceof Number){ 35 | l = ((Number)obj).longValue(); 36 | } else{ 37 | LOG.warn("Could not coerce {} to Long", obj.getClass().getName()); 38 | } 39 | } 40 | return l; 41 | } 42 | 43 | public static byte[] toBytes(Object obj){ 44 | if(obj instanceof String){ 45 | return ((String)obj).getBytes(); 46 | } else if (obj instanceof Integer){ 47 | return Bytes.toBytes((Integer) obj); 48 | } else if (obj instanceof Long){ 49 | return Bytes.toBytes((Long)obj); 50 | } else if (obj instanceof Short){ 51 | return Bytes.toBytes((Short)obj); 52 | 53 | } else if (obj instanceof Float){ 54 | return Bytes.toBytes((Float)obj); 55 | 56 | } else if (obj instanceof Double){ 57 | return Bytes.toBytes((Double)obj); 58 | 59 | } else if (obj instanceof Boolean){ 60 | return Bytes.toBytes((Boolean)obj); 61 | 62 | } else if (obj instanceof BigDecimal){ 63 | return Bytes.toBytes((BigDecimal)obj); 64 | } else { 65 | LOG.error("Can't convert class to byte array: " + obj.getClass().getName()); 66 | return new byte[0]; 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/topology/TotalWordCounter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.topology; 19 | 20 | import backtype.storm.task.TopologyContext; 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.IBasicBolt; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Tuple; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.math.BigInteger; 30 | import java.util.Map; 31 | import java.util.Random; 32 | 33 | import static backtype.storm.utils.Utils.tuple; 34 | 35 | public class TotalWordCounter implements IBasicBolt { 36 | 37 | private BigInteger total = BigInteger.ZERO; 38 | private static final Logger LOG = LoggerFactory.getLogger(TotalWordCounter.class); 39 | private static final Random RANDOM = new Random(); 40 | @SuppressWarnings("rawtypes") 41 | public void prepare(Map stormConf, TopologyContext context) { 42 | } 43 | 44 | /* 45 | * Just output the word value with a count of 1. 46 | * The HBaseBolt will handle incrementing the counter. 47 | */ 48 | public void execute(Tuple input, BasicOutputCollector collector) { 49 | total = total.add(new BigInteger(input.getValues().get(1).toString())); 50 | collector.emit(tuple(total.toString())); 51 | //prints the total with low probability. 52 | if(RANDOM.nextInt(1000) > 995) { 53 | LOG.info("Running total = " + total); 54 | } 55 | } 56 | 57 | public void cleanup() { 58 | LOG.info("Final total = " + total); 59 | } 60 | 61 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 62 | declarer.declare(new Fields("total")); 63 | } 64 | 65 | @Override 66 | public Map getComponentConfiguration() { 67 | return null; 68 | } 69 | 70 | } -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/bolt/HBaseBolt.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.bolt; 19 | 20 | import backtype.storm.topology.OutputFieldsDeclarer; 21 | import backtype.storm.tuple.Tuple; 22 | import org.apache.hadoop.hbase.client.Durability; 23 | import org.apache.hadoop.hbase.client.Mutation; 24 | import org.apache.storm.hbase.bolt.mapper.HBaseMapper; 25 | import org.apache.storm.hbase.common.ColumnList; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.util.List; 30 | 31 | /** 32 | * Basic bolt for writing to HBase. 33 | * 34 | * Note: Each HBaseBolt defined in a topology is tied to a specific table. 35 | * 36 | */ 37 | public class HBaseBolt extends AbstractHBaseBolt { 38 | private static final Logger LOG = LoggerFactory.getLogger(HBaseBolt.class); 39 | 40 | boolean writeToWAL = true; 41 | 42 | public HBaseBolt(String tableName, HBaseMapper mapper) { 43 | super(tableName, mapper); 44 | } 45 | 46 | public HBaseBolt writeToWAL(boolean writeToWAL) { 47 | this.writeToWAL = writeToWAL; 48 | return this; 49 | } 50 | 51 | public HBaseBolt withConfigKey(String configKey) { 52 | this.configKey = configKey; 53 | return this; 54 | } 55 | 56 | @Override 57 | public void execute(Tuple tuple) { 58 | byte[] rowKey = this.mapper.rowKey(tuple); 59 | ColumnList cols = this.mapper.columns(tuple); 60 | List mutations = hBaseClient.constructMutationReq(rowKey, cols, writeToWAL? Durability.SYNC_WAL : Durability.SKIP_WAL); 61 | 62 | try { 63 | this.hBaseClient.batchMutate(mutations); 64 | } catch(Exception e){ 65 | LOG.warn("Failing tuple. Error writing rowKey " + rowKey, e); 66 | this.collector.fail(tuple); 67 | return; 68 | } 69 | 70 | this.collector.ack(tuple); 71 | } 72 | 73 | @Override 74 | public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { 75 | 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/topology/WordSpout.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.topology; 19 | 20 | import backtype.storm.spout.SpoutOutputCollector; 21 | import backtype.storm.task.TopologyContext; 22 | import backtype.storm.topology.IRichSpout; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Values; 26 | 27 | import java.util.Map; 28 | import java.util.Random; 29 | import java.util.UUID; 30 | 31 | public class WordSpout implements IRichSpout { 32 | boolean isDistributed; 33 | SpoutOutputCollector collector; 34 | public static final String[] words = new String[] { "apple", "orange", "pineapple", "banana", "watermelon" }; 35 | 36 | public WordSpout() { 37 | this(true); 38 | } 39 | 40 | public WordSpout(boolean isDistributed) { 41 | this.isDistributed = isDistributed; 42 | } 43 | 44 | public boolean isDistributed() { 45 | return this.isDistributed; 46 | } 47 | 48 | @SuppressWarnings("rawtypes") 49 | public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { 50 | this.collector = collector; 51 | } 52 | 53 | public void close() { 54 | 55 | } 56 | 57 | public void nextTuple() { 58 | final Random rand = new Random(); 59 | final String word = words[rand.nextInt(words.length)]; 60 | this.collector.emit(new Values(word), UUID.randomUUID()); 61 | Thread.yield(); 62 | } 63 | 64 | public void ack(Object msgId) { 65 | 66 | } 67 | 68 | public void fail(Object msgId) { 69 | 70 | } 71 | 72 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 73 | declarer.declare(new Fields("word")); 74 | } 75 | 76 | @Override 77 | public void activate() { 78 | } 79 | 80 | @Override 81 | public void deactivate() { 82 | } 83 | 84 | @Override 85 | public Map getComponentConfiguration() { 86 | return null; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseProjectionCriteria.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.bolt.mapper; 19 | 20 | import com.google.common.collect.Lists; 21 | 22 | import java.io.Serializable; 23 | import java.util.List; 24 | 25 | /** 26 | * Allows the user to specify the projection criteria. 27 | * If only columnFamily is specified all columns from that family will be returned. 28 | * If a column is specified only that column from that family will be returned. 29 | 30 | */ 31 | public class HBaseProjectionCriteria implements Serializable { 32 | private List columnFamilies; 33 | private List columns; 34 | 35 | public static class ColumnMetaData implements Serializable { 36 | private byte[] columnFamily; 37 | private byte[] qualifier; 38 | 39 | public ColumnMetaData(String columnFamily, String qualifier) { 40 | this.columnFamily = columnFamily.getBytes(); 41 | this.qualifier = qualifier.getBytes(); 42 | } 43 | 44 | public byte[] getColumnFamily() { 45 | return columnFamily; 46 | } 47 | 48 | public byte[] getQualifier() { 49 | return qualifier; 50 | } 51 | } 52 | 53 | public HBaseProjectionCriteria() { 54 | columnFamilies = Lists.newArrayList(); 55 | columns = Lists.newArrayList(); 56 | } 57 | 58 | /** 59 | * all columns from this family will be included as result of HBase lookup. 60 | * @param columnFamily 61 | * @return 62 | */ 63 | public HBaseProjectionCriteria addColumnFamily(String columnFamily) { 64 | this.columnFamilies.add(columnFamily.getBytes()); 65 | return this; 66 | } 67 | 68 | /** 69 | * Only this column from the the columnFamily will be included as result of HBase lookup. 70 | * @param column 71 | * @return 72 | */ 73 | public HBaseProjectionCriteria addColumn(ColumnMetaData column) { 74 | this.columns.add(column); 75 | return this; 76 | } 77 | 78 | public List getColumns() { 79 | return columns; 80 | } 81 | 82 | public List getColumnFamilies() { 83 | return columnFamilies; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/bolt/AbstractHBaseBolt.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.bolt; 19 | 20 | import backtype.storm.task.OutputCollector; 21 | import backtype.storm.task.TopologyContext; 22 | import backtype.storm.topology.base.BaseRichBolt; 23 | import org.apache.commons.lang.Validate; 24 | import org.apache.hadoop.conf.Configuration; 25 | import org.apache.hadoop.hbase.HBaseConfiguration; 26 | import org.apache.storm.hbase.bolt.mapper.HBaseMapper; 27 | import org.apache.storm.hbase.common.HBaseClient; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.util.Map; 32 | 33 | // TODO support more configuration options, for now we're defaulting to the hbase-*.xml files found on the classpath 34 | public abstract class AbstractHBaseBolt extends BaseRichBolt { 35 | private static final Logger LOG = LoggerFactory.getLogger(AbstractHBaseBolt.class); 36 | 37 | protected OutputCollector collector; 38 | 39 | protected transient HBaseClient hBaseClient; 40 | protected String tableName; 41 | protected HBaseMapper mapper; 42 | protected String configKey; 43 | 44 | public AbstractHBaseBolt(String tableName, HBaseMapper mapper) { 45 | Validate.notEmpty(tableName, "Table name can not be blank or null"); 46 | Validate.notNull(mapper, "mapper can not be null"); 47 | this.tableName = tableName; 48 | this.mapper = mapper; 49 | } 50 | 51 | @Override 52 | public void prepare(Map map, TopologyContext topologyContext, OutputCollector collector) { 53 | this.collector = collector; 54 | final Configuration hbConfig = HBaseConfiguration.create(); 55 | 56 | Map conf = (Map)map.get(this.configKey); 57 | if(conf == null) { 58 | throw new IllegalArgumentException("HBase configuration not found using key '" + this.configKey + "'"); 59 | } 60 | if(conf.get("hbase.rootdir") == null) { 61 | LOG.warn("No 'hbase.rootdir' value found in configuration! Using HBase defaults."); 62 | } 63 | for(String key : conf.keySet()) { 64 | hbConfig.set(key, String.valueOf(conf.get(key))); 65 | } 66 | 67 | this.hBaseClient = new HBaseClient(conf, hbConfig, tableName); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/bolt/mapper/SimpleHBaseMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.bolt.mapper; 19 | 20 | import backtype.storm.tuple.Fields; 21 | import backtype.storm.tuple.Tuple; 22 | import org.apache.storm.hbase.common.ColumnList; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | import static org.apache.storm.hbase.common.Utils.*; 26 | 27 | /** 28 | * 29 | */ 30 | public class SimpleHBaseMapper implements HBaseMapper { 31 | private static final Logger LOG = LoggerFactory.getLogger(SimpleHBaseMapper.class); 32 | 33 | private String rowKeyField; 34 | // private String timestampField; 35 | private byte[] columnFamily; 36 | private Fields columnFields; 37 | private Fields counterFields; 38 | 39 | public SimpleHBaseMapper(){ 40 | } 41 | 42 | 43 | public SimpleHBaseMapper withRowKeyField(String rowKeyField){ 44 | this.rowKeyField = rowKeyField; 45 | return this; 46 | } 47 | 48 | public SimpleHBaseMapper withColumnFields(Fields columnFields){ 49 | this.columnFields = columnFields; 50 | return this; 51 | } 52 | 53 | public SimpleHBaseMapper withCounterFields(Fields counterFields){ 54 | this.counterFields = counterFields; 55 | return this; 56 | } 57 | 58 | public SimpleHBaseMapper withColumnFamily(String columnFamily){ 59 | this.columnFamily = columnFamily.getBytes(); 60 | return this; 61 | } 62 | 63 | // public SimpleTridentHBaseMapper withTimestampField(String timestampField){ 64 | // this.timestampField = timestampField; 65 | // return this; 66 | // } 67 | 68 | @Override 69 | public byte[] rowKey(Tuple tuple) { 70 | Object objVal = tuple.getValueByField(this.rowKeyField); 71 | return toBytes(objVal); 72 | } 73 | 74 | @Override 75 | public ColumnList columns(Tuple tuple) { 76 | ColumnList cols = new ColumnList(); 77 | if(this.columnFields != null){ 78 | // TODO timestamps 79 | for(String field : this.columnFields){ 80 | cols.addColumn(this.columnFamily, field.getBytes(), toBytes(tuple.getValueByField(field))); 81 | } 82 | } 83 | if(this.counterFields != null){ 84 | for(String field : this.counterFields){ 85 | cols.addCounter(this.columnFamily, field.getBytes(), toLong(tuple.getValueByField(field))); 86 | } 87 | } 88 | return cols; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/topology/PersistentWordCount.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.topology; 19 | 20 | import backtype.storm.Config; 21 | import backtype.storm.LocalCluster; 22 | import backtype.storm.StormSubmitter; 23 | import backtype.storm.topology.TopologyBuilder; 24 | import backtype.storm.tuple.Fields; 25 | import org.apache.storm.hbase.bolt.HBaseBolt; 26 | import org.apache.storm.hbase.bolt.mapper.SimpleHBaseMapper; 27 | 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | 31 | 32 | public class PersistentWordCount { 33 | private static final String WORD_SPOUT = "WORD_SPOUT"; 34 | private static final String COUNT_BOLT = "COUNT_BOLT"; 35 | private static final String HBASE_BOLT = "HBASE_BOLT"; 36 | 37 | 38 | public static void main(String[] args) throws Exception { 39 | Config config = new Config(); 40 | 41 | Map hbConf = new HashMap(); 42 | if(args.length > 0){ 43 | hbConf.put("hbase.rootdir", args[0]); 44 | } 45 | config.put("hbase.conf", hbConf); 46 | 47 | WordSpout spout = new WordSpout(); 48 | WordCounter bolt = new WordCounter(); 49 | 50 | SimpleHBaseMapper mapper = new SimpleHBaseMapper() 51 | .withRowKeyField("word") 52 | .withColumnFields(new Fields("word")) 53 | .withCounterFields(new Fields("count")) 54 | .withColumnFamily("cf"); 55 | 56 | HBaseBolt hbase = new HBaseBolt("WordCount", mapper) 57 | .withConfigKey("hbase.conf"); 58 | 59 | 60 | // wordSpout ==> countBolt ==> HBaseBolt 61 | TopologyBuilder builder = new TopologyBuilder(); 62 | 63 | builder.setSpout(WORD_SPOUT, spout, 1); 64 | builder.setBolt(COUNT_BOLT, bolt, 1).shuffleGrouping(WORD_SPOUT); 65 | builder.setBolt(HBASE_BOLT, hbase, 1).fieldsGrouping(COUNT_BOLT, new Fields("word")); 66 | 67 | 68 | if (args.length == 1) { 69 | LocalCluster cluster = new LocalCluster(); 70 | cluster.submitTopology("test", config, builder.createTopology()); 71 | Thread.sleep(30000); 72 | cluster.killTopology("test"); 73 | cluster.shutdown(); 74 | System.exit(0); 75 | } else if (args.length == 2) { 76 | StormSubmitter.submitTopology(args[1], config, builder.createTopology()); 77 | } else{ 78 | System.out.println("Usage: HdfsFileTopology [topology name]"); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.trident.mapper; 19 | 20 | import backtype.storm.tuple.Fields; 21 | import backtype.storm.tuple.Tuple; 22 | import org.apache.storm.hbase.bolt.mapper.HBaseMapper; 23 | import org.apache.storm.hbase.common.ColumnList; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | import storm.trident.tuple.TridentTuple; 27 | 28 | import static org.apache.storm.hbase.common.Utils.toBytes; 29 | import static org.apache.storm.hbase.common.Utils.toLong; 30 | 31 | /** 32 | * 33 | */ 34 | public class SimpleTridentHBaseMapper implements TridentHBaseMapper { 35 | private static final Logger LOG = LoggerFactory.getLogger(SimpleTridentHBaseMapper.class); 36 | 37 | private String rowKeyField; 38 | private byte[] columnFamily; 39 | private Fields columnFields; 40 | private Fields counterFields; 41 | 42 | public SimpleTridentHBaseMapper(){ 43 | } 44 | 45 | 46 | public SimpleTridentHBaseMapper withRowKeyField(String rowKeyField){ 47 | this.rowKeyField = rowKeyField; 48 | return this; 49 | } 50 | 51 | public SimpleTridentHBaseMapper withColumnFields(Fields columnFields){ 52 | this.columnFields = columnFields; 53 | return this; 54 | } 55 | 56 | public SimpleTridentHBaseMapper withCounterFields(Fields counterFields){ 57 | this.counterFields = counterFields; 58 | return this; 59 | } 60 | 61 | public SimpleTridentHBaseMapper withColumnFamily(String columnFamily){ 62 | this.columnFamily = columnFamily.getBytes(); 63 | return this; 64 | } 65 | 66 | 67 | @Override 68 | public byte[] rowKey(TridentTuple tuple) { 69 | Object objVal = tuple.getValueByField(this.rowKeyField); 70 | return toBytes(objVal); 71 | } 72 | 73 | @Override 74 | public ColumnList columns(TridentTuple tuple) { 75 | ColumnList cols = new ColumnList(); 76 | if(this.columnFields != null){ 77 | // TODO timestamps 78 | for(String field : this.columnFields){ 79 | cols.addColumn(this.columnFamily, field.getBytes(), toBytes(tuple.getValueByField(field))); 80 | } 81 | } 82 | if(this.counterFields != null){ 83 | for(String field : this.counterFields){ 84 | cols.addCounter(this.columnFamily, field.getBytes(), toLong(tuple.getValueByField(field))); 85 | } 86 | } 87 | return cols; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/bolt/HBaseLookupBolt.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.bolt; 19 | 20 | import backtype.storm.topology.OutputFieldsDeclarer; 21 | import backtype.storm.tuple.Tuple; 22 | import backtype.storm.tuple.Values; 23 | import com.google.common.collect.Lists; 24 | import org.apache.commons.lang.Validate; 25 | import org.apache.hadoop.hbase.client.Get; 26 | import org.apache.hadoop.hbase.client.Result; 27 | import org.apache.storm.hbase.bolt.mapper.HBaseMapper; 28 | import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria; 29 | import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | /** 34 | * Basic bolt for querying from HBase. 35 | * 36 | * Note: Each HBaseBolt defined in a topology is tied to a specific table. 37 | * 38 | */ 39 | public class HBaseLookupBolt extends AbstractHBaseBolt { 40 | private static final Logger LOG = LoggerFactory.getLogger(HBaseLookupBolt.class); 41 | 42 | private HBaseValueMapper rowToTupleMapper; 43 | 44 | private HBaseProjectionCriteria projectionCriteria; 45 | 46 | public HBaseLookupBolt(String tableName, HBaseMapper mapper, HBaseValueMapper rowToTupleMapper){ 47 | super(tableName, mapper); 48 | Validate.notNull(rowToTupleMapper, "rowToTupleMapper can not be null"); 49 | this.rowToTupleMapper = rowToTupleMapper; 50 | } 51 | 52 | public HBaseLookupBolt withConfigKey(String configKey){ 53 | this.configKey = configKey; 54 | return this; 55 | } 56 | 57 | public HBaseLookupBolt withProjectionCriteria(HBaseProjectionCriteria projectionCriteria) { 58 | this.projectionCriteria = projectionCriteria; 59 | return this; 60 | } 61 | 62 | @Override 63 | public void execute(Tuple tuple) { 64 | byte[] rowKey = this.mapper.rowKey(tuple); 65 | Get get = hBaseClient.constructGetRequests(rowKey, projectionCriteria); 66 | 67 | try { 68 | Result result = hBaseClient.batchGet(Lists.newArrayList(get))[0]; 69 | for(Values values : rowToTupleMapper.toValues(result)) { 70 | this.collector.emit(values); 71 | } 72 | this.collector.ack(tuple); 73 | } catch (Exception e) { 74 | LOG.warn("Could not perform Lookup for rowKey =" + rowKey + " from Hbase.", e); 75 | this.collector.fail(tuple); 76 | } 77 | } 78 | 79 | @Override 80 | public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { 81 | rowToTupleMapper.declareOutputFields(outputFieldsDeclarer); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 4.0.0 20 | 21 | 22 | org.sonatype.oss 23 | oss-parent 24 | 7 25 | 26 | 27 | com.github.ptgoetz 28 | storm-hbase 29 | 0.1.3-SNAPSHOT 30 | 31 | 32 | 33 | The Apache Software License, Version 2.0 34 | http://www.apache.org/licenses/LICENSE-2.0.txt 35 | 36 | 37 | 38 | scm:git:git@github.com:ptgoetz/storm-hbase.git 39 | scm:git:git@github.com:ptgoetz/storm-hbase.git 40 | :git@github.com:ptgoetz/storm-hbase.git 41 | 42 | 43 | 44 | 45 | ptgoetz 46 | P. Taylor Goetz 47 | ptgoetz@gmail.com 48 | 49 | 50 | 51 | 52 | 0.98.1-hadoop2 53 | 54 | 55 | 56 | 57 | org.apache.storm 58 | storm-core 59 | 0.9.2-incubating-SNAPSHOT 60 | provided 61 | 62 | 63 | org.apache.hbase 64 | hbase-client 65 | ${hbase.version} 66 | 67 | 68 | org.slf4j 69 | slf4j-log4j12 70 | 71 | 72 | org.apache.zookeeper 73 | zookeeper 74 | 75 | 76 | 77 | 78 | org.apache.hadoop 79 | hadoop-hdfs 80 | 2.2.0 81 | 82 | 83 | org.slf4j 84 | slf4j-log4j12 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/topology/LookupWordCount.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.topology; 19 | 20 | import backtype.storm.Config; 21 | import backtype.storm.LocalCluster; 22 | import backtype.storm.StormSubmitter; 23 | import backtype.storm.topology.TopologyBuilder; 24 | import backtype.storm.tuple.Fields; 25 | import org.apache.storm.hbase.bolt.HBaseLookupBolt; 26 | import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria; 27 | import org.apache.storm.hbase.bolt.mapper.SimpleHBaseMapper; 28 | 29 | import java.util.HashMap; 30 | import java.util.Map; 31 | 32 | 33 | public class LookupWordCount { 34 | private static final String WORD_SPOUT = "WORD_SPOUT"; 35 | private static final String LOOKUP_BOLT = "LOOKUP_BOLT"; 36 | private static final String TOTAL_COUNT_BOLT = "TOTAL_COUNT_BOLT"; 37 | 38 | public static void main(String[] args) throws Exception { 39 | Config config = new Config(); 40 | 41 | Map hbConf = new HashMap(); 42 | if(args.length > 0){ 43 | hbConf.put("hbase.rootdir", args[0]); 44 | } 45 | config.put("hbase.conf", hbConf); 46 | 47 | WordSpout spout = new WordSpout(); 48 | TotalWordCounter totalBolt = new TotalWordCounter(); 49 | 50 | SimpleHBaseMapper mapper = new SimpleHBaseMapper().withRowKeyField("word"); 51 | HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria(); 52 | projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count")); 53 | 54 | WordCountValueMapper rowToTupleMapper = new WordCountValueMapper(); 55 | 56 | HBaseLookupBolt hBaseLookupBolt = new HBaseLookupBolt("WordCount", mapper, rowToTupleMapper) 57 | .withConfigKey("hbase.conf") 58 | .withProjectionCriteria(projectionCriteria); 59 | 60 | //wordspout -> lookupbolt -> totalCountBolt 61 | TopologyBuilder builder = new TopologyBuilder(); 62 | builder.setSpout(WORD_SPOUT, spout, 1); 63 | builder.setBolt(LOOKUP_BOLT, hBaseLookupBolt, 1).shuffleGrouping(WORD_SPOUT); 64 | builder.setBolt(TOTAL_COUNT_BOLT, totalBolt, 1).fieldsGrouping(LOOKUP_BOLT, new Fields("columnName")); 65 | 66 | if (args.length == 1) { 67 | LocalCluster cluster = new LocalCluster(); 68 | cluster.submitTopology("test", config, builder.createTopology()); 69 | Thread.sleep(30000); 70 | cluster.killTopology("test"); 71 | cluster.shutdown(); 72 | System.exit(0); 73 | } else if (args.length == 2) { 74 | StormSubmitter.submitTopology(args[1], config, builder.createTopology()); 75 | } else{ 76 | System.out.println("Usage: LookupWordCount "); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/test/java/org/apache/storm/hbase/trident/WordCountTrident.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.trident; 19 | 20 | import backtype.storm.Config; 21 | import backtype.storm.LocalCluster; 22 | import backtype.storm.StormSubmitter; 23 | import backtype.storm.generated.StormTopology; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Values; 26 | import org.apache.hadoop.hbase.client.Durability; 27 | import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria; 28 | import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper; 29 | import org.apache.storm.hbase.topology.WordCountValueMapper; 30 | import org.apache.storm.hbase.trident.mapper.SimpleTridentHBaseMapper; 31 | import org.apache.storm.hbase.trident.mapper.TridentHBaseMapper; 32 | import org.apache.storm.hbase.trident.state.HBaseQuery; 33 | import org.apache.storm.hbase.trident.state.HBaseState; 34 | import org.apache.storm.hbase.trident.state.HBaseStateFactory; 35 | import org.apache.storm.hbase.trident.state.HBaseUpdater; 36 | import storm.trident.Stream; 37 | import storm.trident.TridentState; 38 | import storm.trident.TridentTopology; 39 | import storm.trident.state.StateFactory; 40 | import storm.trident.testing.FixedBatchSpout; 41 | 42 | public class WordCountTrident { 43 | public static StormTopology buildTopology(String hbaseRoot){ 44 | Fields fields = new Fields("word", "count"); 45 | FixedBatchSpout spout = new FixedBatchSpout(fields, 4, 46 | new Values("storm", 1), 47 | new Values("trident", 1), 48 | new Values("needs", 1), 49 | new Values("javadoc", 1) 50 | ); 51 | spout.setCycle(true); 52 | 53 | TridentHBaseMapper tridentHBaseMapper = new SimpleTridentHBaseMapper() 54 | .withColumnFamily("cf") 55 | .withColumnFields(new Fields("word")) 56 | .withCounterFields(new Fields("count")) 57 | .withRowKeyField("word"); 58 | 59 | HBaseValueMapper rowToStormValueMapper = new WordCountValueMapper(); 60 | 61 | HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria(); 62 | projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count")); 63 | 64 | HBaseState.Options options = new HBaseState.Options() 65 | .withConfigKey(hbaseRoot) 66 | .withDurability(Durability.SYNC_WAL) 67 | .withMapper(tridentHBaseMapper) 68 | .withProjectionCriteria(projectionCriteria) 69 | .withRowToStormValueMapper(rowToStormValueMapper) 70 | .withTableName("WordCount"); 71 | 72 | StateFactory factory = new HBaseStateFactory(options); 73 | 74 | TridentTopology topology = new TridentTopology(); 75 | Stream stream = topology.newStream("spout1", spout); 76 | 77 | stream.partitionPersist(factory, fields, new HBaseUpdater(), new Fields()); 78 | 79 | TridentState state = topology.newStaticState(factory); 80 | stream = stream.stateQuery(state, new Fields("word"), new HBaseQuery(), new Fields("columnName","columnValue")); 81 | stream.each(new Fields("word","columnValue"), new PrintFunction(), new Fields()); 82 | return topology.build(); 83 | } 84 | 85 | public static void main(String[] args) throws Exception { 86 | Config conf = new Config(); 87 | conf.setMaxSpoutPending(5); 88 | if (args.length == 1) { 89 | LocalCluster cluster = new LocalCluster(); 90 | cluster.submitTopology("wordCounter", conf, buildTopology(args[0])); 91 | Thread.sleep(60 * 1000); 92 | cluster.killTopology("wordCounter"); 93 | cluster.shutdown(); 94 | System.exit(0); 95 | } 96 | else if(args.length == 2) { 97 | conf.setNumWorkers(3); 98 | StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0])); 99 | } else{ 100 | System.out.println("Usage: TridentFileTopology [topology name]"); 101 | } 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/common/HBaseClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.common; 19 | 20 | import com.google.common.collect.Lists; 21 | import org.apache.hadoop.conf.Configuration; 22 | import org.apache.hadoop.hbase.client.*; 23 | import org.apache.hadoop.hbase.security.UserProvider; 24 | import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria; 25 | import org.apache.storm.hbase.security.HBaseSecurityUtil; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.io.IOException; 30 | import java.security.PrivilegedExceptionAction; 31 | import java.util.List; 32 | import java.util.Map; 33 | 34 | public class HBaseClient { 35 | private static final Logger LOG = LoggerFactory.getLogger(HBaseClient.class); 36 | 37 | private HTable table; 38 | 39 | public HBaseClient(Map map , final Configuration configuration, final String tableName) { 40 | try { 41 | UserProvider provider = HBaseSecurityUtil.login(map, configuration); 42 | this.table = provider.getCurrent().getUGI().doAs(new PrivilegedExceptionAction() { 43 | @Override 44 | public HTable run() throws IOException { 45 | return new HTable(configuration, tableName); 46 | } 47 | }); 48 | } catch(Exception e) { 49 | throw new RuntimeException("HBase bolt preparation failed: " + e.getMessage(), e); 50 | } 51 | } 52 | 53 | public List constructMutationReq(byte[] rowKey, ColumnList cols, Durability durability) { 54 | List mutations = Lists.newArrayList(); 55 | 56 | if (cols.hasColumns()) { 57 | Put put = new Put(rowKey); 58 | put.setDurability(durability); 59 | for (ColumnList.Column col : cols.getColumns()) { 60 | if (col.getTs() > 0) { 61 | put.add( 62 | col.getFamily(), 63 | col.getQualifier(), 64 | col.getTs(), 65 | col.getValue() 66 | ); 67 | } else { 68 | put.add( 69 | col.getFamily(), 70 | col.getQualifier(), 71 | col.getValue() 72 | ); 73 | } 74 | } 75 | mutations.add(put); 76 | } 77 | 78 | if (cols.hasCounters()) { 79 | Increment inc = new Increment(rowKey); 80 | inc.setDurability(durability); 81 | for (ColumnList.Counter cnt : cols.getCounters()) { 82 | inc.addColumn( 83 | cnt.getFamily(), 84 | cnt.getQualifier(), 85 | cnt.getIncrement() 86 | ); 87 | } 88 | mutations.add(inc); 89 | } 90 | 91 | if (mutations.isEmpty()) { 92 | mutations.add(new Put(rowKey)); 93 | } 94 | return mutations; 95 | } 96 | 97 | public void batchMutate(List mutations) throws Exception { 98 | Object[] result = new Object[mutations.size()]; 99 | try { 100 | table.batch(mutations, result); 101 | } catch (InterruptedException e) { 102 | LOG.warn("Error performing a mutation to HBase.", e); 103 | throw e; 104 | } catch (IOException e) { 105 | LOG.warn("Error performing a mutation to HBase.", e); 106 | throw e; 107 | } 108 | } 109 | 110 | 111 | public Get constructGetRequests(byte[] rowKey, HBaseProjectionCriteria projectionCriteria) { 112 | Get get = new Get(rowKey); 113 | 114 | if (projectionCriteria != null) { 115 | for (byte[] columnFamily : projectionCriteria.getColumnFamilies()) { 116 | get.addFamily(columnFamily); 117 | } 118 | 119 | for (HBaseProjectionCriteria.ColumnMetaData columnMetaData : projectionCriteria.getColumns()) { 120 | get.addColumn(columnMetaData.getColumnFamily(), columnMetaData.getQualifier()); 121 | } 122 | } 123 | 124 | return get; 125 | } 126 | 127 | public Result[] batchGet(List gets) throws Exception { 128 | try { 129 | return table.get(gets); 130 | } catch (Exception e) { 131 | LOG.warn("Could not perform HBASE lookup.", e); 132 | throw e; 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/common/ColumnList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.common; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | /** 24 | * Represents a list of HBase columns. 25 | * 26 | * There are two types of columns, standard and counter. 27 | * 28 | * Standard columns have column family (required), qualifier (optional), 29 | * timestamp (optional), and a value (optional) values. 30 | * 31 | * Counter columns have column family (required), qualifier (optional), 32 | * and an increment (optional, but recommended) values. 33 | * 34 | * Inserts/Updates can be added via the addColumn() and addCounter() 35 | * methods. 36 | * 37 | */ 38 | public class ColumnList { 39 | 40 | public static abstract class AbstractColumn { 41 | byte[] family, qualifier; 42 | 43 | AbstractColumn(byte[] family, byte[] qualifier){ 44 | this.family = family; 45 | this.qualifier = qualifier; 46 | } 47 | 48 | public byte[] getFamily() { 49 | return family; 50 | } 51 | 52 | public byte[] getQualifier() { 53 | return qualifier; 54 | } 55 | 56 | } 57 | 58 | public static class Column extends AbstractColumn { 59 | byte[] value; 60 | long ts = -1; 61 | Column(byte[] family, byte[] qualifier, long ts, byte[] value){ 62 | super(family, qualifier); 63 | this.value = value; 64 | this.ts = ts; 65 | } 66 | 67 | public byte[] getValue() { 68 | return value; 69 | } 70 | 71 | public long getTs() { 72 | return ts; 73 | } 74 | } 75 | 76 | public static class Counter extends AbstractColumn { 77 | long incr = 0; 78 | Counter(byte[] family, byte[] qualifier, long incr){ 79 | super(family, qualifier); 80 | this.incr = incr; 81 | } 82 | 83 | public long getIncrement() { 84 | return incr; 85 | } 86 | } 87 | 88 | 89 | private ArrayList columns; 90 | private ArrayList counters; 91 | 92 | 93 | private ArrayList columns(){ 94 | if(this.columns == null){ 95 | this.columns = new ArrayList(); 96 | } 97 | return this.columns; 98 | } 99 | 100 | private ArrayList counters(){ 101 | if(this.counters == null){ 102 | this.counters = new ArrayList(); 103 | } 104 | return this.counters; 105 | } 106 | 107 | /** 108 | * Add a standard HBase column. 109 | * 110 | * @param family 111 | * @param qualifier 112 | * @param ts 113 | * @param value 114 | * @return 115 | */ 116 | public ColumnList addColumn(byte[] family, byte[] qualifier, long ts, byte[] value){ 117 | columns().add(new Column(family, qualifier, ts, value)); 118 | return this; 119 | } 120 | 121 | /** 122 | * Add a standard HBase column 123 | * @param family 124 | * @param qualifier 125 | * @param value 126 | * @return 127 | */ 128 | public ColumnList addColumn(byte[] family, byte[] qualifier, byte[] value){ 129 | columns().add(new Column(family, qualifier, -1, value)); 130 | return this; 131 | } 132 | 133 | /** 134 | * Add a standard HBase column given an instance of a class that implements 135 | * the IColumn interface. 136 | * @param column 137 | * @return 138 | */ 139 | public ColumnList addColumn(IColumn column){ 140 | return this.addColumn(column.family(), column.qualifier(), column.timestamp(), column.value()); 141 | } 142 | 143 | /** 144 | * Add an HBase counter column. 145 | * 146 | * @param family 147 | * @param qualifier 148 | * @param incr 149 | * @return 150 | */ 151 | public ColumnList addCounter(byte[] family, byte[] qualifier, long incr){ 152 | counters().add(new Counter(family, qualifier, incr)); 153 | return this; 154 | } 155 | 156 | /** 157 | * Add an HBase counter column given an instance of a class that implements the 158 | * ICounter interface. 159 | * @param counter 160 | * @return 161 | */ 162 | public ColumnList addCounter(ICounter counter){ 163 | return this.addCounter(counter.family(), counter.qualifier(), counter.increment()); 164 | } 165 | 166 | 167 | /** 168 | * Query to determine if we have column definitions. 169 | * 170 | * @return 171 | */ 172 | public boolean hasColumns(){ 173 | return this.columns != null; 174 | } 175 | 176 | /** 177 | * Query to determine if we have counter definitions. 178 | * 179 | * @return 180 | */ 181 | public boolean hasCounters(){ 182 | return this.counters != null; 183 | } 184 | 185 | /** 186 | * Get the list of column definitions. 187 | * 188 | * @return 189 | */ 190 | public List getColumns(){ 191 | return this.columns; 192 | } 193 | 194 | /** 195 | * Get the list of counter definitions. 196 | * @return 197 | */ 198 | public List getCounters(){ 199 | return this.counters; 200 | } 201 | 202 | } 203 | -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/trident/state/HBaseState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.storm.hbase.trident.state; 19 | 20 | import backtype.storm.topology.FailedException; 21 | import backtype.storm.tuple.Values; 22 | import com.google.common.collect.Lists; 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.hbase.HBaseConfiguration; 25 | import org.apache.hadoop.hbase.client.*; 26 | import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria; 27 | import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper; 28 | import org.apache.storm.hbase.common.ColumnList; 29 | import org.apache.storm.hbase.common.HBaseClient; 30 | import org.apache.storm.hbase.trident.mapper.TridentHBaseMapper; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | import storm.trident.operation.TridentCollector; 34 | import storm.trident.state.State; 35 | import storm.trident.tuple.TridentTuple; 36 | 37 | import java.io.Serializable; 38 | import java.util.List; 39 | import java.util.Map; 40 | 41 | public class HBaseState implements State { 42 | 43 | private static final Logger LOG = LoggerFactory.getLogger(HBaseState.class); 44 | 45 | private Options options; 46 | private HBaseClient hBaseClient; 47 | private Map map; 48 | private int numPartitions; 49 | private int partitionIndex; 50 | 51 | protected HBaseState(Map map, int partitionIndex, int numPartitions, Options options) { 52 | this.options = options; 53 | this.map = map; 54 | this.partitionIndex = partitionIndex; 55 | this.numPartitions = numPartitions; 56 | } 57 | 58 | public static class Options implements Serializable { 59 | private TridentHBaseMapper mapper; 60 | private Durability durability = Durability.SKIP_WAL; 61 | private HBaseProjectionCriteria projectionCriteria; 62 | private HBaseValueMapper rowToStormValueMapper; 63 | private String configKey; 64 | private String tableName; 65 | 66 | public Options withDurability(Durability durability) { 67 | this.durability = durability; 68 | return this; 69 | } 70 | 71 | public Options withProjectionCriteria(HBaseProjectionCriteria projectionCriteria) { 72 | this.projectionCriteria = projectionCriteria; 73 | return this; 74 | } 75 | 76 | public Options withConfigKey(String configKey) { 77 | this.configKey = configKey; 78 | return this; 79 | } 80 | 81 | public Options withTableName(String tableName) { 82 | this.tableName = tableName; 83 | return this; 84 | } 85 | 86 | public Options withRowToStormValueMapper(HBaseValueMapper rowToStormValueMapper) { 87 | this.rowToStormValueMapper = rowToStormValueMapper; 88 | return this; 89 | } 90 | 91 | public Options withMapper(TridentHBaseMapper mapper) { 92 | this.mapper = mapper; 93 | return this; 94 | } 95 | } 96 | 97 | protected void prepare() { 98 | final Configuration hbConfig = HBaseConfiguration.create(); 99 | Map conf = (Map) map.get(options.configKey); 100 | if(conf == null){ 101 | LOG.info("HBase configuration not found using key '" + options.configKey + "'"); 102 | LOG.info("Using HBase config from first hbase-site.xml found on classpath."); 103 | } else { 104 | if (conf.get("hbase.rootdir") == null) { 105 | LOG.warn("No 'hbase.rootdir' value found in configuration! Using HBase defaults."); 106 | } 107 | for (String key : conf.keySet()) { 108 | hbConfig.set(key, String.valueOf(map.get(key))); 109 | } 110 | } 111 | 112 | this.hBaseClient = new HBaseClient(conf, hbConfig, options.tableName); 113 | } 114 | 115 | @Override 116 | public void beginCommit(Long aLong) { 117 | LOG.debug("beginCommit is noop."); 118 | } 119 | 120 | @Override 121 | public void commit(Long aLong) { 122 | LOG.debug("commit is noop."); 123 | } 124 | 125 | public void updateState(List tuples, TridentCollector collector) { 126 | List mutations = Lists.newArrayList(); 127 | 128 | for (TridentTuple tuple : tuples) { 129 | byte[] rowKey = options.mapper.rowKey(tuple); 130 | ColumnList cols = options.mapper.columns(tuple); 131 | mutations.addAll(hBaseClient.constructMutationReq(rowKey, cols, options.durability)); 132 | } 133 | 134 | try { 135 | hBaseClient.batchMutate(mutations); 136 | } catch (Exception e) { 137 | LOG.warn("Batch write failed but some requests might have succeeded. Triggering replay.", e); 138 | throw new FailedException(e); 139 | } 140 | } 141 | 142 | public List> batchRetrieve(List tridentTuples) { 143 | List> batchRetrieveResult = Lists.newArrayList(); 144 | List gets = Lists.newArrayList(); 145 | for (TridentTuple tuple : tridentTuples) { 146 | byte[] rowKey = options.mapper.rowKey(tuple); 147 | gets.add(hBaseClient.constructGetRequests(rowKey, options.projectionCriteria)); 148 | } 149 | 150 | try { 151 | Result[] results = hBaseClient.batchGet(gets); 152 | for(Result result : results) { 153 | List values = options.rowToStormValueMapper.toValues(result); 154 | batchRetrieveResult.add(values); 155 | } 156 | } catch (Exception e) { 157 | LOG.warn("Batch get operation failed. Triggering replay.", e); 158 | throw new FailedException(e); 159 | } 160 | return batchRetrieveResult; 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #Storm HBase 2 | 3 | Storm/Trident integration for [Apache HBase](https://hbase.apache.org) 4 | 5 | ## Usage 6 | The main API for interacting with HBase is the `org.apache.storm.hbase.bolt.mapper.HBaseMapper` 7 | interface: 8 | 9 | ```java 10 | public interface HBaseMapper extends Serializable { 11 | byte[] rowKey(Tuple tuple); 12 | 13 | ColumnList columns(Tuple tuple); 14 | } 15 | ``` 16 | 17 | The `rowKey()` method is straightforward: given a Storm tuple, return a byte array representing the 18 | row key. 19 | 20 | The `columns()` method defines what will be written to an HBase row. The `ColumnList` class allows you 21 | to add both standard HBase columns as well as HBase counter columns. 22 | 23 | To add a standard column, use one of the `addColumn()` methods: 24 | 25 | ```java 26 | ColumnList cols = new ColumnList(); 27 | cols.addColumn(this.columnFamily, field.getBytes(), toBytes(tuple.getValueByField(field))); 28 | ``` 29 | 30 | To add a counter column, use one of the `addCounter()` methods: 31 | 32 | ```java 33 | ColumnList cols = new ColumnList(); 34 | cols.addCounter(this.columnFamily, field.getBytes(), toLong(tuple.getValueByField(field))); 35 | ``` 36 | 37 | ### SimpleHBaseMapper 38 | `storm-hdfs` includes a general purpose `HBaseMapper` implementation called `SimpleHBaseMapper` that can map Storm 39 | tuples to both regular HBase columns as well as counter columns. 40 | 41 | To use `SimpleHBaseMapper`, you simply tell it which fields to map to which types of columns. 42 | 43 | The following code create a `SimpleHBaseMapper` instance that: 44 | 45 | 1. Uses the `word` tuple value as a row key. 46 | 2. Adds a standard HBase column for the tuple field `word`. 47 | 3. Adds an HBase counter column for the tuple field `count`. 48 | 4. Writes values to the `cf` column family. 49 | 50 | ```java 51 | SimpleHBaseMapper mapper = new SimpleHBaseMapper() 52 | .withRowKeyField("word") 53 | .withColumnFields(new Fields("word")) 54 | .withCounterFields(new Fields("count")) 55 | .withColumnFamily("cf"); 56 | ``` 57 | ### HBaseBolt 58 | To use the `HBaseBolt`, construct it with the name of the table to write to, an a `HBaseMapper` implementation: 59 | 60 | ```java 61 | HBaseBolt hbase = new HBaseBolt("WordCount", mapper); 62 | ``` 63 | 64 | The `HBaseBolt` will delegate to the `mapper` instance to figure out how to persist tuple data to HBase. 65 | 66 | ###HBaseValueMapper 67 | This class allows you to transform the HBase lookup result into storm Values that will be emitted by the `HBaseLookupBolt`. 68 | 69 | ```java 70 | public interface HBaseValueMapper extends Serializable { 71 | public List toTuples(Result result) throws Exception; 72 | void declareOutputFields(OutputFieldsDeclarer declarer); 73 | } 74 | ``` 75 | 76 | The `toTuples` method takes in a HBase `Result` instance and expects a List of `Values` instant. 77 | Each of the value returned by this function will be emitted by the `HBaseLookupBolt`. 78 | 79 | The `declareOutputFields` should be used to declare the outputFields of the `HBaseLookupBolt`. 80 | 81 | There is an example implementation in `src/test/java` directory. 82 | ###HBaseProjectionCriteria 83 | This class allows you to specify the projection criteria for your HBase Get function. This is optional parameter 84 | for the lookupBolt and if you do not specify this instance all the columns will be returned by `HBaseLookupBolt`. 85 | 86 | ```java 87 | public class HBaseProjectionCriteria implements Serializable { 88 | public HBaseProjectionCriteria addColumnFamily(String columnFamily); 89 | public HBaseProjectionCriteria addColumn(ColumnMetaData column); 90 | ``` 91 | `addColumnFamily` takes in columnFamily. Setting this parameter means all columns for this family will be included 92 | in the projection. 93 | 94 | `addColumn` takes in a columnMetaData instance. Setting this parameter means only this column from the column familty 95 | will be part of your projection. 96 | The following code creates a projectionCriteria which specifies a projection criteria that: 97 | 98 | 1. includes count column from column family cf. 99 | 2. includes all columns from column family cf2. 100 | 101 | ```java 102 | HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria() 103 | .addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count")) 104 | .addColumnFamily("cf2"); 105 | ``` 106 | 107 | ###HBaseLookupBolt 108 | To use the `HBaseLookupBolt`, Construct it with the name of the table to write to, an implementation of `HBaseMapper` 109 | and an implementation of `HBaseRowToStormValueMapper`. You can optionally specify a `HBaseProjectionCriteria`. 110 | 111 | The `HBaseLookupBolt` will use the mapper to get rowKey to lookup for. It will use the `HBaseProjectionCriteria` to 112 | figure out which columns to include in the result and it will leverage the `HBaseRowToStormValueMapper` to get the 113 | values to be emitted by the bolt. 114 | 115 | You can look at an example topology LookupWordCount.java under `src/test/java`. 116 | ## Example: Persistent Word Count 117 | A runnable example can be found in the `src/test/java` directory. 118 | 119 | ### Setup 120 | The following steps assume you are running HBase locally, or there is an `hbase-site.xml` on the 121 | classpath pointing to your HBase cluster. 122 | 123 | Use the `hbase shell` command to create the schema: 124 | 125 | ``` 126 | > create 'WordCount', 'cf' 127 | ``` 128 | 129 | ### Execution 130 | Run the `org.apache.storm.hbase.topology.PersistenWordCount` class (it will run the topology for 10 seconds, then exit). 131 | 132 | After (or while) the word count topology is running, run the `org.apache.storm.hbase.topology.WordCountClient` class 133 | to view the counter values stored in HBase. You should see something like to following: 134 | 135 | ``` 136 | Word: 'apple', Count: 6867 137 | Word: 'orange', Count: 6645 138 | Word: 'pineapple', Count: 6954 139 | Word: 'banana', Count: 6787 140 | Word: 'watermelon', Count: 6806 141 | ``` 142 | 143 | For reference, the sample topology is listed below: 144 | 145 | ```java 146 | public class PersistentWordCount { 147 | private static final String WORD_SPOUT = "WORD_SPOUT"; 148 | private static final String COUNT_BOLT = "COUNT_BOLT"; 149 | private static final String HBASE_BOLT = "HBASE_BOLT"; 150 | 151 | 152 | public static void main(String[] args) throws Exception { 153 | Config config = new Config(); 154 | 155 | WordSpout spout = new WordSpout(); 156 | WordCounter bolt = new WordCounter(); 157 | 158 | SimpleHBaseMapper mapper = new SimpleHBaseMapper() 159 | .withRowKeyField("word") 160 | .withColumnFields(new Fields("word")) 161 | .withCounterFields(new Fields("count")) 162 | .withColumnFamily("cf"); 163 | 164 | HBaseBolt hbase = new HBaseBolt("WordCount", mapper); 165 | 166 | 167 | // wordSpout ==> countBolt ==> HBaseBolt 168 | TopologyBuilder builder = new TopologyBuilder(); 169 | 170 | builder.setSpout(WORD_SPOUT, spout, 1); 171 | builder.setBolt(COUNT_BOLT, bolt, 1).shuffleGrouping(WORD_SPOUT); 172 | builder.setBolt(HBASE_BOLT, hbase, 1).fieldsGrouping(COUNT_BOLT, new Fields("word")); 173 | 174 | 175 | if (args.length == 0) { 176 | LocalCluster cluster = new LocalCluster(); 177 | cluster.submitTopology("test", config, builder.createTopology()); 178 | Thread.sleep(10000); 179 | cluster.killTopology("test"); 180 | cluster.shutdown(); 181 | System.exit(0); 182 | } else { 183 | config.setNumWorkers(3); 184 | StormSubmitter.submitTopology(args[0], config, builder.createTopology()); 185 | } 186 | } 187 | } 188 | ``` -------------------------------------------------------------------------------- /src/main/java/org/apache/storm/hbase/trident/state/HBaseMapState.java: -------------------------------------------------------------------------------- 1 | package org.apache.storm.hbase.trident.state; 2 | 3 | import backtype.storm.task.IMetricsContext; 4 | import backtype.storm.topology.FailedException; 5 | import backtype.storm.tuple.Values; 6 | import com.google.common.collect.Maps; 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.hbase.HBaseConfiguration; 9 | import org.apache.hadoop.hbase.client.*; 10 | import org.apache.hadoop.hbase.security.UserProvider; 11 | import org.apache.storm.hbase.security.HBaseSecurityUtil; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import storm.trident.state.*; 15 | import storm.trident.state.map.*; 16 | 17 | import java.io.ByteArrayOutputStream; 18 | import java.io.IOException; 19 | import java.io.InterruptedIOException; 20 | import java.io.Serializable; 21 | import java.security.PrivilegedExceptionAction; 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | 27 | public class HBaseMapState implements IBackingMap { 28 | private static Logger LOG = LoggerFactory.getLogger(HBaseMapState.class); 29 | 30 | private int partitionNum; 31 | 32 | 33 | @SuppressWarnings("rawtypes") 34 | private static final Map DEFAULT_SERIALZERS = Maps.newHashMap(); 35 | 36 | static { 37 | DEFAULT_SERIALZERS.put(StateType.NON_TRANSACTIONAL, new JSONNonTransactionalSerializer()); 38 | DEFAULT_SERIALZERS.put(StateType.TRANSACTIONAL, new JSONTransactionalSerializer()); 39 | DEFAULT_SERIALZERS.put(StateType.OPAQUE, new JSONOpaqueSerializer()); 40 | } 41 | 42 | private Options options; 43 | private Serializer serializer; 44 | private HTable table; 45 | 46 | public HBaseMapState(final Options options, Map map, int partitionNum) { 47 | this.options = options; 48 | this.serializer = options.serializer; 49 | this.partitionNum = partitionNum; 50 | 51 | final Configuration hbConfig = HBaseConfiguration.create(); 52 | Map conf = (Map)map.get(options.configKey); 53 | if(conf == null){ 54 | LOG.info("HBase configuration not found using key '" + options.configKey + "'"); 55 | LOG.info("Using HBase config from first hbase-site.xml found on classpath."); 56 | } else { 57 | if (conf.get("hbase.rootdir") == null) { 58 | LOG.warn("No 'hbase.rootdir' value found in configuration! Using HBase defaults."); 59 | } 60 | for (String key : conf.keySet()) { 61 | hbConfig.set(key, String.valueOf(map.get(key))); 62 | } 63 | } 64 | 65 | try{ 66 | UserProvider provider = HBaseSecurityUtil.login(map, hbConfig); 67 | this.table = provider.getCurrent().getUGI().doAs(new PrivilegedExceptionAction() { 68 | @Override 69 | public HTable run() throws IOException { 70 | return new HTable(hbConfig, options.tableName); 71 | } 72 | }); 73 | } catch(Exception e){ 74 | throw new RuntimeException("HBase bolt preparation failed: " + e.getMessage(), e); 75 | } 76 | 77 | } 78 | 79 | 80 | public static class Options implements Serializable { 81 | 82 | public Serializer serializer = null; 83 | public int cacheSize = 5000; 84 | public String globalKey = "$HBASE_STATE_GLOBAL$"; 85 | public String configKey = "hbase.config"; 86 | public String tableName; 87 | public String columnFamily; 88 | public String qualifier; 89 | } 90 | 91 | 92 | @SuppressWarnings("rawtypes") 93 | public static StateFactory opaque() { 94 | Options options = new Options(); 95 | return opaque(options); 96 | } 97 | 98 | @SuppressWarnings("rawtypes") 99 | public static StateFactory opaque(Options opts) { 100 | 101 | return new Factory(StateType.OPAQUE, opts); 102 | } 103 | 104 | @SuppressWarnings("rawtypes") 105 | public static StateFactory transactional() { 106 | Options options = new Options(); 107 | return transactional(options); 108 | } 109 | 110 | @SuppressWarnings("rawtypes") 111 | public static StateFactory transactional(Options opts) { 112 | return new Factory(StateType.TRANSACTIONAL, opts); 113 | } 114 | 115 | public static StateFactory nonTransactional() { 116 | Options options = new Options(); 117 | return nonTransactional(options); 118 | } 119 | 120 | public static StateFactory nonTransactional(Options opts) { 121 | return new Factory(StateType.NON_TRANSACTIONAL, opts); 122 | } 123 | 124 | 125 | protected static class Factory implements StateFactory { 126 | private StateType stateType; 127 | private Options options; 128 | 129 | @SuppressWarnings({"rawtypes", "unchecked"}) 130 | public Factory(StateType stateType, Options options) { 131 | this.stateType = stateType; 132 | this.options = options; 133 | 134 | if (this.options.serializer == null) { 135 | this.options.serializer = DEFAULT_SERIALZERS.get(stateType); 136 | } 137 | 138 | if (this.options.serializer == null) { 139 | throw new RuntimeException("Serializer should be specified for type: " + stateType); 140 | } 141 | } 142 | 143 | @SuppressWarnings({"rawtypes", "unchecked"}) 144 | public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { 145 | LOG.info("Preparing HBase State for partition {} of {}.", partitionIndex + 1, numPartitions); 146 | IBackingMap state = new HBaseMapState(options, conf, partitionIndex); 147 | 148 | if(options.cacheSize > 0) { 149 | state = new CachedMap(state, options.cacheSize); 150 | } 151 | 152 | MapState mapState; 153 | switch (stateType) { 154 | case NON_TRANSACTIONAL: 155 | mapState = NonTransactionalMap.build(state); 156 | break; 157 | case OPAQUE: 158 | mapState = OpaqueMap.build(state); 159 | break; 160 | case TRANSACTIONAL: 161 | mapState = TransactionalMap.build(state); 162 | break; 163 | default: 164 | throw new IllegalArgumentException("Unknown state type: " + stateType); 165 | } 166 | return new SnapshottableMap(mapState, new Values(options.globalKey)); 167 | } 168 | 169 | } 170 | 171 | @Override 172 | public List multiGet(List> keys) { 173 | List gets = new ArrayList(); 174 | for(List key : keys){ 175 | LOG.info("Partition: {}, GET: {}", this.partitionNum, key); 176 | Get get = new Get(toRowKey(key)); 177 | get.addColumn(this.options.columnFamily.getBytes(), this.options.qualifier.getBytes()); 178 | gets.add(get); 179 | } 180 | 181 | List retval = new ArrayList(); 182 | try { 183 | Result[] results = this.table.get(gets); 184 | for (Result result : results) { 185 | byte[] value = result.getValue(this.options.columnFamily.getBytes(), this.options.qualifier.getBytes()); 186 | if(value != null) { 187 | retval.add(this.serializer.deserialize(value)); 188 | } else { 189 | retval.add(null); 190 | } 191 | } 192 | } catch(IOException e){ 193 | throw new FailedException("IOException while reading from HBase.", e); 194 | } 195 | return retval; 196 | } 197 | 198 | @Override 199 | public void multiPut(List> keys, List values) { 200 | List puts = new ArrayList(keys.size()); 201 | for (int i = 0; i < keys.size(); i++) { 202 | LOG.info("Partiton: {}, Key: {}, Value: {}", new Object[]{this.partitionNum, keys.get(i), new String(this.serializer.serialize(values.get(i)))}); 203 | Put put = new Put(toRowKey(keys.get(i))); 204 | T val = values.get(i); 205 | put.add(this.options.columnFamily.getBytes(), 206 | this.options.qualifier.getBytes(), 207 | this.serializer.serialize(val)); 208 | 209 | puts.add(put); 210 | } 211 | try { 212 | this.table.put(puts); 213 | } catch (InterruptedIOException e) { 214 | throw new FailedException("Interrupted while writing to HBase", e); 215 | } catch (RetriesExhaustedWithDetailsException e) { 216 | throw new FailedException("Retries exhaused while writing to HBase", e); 217 | } 218 | } 219 | 220 | 221 | private byte[] toRowKey(List keys) { 222 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 223 | try { 224 | for (Object key : keys) { 225 | bos.write(String.valueOf(key).getBytes()); 226 | } 227 | bos.close(); 228 | } catch (IOException e){ 229 | throw new RuntimeException("IOException creating HBase row key.", e); 230 | } 231 | return bos.toByteArray(); 232 | } 233 | } 234 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | --------------------------------------------------------------------------------