├── resources ├── test.txt └── splitsentence.py ├── README.md ├── .travis.yml ├── src ├── test │ ├── java │ │ └── storm │ │ │ └── benchmark │ │ │ ├── metrics │ │ │ └── DRPCMetricsCollectorTest.java │ │ │ ├── util │ │ │ ├── TestUtils.java │ │ │ ├── TupleHelpersTest.java │ │ │ ├── MockTupleHelpers.java │ │ │ ├── MetricsUtilsTest.java │ │ │ ├── FileUtilsTest.java │ │ │ └── BenchmarkUtilsTest.java │ │ │ ├── tools │ │ │ ├── FileReaderTest.java │ │ │ ├── producer │ │ │ │ └── kafka │ │ │ │ │ ├── KafkaProducerTest.java │ │ │ │ │ ├── FileReadKafkaProducerTest.java │ │ │ │ │ └── PageViewKafkaProducerTest.java │ │ │ ├── RunnerTest.java │ │ │ └── SlidingWindowTest.java │ │ │ ├── lib │ │ │ ├── spout │ │ │ │ ├── pageview │ │ │ │ │ ├── PageViewTest.java │ │ │ │ │ └── PageViewSpoutTest.java │ │ │ │ ├── TridentFileReadSpoutTest.java │ │ │ │ ├── RandomMessageSpoutTest.java │ │ │ │ └── FileReadSpoutTest.java │ │ │ ├── operation │ │ │ │ ├── OneTest.java │ │ │ │ ├── WordSplitTest.java │ │ │ │ ├── ExpandTest.java │ │ │ │ └── DistinctTest.java │ │ │ ├── reducer │ │ │ │ ├── LongSummerTest.java │ │ │ │ └── SetReducerTest.java │ │ │ └── bolt │ │ │ │ ├── ConstBoltTest.java │ │ │ │ ├── FilterBoltTest.java │ │ │ │ ├── PageViewBoltTest.java │ │ │ │ └── common │ │ │ │ └── RollingBoltTest.java │ │ │ └── benchmarks │ │ │ ├── RollingSortTest.java │ │ │ ├── GrepTest.java │ │ │ ├── SOLTest.java │ │ │ ├── DataCleanTest.java │ │ │ ├── PageViewCountTest.java │ │ │ ├── RollingCountTest.java │ │ │ ├── UniqueVisitorTest.java │ │ │ ├── FileReadWordCountTest.java │ │ │ └── common │ │ │ ├── StormBenchmarkTest.java │ │ │ └── WordCountTest.java │ └── resources │ │ └── logback.xml └── main │ └── java │ └── storm │ └── benchmark │ ├── lib │ ├── reducer │ │ ├── Reducer.java │ │ ├── LongSummer.java │ │ └── SetReducer.java │ ├── operation │ │ ├── One.java │ │ ├── Expand.java │ │ ├── Print.java │ │ ├── WordSplit.java │ │ └── Distinct.java │ ├── bolt │ │ ├── ConstBolt.java │ │ ├── FilterBolt.java │ │ ├── PageViewBolt.java │ │ ├── RollingBolt.java │ │ ├── UniqueVisitorBolt.java │ │ └── RollingCountBolt.java │ └── spout │ │ ├── pageview │ │ ├── PageViewSpout.java │ │ └── PageView.java │ │ ├── FileReadSpout.java │ │ ├── RandomMessageSpout.java │ │ └── TridentFileReadSpout.java │ ├── api │ ├── IProducer.java │ ├── IApplication.java │ └── IBenchmark.java │ ├── metrics │ ├── IMetricsCollector.java │ └── MetricsCollectorConfig.java │ ├── util │ ├── TupleHelpers.java │ ├── FileUtils.java │ ├── BenchmarkUtils.java │ ├── KafkaUtils.java │ └── MetricsUtils.java │ ├── benchmarks │ ├── FileReadWordCount.java │ ├── KafkaWordCount.java │ ├── common │ │ └── StormBenchmark.java │ ├── DataClean.java │ ├── PageViewCount.java │ ├── RollingCount.java │ ├── SOL.java │ ├── UniqueVisitor.java │ └── TridentWordCount.java │ └── tools │ ├── FileReader.java │ ├── producer │ └── kafka │ │ ├── PageViewKafkaProducer.java │ │ └── FileReadKafkaProducer.java │ ├── LocalRunner.java │ └── Runner.java ├── conf ├── fileread_producer.yaml ├── pageview_producer.yaml ├── sol.yaml ├── wordcount.yaml ├── rollingcount.yaml ├── rollingsort.yaml ├── kafka_wordcount.yaml ├── dataclean.yaml ├── pageview.yaml ├── trident.yaml ├── uniquevisitor.yaml ├── grep.yaml └── drpc.yaml ├── .gitignore └── bin └── stormbench /resources/test.txt: -------------------------------------------------------------------------------- 1 | first line 2 | second line 3 | third line -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This project has been moved to https://github.com/intel-hadoop/storm-benchmark 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: 2 | - java 3 | jdk: 4 | - oraclejdk7 5 | cache: 6 | directories: 7 | - $HOME/.m2 8 | git: 9 | depth: 1 -------------------------------------------------------------------------------- /resources/splitsentence.py: -------------------------------------------------------------------------------- 1 | import storm 2 | 3 | class SplitSentenceBolt(storm.BasicBolt): 4 | def process(self, tup): 5 | words = tup.values[0].split(" ") 6 | for word in words: 7 | storm.emit([word]) 8 | 9 | SplitSentenceBolt().run() -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/metrics/DRPCMetricsCollectorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.metrics; 20 | 21 | public class DRPCMetricsCollectorTest { 22 | 23 | // TODO: add implementations 24 | } 25 | -------------------------------------------------------------------------------- /conf/fileread_producer.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # kafka configurations 18 | broker.list: "127.0.0.1:9092" 19 | zookeeper.servers: "127.0.0.1:2181" 20 | kafka.root.path: "/kafka" 21 | topic: "fileread" 22 | 23 | # topology configurations 24 | topology.name: "FileReadKafkaProducer" 25 | component.spout_num: 4 26 | component.bolt_num: 4 -------------------------------------------------------------------------------- /conf/pageview_producer.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # kafka configurations 18 | broker.list: "127.0.0.1:9092" 19 | zookeeper.servers: "127.0.0.1:2181" 20 | kafka.root.path: "/kafka" 21 | topic: "pageview" 22 | 23 | # topology configurations 24 | topology.name: "PageViewKafkaProducer" 25 | component.spout_num: 4 26 | component.bolt_num: 4 -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/reducer/Reducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.reducer; 20 | 21 | import java.io.Serializable; 22 | 23 | public interface Reducer extends Serializable { 24 | public V reduce(V v1, V v2); 25 | public V zero(); 26 | public boolean isZero(V v); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/api/IProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.api; 20 | 21 | /** 22 | * benchmarks get data from a producer, which is 23 | * also a storm application. Refer to 24 | * {@link storm.benchmark.tools.producer.kafka.KafkaProducer} 25 | * for an example 26 | */ 27 | public interface IProducer extends IApplication { 28 | } 29 | -------------------------------------------------------------------------------- /conf/sol.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # topology configurations 24 | topology.workers: 4 25 | topology.acker.executors: 4 26 | topology.max.spout.pending: 200 27 | topology.name: "SOL" 28 | topology.level: 3 29 | message.size: 100 30 | component.spout_num: 4 31 | component.bolt_num: 8 -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/metrics/IMetricsCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.metrics; 20 | 21 | public interface IMetricsCollector { 22 | public static enum MetricsItem { 23 | SUPERVISOR_STATS, 24 | TOPOLOGY_STATS, 25 | THROUGHPUT, 26 | SPOUT_THROUGHPUT, 27 | SPOUT_LATENCY, 28 | THROUGHPUT_IN_MB, 29 | ALL 30 | } 31 | public void run(); 32 | } 33 | -------------------------------------------------------------------------------- /conf/wordcount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # topology configurations 24 | topology.workers: 4 25 | topology.acker.executors: 4 26 | topology.max.spout.pending: 200 27 | topology.name: "WordCount" 28 | component.spout_num: 4 29 | component.split_bolt_num: 8 30 | component.count_bolt_num: 8 31 | -------------------------------------------------------------------------------- /src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | %-4r [%t] %-5p %c - %m%n 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /conf/rollingcount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # topology configurations 24 | topology.workers: 4 25 | topology.acker.executors: 4 26 | topology.max.spout.pending: 200 27 | topology.name: "RollingCount" 28 | component.spout_num: 4 29 | component.split_bolt_num: 8 30 | component.count_bolt_num: 8 31 | window.length: 150 # 150s 32 | emit.frequency: 30 # 30s -------------------------------------------------------------------------------- /conf/rollingsort.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # topology configurations 24 | topology.workers: 4 25 | topology.acker.executors: 4 26 | topology.max.spout.pending: 200 27 | topology.name: "RollingSort" 28 | component.spout_num: 4 29 | component.sort_bolt_num: 8 30 | emit.frequency: 300 # 5mins 31 | chunk.size: 2000000 # 2M 32 | message.size: 10000 # 10KB -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/api/IApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.api; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | 24 | /** 25 | * superclass of any runnable storm applications 26 | * users should not implement this interface directly but 27 | * {@link IBenchmark} or {@link IProducer} 28 | */ 29 | public interface IApplication { 30 | public StormTopology getTopology(Config config); 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/util/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.generated.ComponentCommon; 22 | 23 | import static org.fest.assertions.api.Assertions.assertThat; 24 | 25 | public class TestUtils { 26 | private TestUtils() { 27 | } 28 | 29 | public static void verifyParallelism(ComponentCommon component, int expected) { 30 | assertThat(component.get_parallelism_hint()).isEqualTo(expected); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/util/TupleHelpersTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import org.testng.annotations.Test; 22 | 23 | import static org.fest.assertions.api.Assertions.assertThat; 24 | 25 | public class TupleHelpersTest { 26 | 27 | @Test 28 | public void testIsTickTuple() { 29 | assertThat(TupleHelpers.isTickTuple(MockTupleHelpers.mockTickTuple())).isTrue(); 30 | assertThat(TupleHelpers.isTickTuple(MockTupleHelpers.mockAnyTuple())).isFalse(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/util/TupleHelpers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.Constants; 22 | import backtype.storm.tuple.Tuple; 23 | 24 | public final class TupleHelpers { 25 | 26 | private TupleHelpers() { 27 | } 28 | 29 | public static boolean isTickTuple(Tuple tuple) { 30 | return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) && tuple.getSourceStreamId().equals( 31 | Constants.SYSTEM_TICK_STREAM_ID); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/reducer/LongSummer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.reducer; 20 | 21 | 22 | public class LongSummer implements Reducer { 23 | 24 | private static final long serialVersionUID = -1102373670176409091L; 25 | 26 | @Override 27 | public Long reduce(Long v1, Long v2) { 28 | return v1 + v2; 29 | } 30 | 31 | @Override 32 | public Long zero() { 33 | return 0L; 34 | } 35 | 36 | @Override 37 | public boolean isZero(Long aLong) { 38 | return 0L == aLong; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/operation/One.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import storm.trident.operation.CombinerAggregator; 22 | import storm.trident.tuple.TridentTuple; 23 | 24 | public class One implements CombinerAggregator { 25 | @Override 26 | public Integer init(TridentTuple tuple) { 27 | return 1; 28 | } 29 | 30 | @Override 31 | public Integer combine(Integer val1, Integer val2) { 32 | return 1; 33 | } 34 | 35 | @Override 36 | public Integer zero() { 37 | return 1; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /conf/kafka_wordcount.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # kafka configurations 24 | broker.list: "127.0.0.1:9092" 25 | zookeeper.servers: "127.0.0.1:2181" 26 | kafka.root.path: "/kafka" 27 | topic: "fileread" 28 | client_id: "wordcount" 29 | 30 | # topology configurations 31 | topology.workers: 4 32 | topology.acker.executors: 4 33 | topology.max.spout.pending: 200 34 | topology.name: "WordCount" 35 | component.spout_num: 4 36 | component.split_bolt_num: 8 37 | component.count_bolt_num: 8 -------------------------------------------------------------------------------- /conf/dataclean.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # kafka configurations 24 | broker.list: "127.0.0.1:9092" 25 | zookeeper.servers: "127.0.0.1:2181" 26 | kafka.root.path: "/kafka" 27 | topic: "pageview" 28 | client_id: "data_cleanse" 29 | 30 | # topology configurations 31 | topology.workers: 4 32 | topology.acker.executors: 4 33 | topology.max.spout.pending: 200 34 | topology.name: "DataCleanse" 35 | component.spout_num: 4 36 | component.view_bolt_num: 8 37 | component.filter_bolt_num: 8 38 | -------------------------------------------------------------------------------- /conf/pageview.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # kafka configurations 24 | broker.list: "127.0.0.1:9092" 25 | zookeeper.servers: "127.0.0.1:2181" 26 | kafka.root.path: "/kafka" 27 | topic: "pageview" 28 | client_id: "pageview_count" 29 | 30 | # topology configurations 31 | topology.workers: 4 32 | topology.acker.executors: 4 33 | topology.max.spout.pending: 200 34 | topology.name: "PageViewCount" 35 | component.spout_num: 4 36 | component.view_bolt_num: 8 37 | component.count_bolt_num: 8 38 | -------------------------------------------------------------------------------- /conf/trident.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # kafka configurations 24 | broker.list: "127.0.0.1:9092" 25 | zookeeper.servers: "127.0.0.1:2181" 26 | kafka.root.path: "/kafka" 27 | topic: "fileread" 28 | client_id: "trident" 29 | 30 | # topology configurations 31 | topology.workers: 4 32 | topology.acker.executors: 4 33 | topology.max.spout.pending: 200 34 | topology.name: "TridentWordCount" 35 | component.spout_num: 4 36 | component.split_bolt_num: 8 37 | component.count_bolt_num: 8 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /conf/uniquevisitor.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # kafka configurations 24 | broker.list: "127.0.0.1:9092" 25 | zookeeper.servers: "127.0.0.1:2181" 26 | kafka.root.path: "/kafka" 27 | topic: "pageview" 28 | client_id: "unique_visitor" 29 | 30 | # topology configurations 31 | topology.workers: 4 32 | topology.acker.executors: 4 33 | topology.max.spout.pending: 200 34 | topology.name: "UniqueVisitor" 35 | component.spout_num: 4 36 | component.view_bolt_num: 8 37 | component.uniquer_bolt_num: 8 38 | 39 | 40 | -------------------------------------------------------------------------------- /conf/grep.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # kafka configurations 24 | broker.list: "127.0.0.1:9092" 25 | zookeeper.servers: "127.0.0.1:2181" 26 | kafka.root.path: "/kafka" 27 | topic: "fileread" 28 | client_id: "grep" 29 | 30 | # topology configurations 31 | topology.workers: 4 32 | topology.acker.executors: 4 33 | topology.max.spout.pending: 200 34 | topology.name: "Grep" 35 | component.spout_name: 4 36 | component.find_bolt_name: 8 37 | component.count_bolt_name: 8 38 | pattern_string: "string" 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/FileReadWordCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import storm.benchmark.benchmarks.common.WordCount; 24 | import storm.benchmark.lib.spout.FileReadSpout; 25 | import storm.benchmark.util.BenchmarkUtils; 26 | 27 | public class FileReadWordCount extends WordCount { 28 | 29 | @Override 30 | public StormTopology getTopology(Config config) { 31 | spout = new FileReadSpout(BenchmarkUtils.ifAckEnabled(config)); 32 | return super.getTopology(config); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/api/IBenchmark.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.api; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import storm.benchmark.metrics.IMetricsCollector; 24 | 25 | /** 26 | * users could write their own benchmarks against this interface. 27 | * a benchmark is a storm application augmented with a 28 | * {@link storm.benchmark.metrics.IMetricsCollector}, which collects 29 | * metrics like throughout or latency 30 | */ 31 | public interface IBenchmark extends IApplication { 32 | public IMetricsCollector getMetricsCollector(Config config, StormTopology topology); 33 | } 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Mobile Tools for Java (J2ME) 4 | .mtj.tmp/ 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 12 | hs_err_pid* 13 | 14 | # SVN 15 | .svn/ 16 | 17 | # Eclipse 18 | *.pydevproject 19 | .metadata 20 | .gradle 21 | bin/ 22 | tmp/ 23 | *.tmp 24 | *.bak 25 | *.swp 26 | *~.nib 27 | local.properties 28 | .settings/ 29 | .loadpath 30 | 31 | # External tool builders 32 | .externalToolBuilders/ 33 | 34 | # Locally stored "Eclipse launch configurations" 35 | *.launch 36 | 37 | # CDT-specific 38 | .cproject 39 | 40 | # PDT-specific 41 | .buildpath 42 | 43 | # sbteclipse plugin 44 | .target 45 | 46 | # TeXlipse plugin 47 | .texlipse 48 | 49 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 50 | 51 | ## Directory-based project format 52 | .idea/ 53 | # if you remove the above rule, at least ignore user-specific stuff: 54 | # .idea/workspace.xml 55 | # .idea/tasks.xml 56 | # and these sensitive or high-churn files: 57 | # .idea/dataSources.ids 58 | # .idea/dataSources.xml 59 | # .idea/sqlDataSources.xml 60 | # .idea/dynamic.xml 61 | 62 | ## File-based project format 63 | *.ipr 64 | *.iws 65 | *.iml 66 | 67 | ## Additional for IntelliJ 68 | out/ 69 | 70 | # generated by mpeltonen/sbt-idea plugin 71 | .idea_modules/ 72 | 73 | # generated by JIRA plugin 74 | atlassian-ide-plugin.xml 75 | 76 | # generated by Crashlytics plugin (for Android Studio and Intellij) 77 | com_crashlytics_export_strings.xml 78 | 79 | !bin -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/reducer/SetReducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.reducer; 20 | 21 | import com.google.common.collect.ImmutableSet; 22 | import com.google.common.collect.Sets; 23 | 24 | import java.util.Set; 25 | 26 | public class SetReducer implements Reducer> { 27 | 28 | private static final long serialVersionUID = -5236751818829417995L; 29 | 30 | @Override 31 | public Set reduce(Set v1, Set v2) { 32 | return Sets.union(v1, v2); 33 | } 34 | 35 | @Override 36 | public Set zero() { 37 | return ImmutableSet.of(); 38 | } 39 | 40 | @Override 41 | public boolean isZero(Set ts) { 42 | return ts.isEmpty(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/KafkaWordCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.spout.SchemeAsMultiScheme; 24 | import storm.benchmark.benchmarks.common.WordCount; 25 | import storm.benchmark.util.KafkaUtils; 26 | import storm.kafka.KafkaSpout; 27 | import storm.kafka.StringScheme; 28 | 29 | public class KafkaWordCount extends WordCount { 30 | 31 | @Override 32 | public StormTopology getTopology(Config config) { 33 | spout = new KafkaSpout(KafkaUtils.getSpoutConfig(config, new SchemeAsMultiScheme(new StringScheme()))); 34 | return super.getTopology(config); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /conf/drpc.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # metrics configurations 18 | metrics.enabled: true 19 | metrics.poll: 60000 # 60 secs 20 | metrics.time: 300000 # 5 mins 21 | metrics.path: "reports" 22 | 23 | # kafka configurations 24 | broker.list: "127.0.0.1:9092" 25 | zookeeper.servers: "127.0.0.1:2181" 26 | kafka.root.path: "/kafka" 27 | topic: "pageview" 28 | client_id: "drpc" 29 | 30 | # topology configurations 31 | topology.workers: 4 32 | topology.acker.executors: 4 33 | topology.max.spout.pending: 200 34 | topology.name: "DRPC" 35 | component.spout_num: 3 36 | component.page_bolt_num: 6 37 | component.view_bolt_num: 9 38 | component.user_bolt_num: 4 39 | component.follower_bolt_num: 8 40 | 41 | # drpc configurations 42 | drpc.server: "127.0.0.1" 43 | drpc.server: "3772" 44 | 45 | 46 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/tools/FileReaderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools; 20 | 21 | import org.testng.annotations.Test; 22 | import storm.benchmark.tools.FileReader; 23 | 24 | import static org.fest.assertions.api.Assertions.assertThat; 25 | 26 | public class FileReaderTest { 27 | private static final String FILE = "/resources/test.txt"; 28 | 29 | @Test 30 | public void testNextLine() { 31 | FileReader reader = new FileReader(FILE); 32 | assertThat(reader.nextLine()).isEqualTo("first line"); 33 | assertThat(reader.nextLine()).isEqualTo("second line"); 34 | assertThat(reader.nextLine()).isEqualTo("third line"); 35 | assertThat(reader.nextLine()).isEqualTo("first line"); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/spout/pageview/PageViewTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout.pageview; 20 | 21 | import org.testng.annotations.Test; 22 | 23 | import static org.fest.assertions.api.Assertions.assertThat; 24 | 25 | public class PageViewTest { 26 | 27 | @Test 28 | public void testFromString() { 29 | String pvString = "http://foo.com\t200\t100000\t1"; 30 | PageView pageView = PageView.fromString(pvString); 31 | assertThat(pageView.url).isEqualTo("http://foo.com"); 32 | assertThat(pageView.status).isEqualTo(200); 33 | assertThat(pageView.zipCode).isEqualTo(100000); 34 | assertThat(pageView.userID).isEqualTo(1); 35 | assertThat(pageView.toString()).isEqualTo(pvString); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/operation/Expand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import backtype.storm.tuple.Values; 22 | import storm.trident.operation.BaseFunction; 23 | import storm.trident.operation.TridentCollector; 24 | import storm.trident.tuple.TridentTuple; 25 | 26 | import java.util.Iterator; 27 | 28 | public class Expand extends BaseFunction { 29 | 30 | private static final long serialVersionUID = -8912233679276693760L; 31 | 32 | @Override 33 | public void execute(TridentTuple tuple, TridentCollector collector) { 34 | Iterable iterable = (Iterable) tuple.getValue(0); 35 | Iterator iterator = iterable.iterator(); 36 | while (iterator.hasNext()) { 37 | collector.emit(new Values(iterator.next())); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/operation/Print.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import org.apache.log4j.Logger; 22 | import storm.trident.operation.BaseFunction; 23 | import storm.trident.operation.TridentCollector; 24 | import storm.trident.tuple.TridentTuple; 25 | 26 | 27 | /** 28 | * for debug purpose 29 | */ 30 | public class Print extends BaseFunction { 31 | 32 | private static final Logger LOG = Logger.getLogger(Print.class); 33 | private final String fields; 34 | 35 | public Print(String fields) { 36 | this.fields = fields; 37 | } 38 | 39 | @Override 40 | public void execute(TridentTuple tuple, TridentCollector collector) { 41 | System.out.println("Print " + fields + ": " + tuple.toString()); 42 | collector.emit(tuple); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/operation/OneTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import org.testng.annotations.Test; 22 | import storm.trident.tuple.TridentTuple; 23 | 24 | import static org.fest.assertions.api.Assertions.assertThat; 25 | import static org.mockito.Mockito.mock; 26 | 27 | public class OneTest { 28 | private static final int ANY_INT_ONE = 1; 29 | private static final int ANY_INT_TWO = 1; 30 | private static final One one = new One(); 31 | 32 | @Test 33 | public void testInit() throws Exception { 34 | TridentTuple tuple = mock(TridentTuple.class); 35 | assertThat(one.init(tuple)).isEqualTo(1); 36 | } 37 | 38 | @Test 39 | public void testCombine() throws Exception { 40 | assertThat(one.combine(ANY_INT_ONE, ANY_INT_TWO)).isEqualTo(1); 41 | } 42 | 43 | @Test 44 | public void testZero() throws Exception { 45 | assertThat(one.zero()).isEqualTo(1); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/util/MockTupleHelpers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.Constants; 22 | import backtype.storm.tuple.Tuple; 23 | 24 | import static org.mockito.Mockito.*; 25 | 26 | public final class MockTupleHelpers { 27 | public static final String ANY_COMPONENT_ID = "any_component_id"; 28 | public static final String ANY_STREAM_ID = "any_stream_id"; 29 | 30 | private MockTupleHelpers() { 31 | } 32 | 33 | public static Tuple mockTickTuple() { 34 | return mockTuple(Constants.SYSTEM_COMPONENT_ID, Constants.SYSTEM_TICK_STREAM_ID); 35 | } 36 | 37 | public static Tuple mockAnyTuple() { 38 | return mockTuple(ANY_COMPONENT_ID, ANY_STREAM_ID); 39 | } 40 | 41 | public static Tuple mockTuple(String componentId, String streamId) { 42 | Tuple tuple = mock(Tuple.class); 43 | when(tuple.getSourceComponent()).thenReturn(componentId); 44 | when(tuple.getSourceStreamId()).thenReturn(streamId); 45 | return tuple; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/tools/FileReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools; 20 | 21 | import storm.benchmark.util.FileUtils; 22 | 23 | import java.io.Serializable; 24 | import java.util.List; 25 | 26 | public class FileReader implements Serializable { 27 | 28 | private static final long serialVersionUID = -7012334600647556267L; 29 | 30 | public final String file; 31 | private final List contents; 32 | private int index = 0; 33 | private int limit = 0; 34 | 35 | public FileReader(String file) { 36 | this.file = file; 37 | if (this.file != null) { 38 | this.contents = FileUtils.readLines(this.getClass().getResourceAsStream(this.file)); 39 | this.limit = contents.size(); 40 | } else { 41 | throw new IllegalArgumentException("file name cannot be null"); 42 | } 43 | } 44 | 45 | public String nextLine() { 46 | if (index >= limit) { 47 | index = 0; 48 | } 49 | String line = contents.get(index); 50 | index++; 51 | return line; 52 | } 53 | 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/tools/producer/kafka/PageViewKafkaProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools.producer.kafka; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import storm.benchmark.tools.PageViewGenerator; 24 | 25 | public class PageViewKafkaProducer extends KafkaProducer { 26 | 27 | @Override 28 | public StormTopology getTopology(Config config) { 29 | spout = new PageViewSpout(); 30 | return super.getTopology(config); 31 | } 32 | 33 | static class PageViewSpout extends KafkaProducerSpout { 34 | 35 | private static final long serialVersionUID = 1772211150101656352L; 36 | private PageViewGenerator generator; 37 | 38 | public PageViewSpout() { 39 | this.generator = new PageViewGenerator(); 40 | } 41 | 42 | public PageViewSpout(PageViewGenerator generator) { 43 | this.generator = generator; 44 | } 45 | 46 | @Override 47 | public void nextTuple() { 48 | nextMessage(generator.getNextClickEvent()); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/RollingSortTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.util.TestUtils; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class RollingSortTest { 31 | @Test 32 | public void componentParallelismCouldBeSetThroughConfig() { 33 | StormBenchmark benchmark = new RollingSort(); 34 | Config config = new Config(); 35 | config.put(RollingSort.SPOUT_NUM, 4); 36 | config.put(RollingSort.SORT_BOLT_NUM, 5); 37 | 38 | StormTopology topology = benchmark.getTopology(config); 39 | assertThat(topology).isNotNull(); 40 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, RollingSort.SPOUT_ID), 4); 41 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, RollingSort.SORT_BOLT_ID), 5); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/operation/WordSplitTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import org.testng.annotations.DataProvider; 22 | import org.testng.annotations.Test; 23 | 24 | import static org.junit.Assert.assertEquals; 25 | 26 | public class WordSplitTest { 27 | 28 | final String[] ANY_WORDS = { "foo", "bar" }; 29 | final String sep = System.getProperty("line.separator"); 30 | @Test(dataProvider = "getSentenceWithWhiteSpace") 31 | public void sentenceShouldBeSplittedWithWhitespace(String sentence) { 32 | String[] rets = WordSplit.splitSentence(sentence); 33 | assertEquals(ANY_WORDS.length, rets.length); 34 | for (int i = 0; i < rets.length; i++) { 35 | assertEquals(ANY_WORDS[i], rets[i]); 36 | } 37 | } 38 | 39 | @DataProvider 40 | private Object[][] getSentenceWithWhiteSpace() { 41 | return new String[][] { 42 | {ANY_WORDS[0] + " " + ANY_WORDS[1]}, 43 | {ANY_WORDS[0] + sep + ANY_WORDS[1]}, 44 | {ANY_WORDS[0] + " " + sep + " " + ANY_WORDS[1]} 45 | }; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/bolt/ConstBolt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.task.TopologyContext; 22 | import backtype.storm.topology.BasicOutputCollector; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.topology.base.BaseBasicBolt; 25 | import backtype.storm.tuple.Fields; 26 | import backtype.storm.tuple.Tuple; 27 | import backtype.storm.tuple.Values; 28 | 29 | import java.util.Map; 30 | 31 | public class ConstBolt extends BaseBasicBolt { 32 | private static final long serialVersionUID = -5313598399155365865L; 33 | public static final String FIELDS = "message"; 34 | 35 | public ConstBolt() { 36 | } 37 | 38 | @Override 39 | public void prepare(Map conf, TopologyContext context) { 40 | } 41 | 42 | @Override 43 | public void execute(Tuple tuple, BasicOutputCollector collector) { 44 | collector.emit(new Values(tuple.getValue(0))); 45 | } 46 | 47 | @Override 48 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 49 | declarer.declare(new Fields(FIELDS)); 50 | } 51 | } -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/reducer/LongSummerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.reducer; 20 | 21 | import org.testng.annotations.DataProvider; 22 | import org.testng.annotations.Test; 23 | 24 | import static org.fest.assertions.api.Assertions.assertThat; 25 | 26 | public class LongSummerTest { 27 | 28 | private static final LongSummer summer = new LongSummer(); 29 | private static final long NON_ZERO = 1L; 30 | 31 | @Test(dataProvider = "getLong") 32 | public void testReduce(long v1, long v2, long sum) { 33 | assertThat(summer.reduce(v1, v2)).isEqualTo(sum); 34 | } 35 | 36 | @DataProvider 37 | private Object[][] getLong() { 38 | return new Object[][] { 39 | { 1L, 1L, 2L }, 40 | { 3L, 5L, 8L }, 41 | { 13L, 21L, 34L } 42 | }; 43 | } 44 | 45 | @Test 46 | public void testZero() { 47 | assertThat(summer.zero()).isEqualTo(0L); 48 | } 49 | 50 | @Test 51 | public void testIsZero() { 52 | assertThat(summer.isZero(0L)).isTrue(); 53 | assertThat(summer.isZero(NON_ZERO)).isFalse(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/operation/WordSplit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import backtype.storm.tuple.Values; 22 | import org.apache.log4j.Logger; 23 | import storm.trident.operation.BaseFunction; 24 | import storm.trident.operation.TridentCollector; 25 | import storm.trident.tuple.TridentTuple; 26 | 27 | public class WordSplit extends BaseFunction { 28 | 29 | private static final Logger LOG = Logger.getLogger(WordSplit.class); 30 | private static final long serialVersionUID = -8605358179216330897L; 31 | 32 | public static String[] splitSentence(String sentence) { 33 | if (sentence != null) { 34 | return sentence.split("\\s+"); 35 | } 36 | return null; 37 | } 38 | 39 | @Override 40 | public void execute(TridentTuple tuple, TridentCollector collector) { 41 | String sentence = tuple.getString(0); 42 | LOG.debug("receive sentence: '" + sentence + "'"); 43 | if (sentence != null) { 44 | for (String word : splitSentence(sentence)) { 45 | collector.emit(new Values(word)); 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/tools/producer/kafka/FileReadKafkaProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools.producer.kafka; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import storm.benchmark.tools.FileReader; 24 | 25 | public class FileReadKafkaProducer extends KafkaProducer { 26 | 27 | public static final String FILE = "/resources/A_Tale_of_Two_City.txt"; 28 | 29 | @Override 30 | public StormTopology getTopology(Config config) { 31 | spout = new FileReadSpout(FILE); 32 | return super.getTopology(config); 33 | } 34 | 35 | static class FileReadSpout extends KafkaProducerSpout { 36 | 37 | private static final long serialVersionUID = -7503987913879480348L; 38 | private final FileReader reader; 39 | 40 | public FileReadSpout(String file) { 41 | this.reader = new FileReader(file); 42 | } 43 | 44 | public FileReadSpout(FileReader reader) { 45 | this.reader = reader; 46 | } 47 | 48 | @Override 49 | public void nextTuple() { 50 | nextMessage(reader.nextLine()); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/operation/Distinct.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import org.apache.log4j.Logger; 22 | import storm.trident.operation.CombinerAggregator; 23 | import storm.trident.tuple.TridentTuple; 24 | 25 | import java.util.HashSet; 26 | import java.util.Set; 27 | 28 | public class Distinct implements CombinerAggregator> { 29 | 30 | private static final Logger LOG = Logger.getLogger(Distinct.class); 31 | private static final long serialVersionUID = 7592229830682953885L; 32 | 33 | @Override 34 | public Set init(TridentTuple tuple) { 35 | LOG.debug("get tuple: " + tuple); 36 | Set singleton = new HashSet(); 37 | singleton.add(tuple.getInteger(1)); 38 | return singleton; 39 | } 40 | 41 | @Override 42 | public Set combine(Set val1, Set val2) { 43 | Set union = new HashSet(); 44 | union.addAll(val1); 45 | union.addAll(val2); 46 | return union; 47 | } 48 | 49 | @Override 50 | public Set zero() { 51 | return new HashSet(); 52 | } 53 | 54 | 55 | } -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/GrepTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.util.TestUtils; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class GrepTest { 31 | 32 | @Test 33 | public void componentParallelismCouldBeSetThroughConfig() { 34 | StormBenchmark benchmark = new Grep(); 35 | Config config = new Config(); 36 | config.put(Grep.SPOUT_NUM, 3); 37 | config.put(Grep.FM_NUM, 4); 38 | config.put(Grep.CM_NUM, 5); 39 | StormTopology topology = benchmark.getTopology(config); 40 | assertThat(topology).isNotNull(); 41 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, Grep.SPOUT_ID), 3); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, Grep.FM_ID), 4); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, Grep.CM_ID), 5); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/SOLTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.util.TestUtils; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class SOLTest { 31 | 32 | @Test 33 | public void componentParallelismCouldBeSetThroughConfig() { 34 | StormBenchmark benchmark = new SOL(); 35 | Config config = new Config(); 36 | config.put(SOL.SPOUT_NUM, 4); 37 | config.put(SOL.BOLT_NUM, 3); 38 | config.put(SOL.TOPOLOGY_LEVEL, 3); 39 | StormTopology topology = benchmark.getTopology(config); 40 | assertThat(topology).isNotNull(); 41 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, SOL.SPOUT_ID), 4); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, SOL.BOLT_ID + "1"), 3); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, SOL.BOLT_ID + "2"), 3); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/bolt/FilterBolt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.OutputFieldsDeclarer; 23 | import backtype.storm.topology.base.BaseBasicBolt; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Tuple; 26 | import backtype.storm.tuple.Values; 27 | 28 | public class FilterBolt extends BaseBasicBolt { 29 | private static final long serialVersionUID = -4957635695743420459L; 30 | public static final String FIELDS = "filtered"; 31 | private final T toFilter; 32 | 33 | public FilterBolt(T toFilter) { 34 | this.toFilter = toFilter; 35 | } 36 | 37 | @Override 38 | public void execute(Tuple input, BasicOutputCollector collector) { 39 | if (!filter(input, toFilter)) { 40 | collector.emit(new Values(input.getValue(1))); 41 | } 42 | } 43 | 44 | @Override 45 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 46 | declarer.declare(new Fields(FIELDS)); 47 | } 48 | 49 | public static boolean filter(Tuple input, T toFilter) { 50 | return toFilter.equals(input.getValue(0)); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/DataCleanTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.util.TestUtils; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class DataCleanTest { 31 | 32 | @Test 33 | public void componentParallelismCouldBeSetThroughConfig() { 34 | StormBenchmark benchmark = new DataClean(); 35 | Config config = new Config(); 36 | config.put(DataClean.SPOUT_NUM, 3); 37 | config.put(DataClean.VIEW_NUM, 4); 38 | config.put(DataClean.FILTER_NUM, 5); 39 | 40 | StormTopology topology = benchmark.getTopology(config); 41 | assertThat(topology).isNotNull(); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, DataClean.SPOUT_ID), 3); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, DataClean.VIEW_ID), 4); 44 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, DataClean.FILTER_ID), 5); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/PageViewCountTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.util.TestUtils; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class PageViewCountTest { 31 | 32 | @Test 33 | public void componentParallelismCouldBeSetThroughConfig() { 34 | StormBenchmark benchmark = new PageViewCount(); 35 | Config config = new Config(); 36 | config.put(PageViewCount.SPOUT_NUM, 3); 37 | config.put(PageViewCount.VIEW_NUM, 4); 38 | config.put(PageViewCount.COUNT_NUM, 5); 39 | 40 | StormTopology topology = benchmark.getTopology(config); 41 | assertThat(topology).isNotNull(); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.SPOUT_ID), 3); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.VIEW_ID), 4); 44 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.COUNT_ID), 5); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/RollingCountTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.util.TestUtils; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class RollingCountTest { 31 | 32 | @Test 33 | public void componentParallelismCouldBeSetThroughConfig() { 34 | StormBenchmark benchmark = new RollingCount(); 35 | Config config = new Config(); 36 | config.put(RollingCount.SPOUT_NUM, 4); 37 | config.put(RollingCount.SPLIT_NUM, 5); 38 | config.put(RollingCount.COUNTER_NUM, 3); 39 | 40 | StormTopology topology = benchmark.getTopology(config); 41 | assertThat(topology).isNotNull(); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, RollingCount.SPOUT_ID), 4); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, RollingCount.SPLIT_ID), 5); 44 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, RollingCount.COUNTER_ID), 3); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/UniqueVisitorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.util.TestUtils; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class UniqueVisitorTest { 31 | 32 | @Test 33 | public void componentParallelismCouldBeSetThroughConfig() { 34 | StormBenchmark benchmark = new UniqueVisitor(); 35 | Config config = new Config(); 36 | config.put(UniqueVisitor.SPOUT_NUM, 3); 37 | config.put(UniqueVisitor.VIEW_NUM, 4); 38 | config.put(UniqueVisitor.UNIQUER_NUM, 5); 39 | StormTopology topology = benchmark.getTopology(config); 40 | assertThat(topology).isNotNull(); 41 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, UniqueVisitor.SPOUT_ID), 3); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, UniqueVisitor.VIEW_ID), 4); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, UniqueVisitor.UNIQUER_ID), 5); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/common/StormBenchmark.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks.common; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import com.google.common.collect.Sets; 24 | import org.apache.log4j.Logger; 25 | import storm.benchmark.api.IBenchmark; 26 | import storm.benchmark.metrics.BasicMetricsCollector; 27 | import storm.benchmark.metrics.IMetricsCollector; 28 | 29 | import java.util.Set; 30 | 31 | import static storm.benchmark.metrics.IMetricsCollector.MetricsItem; 32 | 33 | public abstract class StormBenchmark implements IBenchmark { 34 | 35 | private static final Logger LOG = Logger.getLogger(StormBenchmark.class); 36 | public static final String DEFAULT_TOPOLOGY_NAME = "benchmark"; 37 | 38 | @Override 39 | public IMetricsCollector getMetricsCollector(Config config, StormTopology topology) { 40 | 41 | Set items = Sets.newHashSet( 42 | MetricsItem.SUPERVISOR_STATS, 43 | MetricsItem.TOPOLOGY_STATS, 44 | MetricsItem.THROUGHPUT, 45 | MetricsItem.SPOUT_THROUGHPUT, 46 | MetricsItem.SPOUT_LATENCY 47 | ); 48 | return new BasicMetricsCollector(config, topology, items); 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/operation/ExpandTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import backtype.storm.tuple.Values; 22 | import com.google.common.collect.Lists; 23 | import com.google.common.collect.Sets; 24 | import org.testng.annotations.DataProvider; 25 | import org.testng.annotations.Test; 26 | import storm.trident.operation.TridentCollector; 27 | import storm.trident.tuple.TridentTuple; 28 | 29 | import static org.mockito.Matchers.any; 30 | import static org.mockito.Mockito.*; 31 | 32 | public class ExpandTest { 33 | 34 | 35 | @Test (dataProvider = "getIterable") 36 | public void testExpand(Iterable iterable, int size) { 37 | final TridentTuple tuple = mock(TridentTuple.class); 38 | final TridentCollector collector = mock(TridentCollector.class); 39 | when(tuple.getValue(0)).thenReturn(iterable); 40 | Expand expand = new Expand(); 41 | expand.execute(tuple, collector); 42 | 43 | verify(tuple, times(1)).getValue(0); 44 | verify(collector, times(size)).emit(any(Values.class)); 45 | } 46 | 47 | @DataProvider 48 | private Object[][] getIterable() { 49 | return new Object[][] { 50 | { Lists.newArrayList(1, 2, 3), 3}, 51 | { Sets.newHashSet(1, 2, 3), 3} 52 | }; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/FileReadWordCountTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.common.StormBenchmark; 26 | import storm.benchmark.benchmarks.common.WordCount; 27 | import storm.benchmark.util.TestUtils; 28 | 29 | import static org.fest.assertions.api.Assertions.assertThat; 30 | 31 | public class FileReadWordCountTest { 32 | 33 | @Test 34 | public void componentParallelismCouldBeSetThroughConfig() { 35 | StormBenchmark benchmark = new FileReadWordCount(); 36 | Config config = new Config(); 37 | config.put(WordCount.SPOUT_NUM, 3); 38 | config.put(WordCount.SPLIT_NUM, 4); 39 | config.put(WordCount.COUNT_NUM, 5); 40 | StormTopology topology = benchmark.getTopology(config); 41 | assertThat(topology).isNotNull(); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, WordCount.SPOUT_ID), 3); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, WordCount.SPLIT_ID), 4); 44 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, WordCount.COUNT_ID), 5); 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/bolt/ConstBoltTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.OutputFieldsDeclarer; 23 | import backtype.storm.tuple.Fields; 24 | import backtype.storm.tuple.Tuple; 25 | import backtype.storm.tuple.Values; 26 | import org.testng.annotations.BeforeMethod; 27 | import org.testng.annotations.Test; 28 | import storm.benchmark.util.MockTupleHelpers; 29 | 30 | import static org.mockito.Mockito.*; 31 | 32 | public class ConstBoltTest { 33 | 34 | private ConstBolt bolt; 35 | private Tuple tuple; 36 | private BasicOutputCollector collector; 37 | private OutputFieldsDeclarer declarer; 38 | 39 | @BeforeMethod 40 | public void setUp() { 41 | bolt = new ConstBolt(); 42 | tuple = MockTupleHelpers.mockAnyTuple(); 43 | collector = mock(BasicOutputCollector.class); 44 | declarer = mock(OutputFieldsDeclarer.class); 45 | } 46 | 47 | @Test 48 | public void shouldDeclareOutputFields() { 49 | bolt.declareOutputFields(declarer); 50 | 51 | verify(declarer, times(1)).declare(any(Fields.class)); 52 | } 53 | 54 | @Test 55 | public void shouldEmitFirstFieldOfTuple() { 56 | bolt.execute(tuple, collector); 57 | 58 | verify(tuple, times(1)).getValue(0); 59 | verify(collector, times(1)).emit(any(Values.class)); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/reducer/SetReducerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.reducer; 20 | 21 | import com.google.common.collect.Sets; 22 | import org.testng.annotations.DataProvider; 23 | import org.testng.annotations.Test; 24 | 25 | import java.util.HashSet; 26 | import java.util.Set; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class SetReducerTest { 31 | 32 | private static final SetReducer reducer = new SetReducer(); 33 | 34 | @Test 35 | public void zeroShouldReturnEmptySet() { 36 | assertThat(reducer.zero()).isEmpty(); 37 | } 38 | 39 | @Test 40 | public void isZeroShouldReturnTrueOnEmptySet() { 41 | assertThat(reducer.isZero(new HashSet())).isTrue(); 42 | } 43 | 44 | @Test (dataProvider = "getSets") 45 | public void reduceShouldReturnUnionOfTwoSets(Set set1, Set set2, Set union) { 46 | assertThat(reducer.reduce(set1, set2)).isEqualTo(union); 47 | } 48 | 49 | @DataProvider 50 | private Object[][] getSets() { 51 | return new Object[][] { 52 | { Sets.newHashSet(), Sets.newHashSet(1, 2, 3), Sets.newHashSet(1, 2, 3)}, 53 | { Sets.newHashSet(1, 2, 3), Sets.newHashSet(), Sets.newHashSet(1, 2, 3)}, 54 | { Sets.newHashSet(1, 2), Sets.newHashSet(3, 4), Sets.newHashSet(1, 2, 3, 4)}, 55 | { Sets.newHashSet(1, 2, 3), Sets.newHashSet(2, 3, 4), Sets.newHashSet(1, 2, 3, 4)} 56 | }; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/util/MetricsUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.generated.ClusterSummary; 22 | import backtype.storm.generated.TopologySummary; 23 | import com.google.common.collect.Lists; 24 | import com.google.common.collect.Maps; 25 | import org.testng.annotations.Test; 26 | 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | import static org.fest.assertions.api.Assertions.assertThat; 31 | import static org.mockito.Mockito.mock; 32 | import static org.mockito.Mockito.when; 33 | 34 | public class MetricsUtilsTest { 35 | 36 | @Test 37 | public void testAddLatency() { 38 | Map> stats = Maps.newHashMap(); 39 | String id = "spout"; 40 | double lat = 0.01; 41 | MetricsUtils.addLatency(stats, id, lat); 42 | assertThat(stats).containsKey(id); 43 | assertThat(stats.get(id)).isNotNull().contains(lat); 44 | } 45 | 46 | @Test 47 | public void testGetTopologySummary() { 48 | ClusterSummary cs = mock(ClusterSummary.class); 49 | TopologySummary ts = mock(TopologySummary.class); 50 | String tsName = "benchmarks"; 51 | String fakeName = "fake"; 52 | 53 | when(cs.get_topologies()).thenReturn(Lists.newArrayList(ts)); 54 | when(ts.get_name()).thenReturn(tsName); 55 | 56 | assertThat(MetricsUtils.getTopologySummary(cs, tsName)).isEqualTo(ts); 57 | assertThat(MetricsUtils.getTopologySummary(cs, fakeName)).isNull(); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/tools/producer/kafka/KafkaProducerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools.producer.kafka; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.utils.Utils; 24 | import org.testng.annotations.DataProvider; 25 | import org.testng.annotations.Test; 26 | import storm.benchmark.tools.producer.kafka.FileReadKafkaProducer; 27 | import storm.benchmark.tools.producer.kafka.PageViewKafkaProducer; 28 | import storm.benchmark.tools.producer.kafka.KafkaProducer; 29 | import storm.benchmark.util.TestUtils; 30 | 31 | import static org.fest.assertions.api.Assertions.assertThat; 32 | 33 | public class KafkaProducerTest { 34 | 35 | @Test(dataProvider = "getKafkaProducer") 36 | public void componentParallelismCouldBeSetThroughConfig(KafkaProducer producer) { 37 | Config config = new Config(); 38 | config.put(KafkaProducer.SPOUT_NUM, 4); 39 | config.put(KafkaProducer.BOLT_NUM, 3); 40 | StormTopology topology = producer.getTopology(config); 41 | assertThat(topology).isNotNull(); 42 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, KafkaProducer.SPOUT_ID), 4); 43 | TestUtils.verifyParallelism(Utils.getComponentCommon(topology, KafkaProducer.BOLT_ID), 3); 44 | } 45 | 46 | @DataProvider 47 | private Object[][] getKafkaProducer() { 48 | return new Object[][] { 49 | { new FileReadKafkaProducer() }, 50 | { new PageViewKafkaProducer() } 51 | }; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/operation/DistinctTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.operation; 20 | 21 | import com.google.common.collect.Sets; 22 | import org.testng.annotations.Test; 23 | import storm.trident.tuple.TridentTuple; 24 | 25 | import java.util.HashSet; 26 | import java.util.Set; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | import static org.mockito.Mockito.mock; 30 | import static org.mockito.Mockito.when; 31 | 32 | public class DistinctTest { 33 | private static final Set ANY_INT_SET_ONE = Sets.newHashSet(1, 2, 3); 34 | private static final Set ANY_INT_SET_TWO = Sets.newHashSet(2, 3, 4); 35 | 36 | private static final Distinct distinct = new Distinct(); 37 | 38 | 39 | @Test 40 | public void testInit() throws Exception { 41 | TridentTuple tuple = mock(TridentTuple.class); 42 | when(tuple.getInteger(1)).thenReturn(1); 43 | Set init = distinct.init(tuple); 44 | assertThat(init) 45 | .isNotNull() 46 | .hasSize(1) 47 | .contains(1); 48 | } 49 | 50 | @Test 51 | public void testCombine() throws Exception { 52 | Set combined = distinct.combine(ANY_INT_SET_ONE, ANY_INT_SET_TWO); 53 | assertThat(combined) 54 | .isNotNull() 55 | .hasSize(4) 56 | .contains(1, 2, 3, 4); 57 | } 58 | 59 | @Test 60 | public void testZero() throws Exception { 61 | assertThat(distinct.zero()) 62 | .isNotNull() 63 | .isInstanceOf(HashSet.class) 64 | .isEmpty(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/util/FileUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import org.testng.annotations.AfterMethod; 22 | import org.testng.annotations.BeforeMethod; 23 | import org.testng.annotations.Test; 24 | 25 | import java.io.File; 26 | import java.io.PrintWriter; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | 30 | public class FileUtilsTest { 31 | 32 | private static final String PARENT = "./test/"; 33 | private static final String PATH = PARENT + "file.txt"; 34 | private static File dir; 35 | private static File file; 36 | private static PrintWriter writer; 37 | 38 | @BeforeMethod 39 | public void setUp() { 40 | dir = new File(PARENT); 41 | file = new File(PATH); 42 | assertThat(dir).doesNotExist(); 43 | assertThat(file).doesNotExist(); 44 | } 45 | 46 | @AfterMethod 47 | public void cleanUp() { 48 | writer.close(); 49 | file.delete(); 50 | assertThat(file).doesNotExist(); 51 | dir.delete(); 52 | assertThat(dir).doesNotExist(); 53 | } 54 | 55 | @Test 56 | public void testFileCreateWhenParentDoesNotExist() { 57 | writer = FileUtils.createFileWriter(PARENT, PATH); 58 | assertThat(writer).isNotNull(); 59 | assertThat(dir).exists(); 60 | assertThat(file).exists(); 61 | } 62 | 63 | @Test 64 | public void testFileCreateWhenParentExists() { 65 | dir.mkdirs(); 66 | assertThat(dir).exists(); 67 | writer = FileUtils.createFileWriter(PARENT, PATH); 68 | assertThat(writer).isNotNull(); 69 | assertThat(file).exists(); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/util/FileUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import java.io.*; 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | public final class FileUtils { 26 | 27 | private FileUtils() { 28 | } 29 | 30 | public static List readLines(InputStream input) { 31 | List lines = new ArrayList(); 32 | try { 33 | BufferedReader reader = new BufferedReader(new InputStreamReader(input)); 34 | try { 35 | String line; 36 | while((line = reader.readLine()) != null) { 37 | lines.add(line); 38 | } 39 | } catch (IOException e) { 40 | throw new RuntimeException("Reading file failed", e); 41 | } finally { 42 | reader.close(); 43 | } 44 | } catch (IOException e) { 45 | throw new RuntimeException("Error closing reader", e); 46 | } 47 | return lines; 48 | } 49 | 50 | public static PrintWriter createFileWriter(String parent, String name) { 51 | try { 52 | final File dir = new File(parent); 53 | if (dir.exists() || dir.mkdirs()) { 54 | final File file = new File(name); 55 | file.createNewFile(); 56 | final PrintWriter writer = new PrintWriter(new OutputStreamWriter( 57 | new FileOutputStream(file, true))); 58 | return writer; 59 | } else { 60 | throw new RuntimeException("fail to create parent directory " + parent); 61 | } 62 | } catch (IOException e) { 63 | throw new RuntimeException("No such file or directory " + name, e); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/tools/producer/kafka/FileReadKafkaProducerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools.producer.kafka; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.spout.SpoutOutputCollector; 23 | import backtype.storm.task.TopologyContext; 24 | import backtype.storm.tuple.Values; 25 | import org.testng.annotations.Test; 26 | import storm.benchmark.tools.FileReader; 27 | 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | 31 | import static org.fest.assertions.api.Assertions.assertThat; 32 | import static org.mockito.Mockito.*; 33 | import static storm.benchmark.tools.producer.kafka.FileReadKafkaProducer.FileReadSpout; 34 | 35 | public class FileReadKafkaProducerTest { 36 | 37 | private static final Map ANY_CONF = new HashMap(); 38 | 39 | @Test 40 | public void spoutShouldBeKafkaFileReadSpout() { 41 | KafkaProducer producer = new FileReadKafkaProducer(); 42 | producer.getTopology(new Config()); 43 | assertThat(producer.getSpout()).isInstanceOf(FileReadSpout.class); 44 | } 45 | 46 | @Test 47 | public void nextTupleShouldEmitNextLineOfFile() throws Exception { 48 | FileReader reader = mock(FileReader.class); 49 | String message = "line"; 50 | FileReadSpout spout = new FileReadSpout(reader); 51 | TopologyContext context = mock(TopologyContext.class); 52 | SpoutOutputCollector collector = mock(SpoutOutputCollector.class); 53 | 54 | when(reader.nextLine()).thenReturn(message); 55 | 56 | spout.open(ANY_CONF, context, collector); 57 | spout.nextTuple(); 58 | 59 | verify(collector, times(1)).emit(any(Values.class)); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/util/BenchmarkUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.utils.Utils; 23 | import org.apache.log4j.Logger; 24 | 25 | import java.util.Iterator; 26 | import java.util.Map; 27 | 28 | public final class BenchmarkUtils { 29 | private static final Logger LOG = Logger.getLogger(BenchmarkUtils.class); 30 | 31 | private BenchmarkUtils() { 32 | } 33 | 34 | public static double max(Iterable iterable) { 35 | Iterator iterator = iterable.iterator(); 36 | double max = Double.MIN_VALUE; 37 | while (iterator.hasNext()) { 38 | double d = iterator.next(); 39 | if (d > max) { 40 | max = d; 41 | } 42 | } 43 | return max; 44 | } 45 | 46 | public static double avg(Iterable iterable) { 47 | Iterator iterator = iterable.iterator(); 48 | double total = 0.0; 49 | int num = 0; 50 | while (iterator.hasNext()) { 51 | total += iterator.next(); 52 | num++; 53 | } 54 | return total / num; 55 | } 56 | 57 | public static void putIfAbsent(Map map, Object key, Object val) { 58 | if (!map.containsKey(key)) { 59 | map.put(key, val); 60 | } 61 | } 62 | 63 | public static int getInt(Map map, Object key, int def) { 64 | return Utils.getInt(Utils.get(map, key, def)); 65 | } 66 | 67 | public static boolean ifAckEnabled(Config config) { 68 | Object ackers = config.get(Config.TOPOLOGY_ACKER_EXECUTORS); 69 | if (null == ackers) { 70 | LOG.warn("acker executors are null"); 71 | return false; 72 | } 73 | return Utils.getInt(ackers) > 0; 74 | }} 75 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/bolt/PageViewBolt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.OutputFieldsDeclarer; 23 | import backtype.storm.topology.base.BaseBasicBolt; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Tuple; 26 | import backtype.storm.tuple.Values; 27 | import storm.benchmark.lib.spout.pageview.PageView; 28 | import static storm.benchmark.lib.spout.pageview.PageView.Item; 29 | 30 | /** 31 | * the incoming tuple has a single pageview field consisting 32 | * of URL, (http) STATUS, ZIP and USER. 33 | * the outgoing tuple is composed of two fields, where each field 34 | * could be one of the above, the pageview itself or a count ONE. 35 | * This is flexible enough to feed downstream bolts for such use cases as 36 | * PageViewCount(URL, ONE) and UniqueVisitorCount(URL, USER) 37 | * 38 | */ 39 | 40 | public class PageViewBolt extends BaseBasicBolt { 41 | 42 | private static final long serialVersionUID = -523726932372993856L; 43 | public final Item field1; 44 | public final Item field2; 45 | 46 | public PageViewBolt(Item field1, Item field2) { 47 | this.field1 = field1; 48 | this.field2 = field2; 49 | } 50 | 51 | @Override 52 | public void execute(Tuple input, BasicOutputCollector collector) { 53 | PageView view = PageView.fromString(input.getString(0)); 54 | collector.emit(new Values(view.getValue(field1), view.getValue(field2))); 55 | } 56 | 57 | @Override 58 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 59 | declarer.declare(new Fields(field1.toString(), field2.toString())); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/spout/pageview/PageViewSpout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout.pageview; 20 | 21 | import backtype.storm.spout.SpoutOutputCollector; 22 | import backtype.storm.task.TopologyContext; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.topology.base.BaseRichSpout; 25 | import backtype.storm.tuple.Fields; 26 | import backtype.storm.tuple.Values; 27 | import storm.benchmark.tools.PageViewGenerator; 28 | 29 | import java.util.Map; 30 | 31 | public class PageViewSpout extends BaseRichSpout { 32 | 33 | public static final String FIELDS = "page_view"; 34 | 35 | private SpoutOutputCollector collector; 36 | private final PageViewGenerator generator; 37 | private final boolean ackEnabled; 38 | private long count = 0; 39 | 40 | public PageViewSpout(boolean ackEnabled) { 41 | this.ackEnabled = ackEnabled; 42 | this.generator = new PageViewGenerator(); 43 | } 44 | 45 | public PageViewSpout(boolean ackEnabled, PageViewGenerator generator) { 46 | this.ackEnabled = ackEnabled; 47 | this.generator = generator; 48 | } 49 | 50 | @Override 51 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 52 | declarer.declare(new Fields(FIELDS)); 53 | } 54 | 55 | @Override 56 | public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { 57 | this.collector = collector; 58 | } 59 | 60 | @Override 61 | public void nextTuple() { 62 | if (ackEnabled) { 63 | collector.emit(new Values(generator.getNextClickEvent()), count); 64 | count++; 65 | } else { 66 | collector.emit(new Values(generator.getNextClickEvent())); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /bin/stormbench: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import subprocess as sub 19 | import sys 20 | 21 | STORM_CMD = "" 22 | CONF_FILE = "" 23 | JAR_FILE = "" 24 | BENCHMARK = "" 25 | CONFIG_LIST = [] 26 | 27 | def exec_benchmark(args_list): 28 | run_topology(args_list) 29 | 30 | def run_topology(args_list): 31 | global CONF_FILE, MAIN_CLASS 32 | conf_parts = os.path.realpath(CONF_FILE).split("/") 33 | conf_file = conf_parts[-1] 34 | conf_dir = "/".join(os.path.realpath(CONF_FILE).split("/")[:-1]) 35 | os.putenv("STORM_CONF_DIR", conf_dir) 36 | run_args = [STORM_CMD, "jar", JAR_FILE, "--config", conf_file] + CONFIG_LIST + args_list 37 | sub.call(run_args) 38 | 39 | 40 | def parse_config_opts(args): 41 | curr = args[:] 42 | curr.reverse() 43 | args_list = [] 44 | 45 | while len(curr) > 0: 46 | token = curr.pop() 47 | if token == "-storm": 48 | global STORM_CMD 49 | STORM_CMD = curr.pop() 50 | elif token == "-jar": 51 | global JAR_FILE 52 | JAR_FILE = curr.pop() 53 | elif token == "-conf": 54 | global CONF_FILE 55 | CONF_FILE = curr.pop() 56 | elif token == "-c": 57 | global CONFIG_LIST 58 | CONFIG_LIST.append(token) 59 | CONFIG_LIST.append(curr.pop()) 60 | else: 61 | args_list.append(token) 62 | 63 | return args_list 64 | 65 | def print_usage(): 66 | print("Usage: stormbench -storm -jar -conf -c ") 67 | 68 | def main(): 69 | if len(sys.argv) <= 1: 70 | print_usage() 71 | sys.exit(-1) 72 | args_list = parse_config_opts(sys.argv[1:]) 73 | exec_benchmark(args_list) 74 | 75 | if __name__ == "__main__": 76 | main() 77 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/tools/producer/kafka/PageViewKafkaProducerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools.producer.kafka; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.spout.SpoutOutputCollector; 23 | import backtype.storm.task.TopologyContext; 24 | import backtype.storm.tuple.Values; 25 | import org.testng.annotations.Test; 26 | import storm.benchmark.tools.PageViewGenerator; 27 | 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | 31 | import static org.fest.assertions.api.Assertions.assertThat; 32 | import static org.mockito.Matchers.any; 33 | import static org.mockito.Mockito.*; 34 | import static storm.benchmark.tools.producer.kafka.PageViewKafkaProducer.PageViewSpout; 35 | 36 | public class PageViewKafkaProducerTest { 37 | private static final Map ANY_CONF = new HashMap(); 38 | private static final String NEXT_CLICK_EVENT = "next click event"; 39 | 40 | @Test 41 | public void spoutShouldBeKafkaPageViewSpout() { 42 | KafkaProducer producer = new PageViewKafkaProducer(); 43 | producer.getTopology(new Config()); 44 | assertThat(producer.getSpout()).isInstanceOf(PageViewSpout.class); 45 | } 46 | 47 | 48 | @Test 49 | public void nextTupleShouldEmitNextClickEvent() throws Exception { 50 | PageViewGenerator generator = mock(PageViewGenerator.class); 51 | PageViewSpout spout = new PageViewSpout(generator); 52 | TopologyContext context = mock(TopologyContext.class); 53 | SpoutOutputCollector collector = mock(SpoutOutputCollector.class); 54 | 55 | when(generator.getNextClickEvent()).thenReturn(NEXT_CLICK_EVENT); 56 | 57 | spout.open(ANY_CONF, context, collector); 58 | spout.nextTuple(); 59 | 60 | verify(collector, times(1)).emit(any(Values.class)); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/spout/TridentFileReadSpoutTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout; 20 | 21 | import backtype.storm.tuple.Values; 22 | import org.testng.annotations.BeforeMethod; 23 | import org.testng.annotations.Test; 24 | import storm.benchmark.tools.FileReader; 25 | import storm.trident.operation.TridentCollector; 26 | 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | import static org.fest.assertions.api.Assertions.assertThat; 31 | import static org.mockito.Mockito.*; 32 | 33 | public class TridentFileReadSpoutTest { 34 | 35 | private static final String NEXT_LINE = "next line"; 36 | private FileReader reader; 37 | private TridentFileReadSpout spout; 38 | private TridentCollector collector; 39 | 40 | @BeforeMethod 41 | public void setUp() { 42 | reader = mock(FileReader.class); 43 | spout = new TridentFileReadSpout(10, reader); 44 | collector = mock(TridentCollector.class); 45 | when(reader.nextLine()).thenReturn(NEXT_LINE); 46 | } 47 | 48 | @Test 49 | public void testEmitBatch() { 50 | spout.emitBatch(0L, collector); 51 | spout.emitBatch(1L, collector); 52 | 53 | verify(collector, times(20)).emit(any(Values.class)); 54 | 55 | Map> batches = spout.getBatches(); 56 | assertThat(batches) 57 | .hasSize(2) 58 | .containsKey(0L) 59 | .containsKey(1L); 60 | assertThat(batches.get(0L)).hasSize(10); 61 | assertThat(batches.get(1L)).hasSize(10); 62 | } 63 | 64 | @Test 65 | public void ackShouldRemoveBatch() { 66 | spout.emitBatch(0L, collector); 67 | Map> batches = spout.getBatches(); 68 | assertThat(batches).containsKey(0L); 69 | 70 | spout.ack(0L); 71 | assertThat(batches).isEmpty(); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/tools/RunnerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools; 20 | 21 | 22 | import org.testng.annotations.DataProvider; 23 | import org.testng.annotations.Test; 24 | import storm.benchmark.api.IApplication; 25 | 26 | import static org.junit.Assert.assertTrue; 27 | 28 | 29 | public class RunnerTest { 30 | 31 | @Test(dataProvider = "getValidNames") 32 | public void getBenchmarkFromValidName(String validName) throws Exception { 33 | assertTrue(Runner.getApplicationFromName(validName) instanceof IApplication); 34 | } 35 | 36 | @Test(dataProvider = "getInValidNames", expectedExceptions = ClassNotFoundException.class) 37 | public void throwsExceptionFromInvalidName(String invalidName) 38 | throws InstantiationException, IllegalAccessException, ClassNotFoundException { 39 | Runner.getApplicationFromName(invalidName); 40 | } 41 | 42 | @DataProvider 43 | private Object[][] getValidNames() { 44 | return new String[][]{ 45 | {"storm.benchmark.benchmarks.FileReadWordCount"}, 46 | {"storm.benchmark.benchmarks.SOL"}, 47 | {"storm.benchmark.benchmarks.Grep"}, 48 | {"storm.benchmark.benchmarks.PageViewCount"}, 49 | {"storm.benchmark.benchmarks.UniqueVisitor"}, 50 | {"storm.benchmark.benchmarks.KafkaWordCount"}, 51 | {"storm.benchmark.benchmarks.DRPC"}, 52 | {"storm.benchmark.benchmarks.RollingCount"}, 53 | {"storm.benchmark.benchmarks.SOL"}, 54 | {"storm.benchmark.benchmarks.TridentWordCount"}, 55 | {"storm.benchmark.tools.producer.kafka.FileReadKafkaProducer"}, 56 | {"storm.benchmark.tools.producer.kafka.PageViewKafkaProducer"} 57 | }; 58 | } 59 | 60 | @DataProvider 61 | private Object[][] getInValidNames() { 62 | return new String[][]{{"foo"}, {"bar"}}; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/bolt/FilterBoltTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.OutputFieldsDeclarer; 23 | import backtype.storm.tuple.Fields; 24 | import backtype.storm.tuple.Tuple; 25 | import backtype.storm.tuple.Values; 26 | import org.testng.annotations.BeforeMethod; 27 | import org.testng.annotations.Test; 28 | import storm.benchmark.util.MockTupleHelpers; 29 | 30 | import static org.mockito.Matchers.any; 31 | import static org.mockito.Mockito.*; 32 | 33 | public class FilterBoltTest { 34 | private static final String TO_FILTER = "to_filter"; 35 | private static final String NON_FILTER = "non_filter"; 36 | private FilterBolt bolt; 37 | private Tuple tuple; 38 | private BasicOutputCollector collector; 39 | private OutputFieldsDeclarer declarer; 40 | 41 | @BeforeMethod 42 | public void setUp() { 43 | bolt = new FilterBolt(TO_FILTER); 44 | tuple = MockTupleHelpers.mockAnyTuple(); 45 | collector = mock(BasicOutputCollector.class); 46 | declarer = mock(OutputFieldsDeclarer.class); 47 | 48 | } 49 | 50 | @Test 51 | public void shouldDeclareOutputFields() { 52 | bolt.declareOutputFields(declarer); 53 | 54 | verify(declarer, times(1)).declare(any(Fields.class)); 55 | } 56 | 57 | @Test 58 | public void shouldNotEmitIfFiltered() { 59 | when(tuple.getValue(0)).thenReturn(TO_FILTER); 60 | 61 | bolt.execute(tuple, collector); 62 | 63 | verify(tuple, never()).getValue(1); 64 | verifyZeroInteractions(collector); 65 | } 66 | 67 | @Test 68 | public void shouldEmitSecondFieldIfNotFiltered() { 69 | when(tuple.getValue(0)).thenReturn(NON_FILTER); 70 | 71 | bolt.execute(tuple, collector); 72 | 73 | verify(tuple, times(1)).getValue(1); 74 | verify(collector, times(1)).emit(any(Values.class)); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/common/StormBenchmarkTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks.common; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import org.testng.annotations.DataProvider; 24 | import org.testng.annotations.Test; 25 | import storm.benchmark.benchmarks.*; 26 | import storm.benchmark.metrics.BasicMetricsCollector; 27 | import storm.benchmark.metrics.DRPCMetricsCollector; 28 | import storm.benchmark.metrics.IMetricsCollector; 29 | 30 | import static org.fest.assertions.api.Assertions.assertThat; 31 | import static org.mockito.Mockito.mock; 32 | 33 | public class StormBenchmarkTest { 34 | private static final Config config = new Config(); 35 | private static final StormTopology topology = mock(StormTopology.class); 36 | 37 | @Test(dataProvider = "getStormBenchmark") 38 | public void getMetricsCollectorShouldReturnProperCollector( 39 | StormBenchmark benchmark, Class collectorClass) { 40 | IMetricsCollector collector = benchmark.getMetricsCollector(config, topology); 41 | assertThat(collector) 42 | .isNotNull() 43 | .isInstanceOf(collectorClass); 44 | } 45 | 46 | @DataProvider 47 | private Object[][] getStormBenchmark() { 48 | return new Object[][] { 49 | { new FileReadWordCount(), BasicMetricsCollector.class }, 50 | { new SOL(), BasicMetricsCollector.class }, 51 | { new RollingCount(), BasicMetricsCollector.class }, 52 | { new Grep(), BasicMetricsCollector.class }, 53 | { new DRPC(), DRPCMetricsCollector.class }, 54 | { new TridentWordCount(), BasicMetricsCollector.class }, 55 | { new PageViewCount(), BasicMetricsCollector.class }, 56 | { new UniqueVisitor(), BasicMetricsCollector.class }, 57 | { new DataClean(), BasicMetricsCollector.class } 58 | 59 | }; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/bolt/RollingBolt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | 22 | import backtype.storm.Config; 23 | import backtype.storm.topology.BasicOutputCollector; 24 | import backtype.storm.topology.base.BaseBasicBolt; 25 | import backtype.storm.tuple.Tuple; 26 | import storm.benchmark.util.TupleHelpers; 27 | 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | 31 | public abstract class RollingBolt extends BaseBasicBolt { 32 | 33 | private static final long serialVersionUID = 5996017217250945842L; 34 | public static final int DEFAULT_NUM_WINDOW_CHUNKS = 5; 35 | public static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = DEFAULT_NUM_WINDOW_CHUNKS * 60; 36 | public static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / DEFAULT_NUM_WINDOW_CHUNKS; 37 | 38 | protected final int windowLengthInSeconds; 39 | protected final int emitFrequencyInSeconds; 40 | 41 | public RollingBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) { 42 | this.windowLengthInSeconds = windowLengthInSeconds; 43 | this.emitFrequencyInSeconds = emitFrequencyInSeconds; 44 | } 45 | 46 | protected int getWindowChunks() { 47 | return windowLengthInSeconds / emitFrequencyInSeconds; 48 | } 49 | 50 | @Override 51 | public void execute(Tuple tuple, BasicOutputCollector collector) { 52 | if (TupleHelpers.isTickTuple(tuple)) { 53 | emitCurrentWindow(collector); 54 | } else { 55 | updateCurrentWindow(tuple); 56 | } 57 | } 58 | 59 | public abstract void emitCurrentWindow(BasicOutputCollector collector); 60 | 61 | public abstract void updateCurrentWindow(Tuple tuple); 62 | 63 | @Override 64 | public Map getComponentConfiguration() { 65 | Map conf = new HashMap(); 66 | conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds); 67 | return conf; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/spout/RandomMessageSpoutTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout; 20 | 21 | import backtype.storm.spout.SpoutOutputCollector; 22 | import backtype.storm.task.TopologyContext; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Values; 26 | import org.testng.annotations.BeforeMethod; 27 | import org.testng.annotations.Test; 28 | 29 | import java.util.HashMap; 30 | import java.util.Map; 31 | 32 | import static org.mockito.Mockito.*; 33 | 34 | public class RandomMessageSpoutTest { 35 | 36 | private static final int messageSize = 100; 37 | private static final Map ANY_CONF = new HashMap(); 38 | private OutputFieldsDeclarer declarer; 39 | private TopologyContext context; 40 | private SpoutOutputCollector collector; 41 | 42 | @BeforeMethod 43 | public void setUp() { 44 | declarer = mock(OutputFieldsDeclarer.class); 45 | collector = mock(SpoutOutputCollector.class); 46 | context = mock(TopologyContext.class); 47 | 48 | } 49 | 50 | @Test 51 | public void shouldDeclareOutputFields() { 52 | RandomMessageSpout spout = new RandomMessageSpout(messageSize, false); 53 | 54 | spout.declareOutputFields(declarer); 55 | 56 | verify(declarer, times(1)).declare(any(Fields.class)); 57 | } 58 | 59 | @Test 60 | public void shouldEmitValueAndIdWhenAckEnabled() { 61 | RandomMessageSpout spout = new RandomMessageSpout(messageSize, true); 62 | 63 | spout.open(ANY_CONF, context, collector); 64 | spout.nextTuple(); 65 | 66 | verify(collector, times(1)).emit(any(Values.class), anyInt()); 67 | } 68 | 69 | @Test 70 | public void shouldEmitValueOnlyWhenAckDisabled() { 71 | RandomMessageSpout spout = new RandomMessageSpout(messageSize, false); 72 | 73 | spout.open(ANY_CONF, context, collector); 74 | spout.nextTuple(); 75 | 76 | verify(collector, times(1)).emit(any(Values.class)); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/spout/FileReadSpout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout; 20 | 21 | import backtype.storm.spout.SpoutOutputCollector; 22 | import backtype.storm.task.TopologyContext; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.topology.base.BaseRichSpout; 25 | import backtype.storm.tuple.Fields; 26 | import backtype.storm.tuple.Values; 27 | import org.apache.log4j.Logger; 28 | import storm.benchmark.tools.FileReader; 29 | 30 | import java.util.Map; 31 | 32 | public class FileReadSpout extends BaseRichSpout { 33 | private static final Logger LOG = Logger.getLogger(FileReadSpout.class); 34 | private static final long serialVersionUID = -2582705611472467172L; 35 | 36 | public static final String DEFAULT_FILE = "/resources/A_Tale_of_Two_City.txt"; 37 | public static final boolean DEFAULT_ACK = false; 38 | public static final String FIELDS = "sentence"; 39 | 40 | public final boolean ackEnabled; 41 | public final FileReader reader; 42 | private SpoutOutputCollector collector; 43 | 44 | private long count = 0; 45 | 46 | 47 | public FileReadSpout() { 48 | this(DEFAULT_ACK, DEFAULT_FILE); 49 | } 50 | 51 | 52 | public FileReadSpout(boolean ackEnabled) { 53 | this(ackEnabled, DEFAULT_FILE); 54 | } 55 | 56 | public FileReadSpout(boolean ackEnabled, String file) { 57 | this(ackEnabled, new FileReader(file)); 58 | } 59 | 60 | public FileReadSpout(boolean ackEnabled, FileReader reader) { 61 | this.ackEnabled = ackEnabled; 62 | this.reader = reader; 63 | } 64 | 65 | @Override 66 | public void open(Map conf, TopologyContext context, 67 | SpoutOutputCollector collector) { 68 | this.collector = collector; 69 | } 70 | 71 | @Override 72 | public void nextTuple() { 73 | if (ackEnabled) { 74 | collector.emit(new Values(reader.nextLine()), count); 75 | count++; 76 | } else { 77 | collector.emit(new Values(reader.nextLine())); 78 | } 79 | } 80 | 81 | @Override 82 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 83 | declarer.declare(new Fields(FIELDS)); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/tools/LocalRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.LocalCluster; 23 | import backtype.storm.StormSubmitter; 24 | import backtype.storm.generated.AlreadyAliveException; 25 | import backtype.storm.generated.InvalidTopologyException; 26 | import backtype.storm.generated.StormTopology; 27 | import backtype.storm.utils.Utils; 28 | import org.apache.log4j.Logger; 29 | import storm.benchmark.api.IBenchmark; 30 | import storm.benchmark.metrics.IMetricsCollector; 31 | import storm.benchmark.metrics.MetricsCollectorConfig; 32 | import storm.benchmark.util.BenchmarkUtils; 33 | 34 | public class LocalRunner { 35 | private static final Logger LOG = Logger.getLogger(Runner.class); 36 | private static final String PACKAGE = "storm.benchmark.benchmarks"; 37 | 38 | public static void main(String[] args) throws Exception { 39 | if (null == args || args.length < 1) { 40 | throw new IllegalArgumentException("no benchmark is set"); 41 | } 42 | run(args[0]); 43 | } 44 | 45 | private static void run(String name) 46 | throws ClassNotFoundException, IllegalAccessException, 47 | InstantiationException, AlreadyAliveException, InvalidTopologyException { 48 | LOG.info("running benchmark " + name); 49 | IBenchmark benchmark = (IBenchmark) Runner.getApplicationFromName(PACKAGE + "." + name); 50 | Config config = new Config(); 51 | config.putAll(Utils.readStormConfig()); 52 | config.setDebug(true); 53 | StormTopology topology = benchmark.getTopology(config); 54 | LocalCluster localCluster = new LocalCluster(); 55 | localCluster.submitTopology(name, config, topology); 56 | final int runtime = BenchmarkUtils.getInt(config, MetricsCollectorConfig.METRICS_TOTAL_TIME, 57 | MetricsCollectorConfig.DEFAULT_TOTAL_TIME); 58 | IMetricsCollector collector = benchmark.getMetricsCollector(config, topology); 59 | collector.run(); 60 | try { 61 | Thread.sleep(runtime); 62 | } catch (InterruptedException e) { 63 | LOG.error("benchmark interrupted", e); 64 | } 65 | localCluster.shutdown(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/spout/FileReadSpoutTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout; 20 | 21 | import backtype.storm.spout.SpoutOutputCollector; 22 | import backtype.storm.task.TopologyContext; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Values; 26 | import org.testng.annotations.BeforeMethod; 27 | import org.testng.annotations.Test; 28 | import storm.benchmark.tools.FileReader; 29 | 30 | import java.util.HashMap; 31 | import java.util.Map; 32 | 33 | import static org.mockito.Matchers.any; 34 | import static org.mockito.Matchers.anyInt; 35 | import static org.mockito.Mockito.*; 36 | 37 | public class FileReadSpoutTest { 38 | private static final int messageSize = 100; 39 | private static final Map ANY_CONF = new HashMap(); 40 | private static final String NEXT_LINE = "next line"; 41 | private OutputFieldsDeclarer declarer; 42 | private TopologyContext context; 43 | private SpoutOutputCollector collector; 44 | private FileReader reader; 45 | 46 | @BeforeMethod 47 | public void setUp() { 48 | declarer = mock(OutputFieldsDeclarer.class); 49 | collector = mock(SpoutOutputCollector.class); 50 | context = mock(TopologyContext.class); 51 | reader = mock(FileReader.class); 52 | when(reader.nextLine()).thenReturn(NEXT_LINE); 53 | } 54 | 55 | @Test 56 | public void shouldDeclareOutputFields() { 57 | FileReadSpout spout = new FileReadSpout(false, reader); 58 | 59 | spout.declareOutputFields(declarer); 60 | 61 | verify(declarer, times(1)).declare(any(Fields.class)); 62 | } 63 | 64 | @Test 65 | public void shouldEmitValueAndIdWhenAckEnabled() { 66 | FileReadSpout spout = new FileReadSpout(true, reader); 67 | 68 | spout.open(ANY_CONF, context, collector); 69 | spout.nextTuple(); 70 | 71 | verify(collector, times(1)).emit(any(Values.class), anyInt()); 72 | } 73 | 74 | @Test 75 | public void shouldEmitValueOnlyWhenAckDisabled() { 76 | FileReadSpout spout = new FileReadSpout(false, reader); 77 | 78 | spout.open(ANY_CONF, context, collector); 79 | spout.nextTuple(); 80 | 81 | verify(collector, times(1)).emit(any(Values.class)); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/spout/pageview/PageViewSpoutTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout.pageview; 20 | 21 | import backtype.storm.spout.SpoutOutputCollector; 22 | import backtype.storm.task.TopologyContext; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Values; 26 | import org.testng.annotations.BeforeMethod; 27 | import org.testng.annotations.Test; 28 | import storm.benchmark.tools.PageViewGenerator; 29 | 30 | import java.util.HashMap; 31 | import java.util.Map; 32 | 33 | import static org.mockito.Matchers.any; 34 | import static org.mockito.Matchers.anyInt; 35 | import static org.mockito.Mockito.*; 36 | 37 | public class PageViewSpoutTest { 38 | private static final Map ANY_CONF = new HashMap(); 39 | private static final String NEXT_CLICK_EVENT = "next click event"; 40 | private OutputFieldsDeclarer declarer; 41 | private TopologyContext context; 42 | private SpoutOutputCollector collector; 43 | private PageViewGenerator generator; 44 | 45 | @BeforeMethod 46 | public void setUp() { 47 | declarer = mock(OutputFieldsDeclarer.class); 48 | collector = mock(SpoutOutputCollector.class); 49 | context = mock(TopologyContext.class); 50 | generator = mock(PageViewGenerator.class); 51 | when(generator.getNextClickEvent()).thenReturn(NEXT_CLICK_EVENT); 52 | } 53 | 54 | @Test 55 | public void shouldDeclareOutputFields() { 56 | PageViewSpout spout = new PageViewSpout(false, generator); 57 | 58 | spout.declareOutputFields(declarer); 59 | 60 | verify(declarer, times(1)).declare(any(Fields.class)); 61 | } 62 | 63 | @Test 64 | public void shouldEmitValueAndIdWhenAckEnabled() { 65 | PageViewSpout spout = new PageViewSpout(true, generator); 66 | 67 | spout.open(ANY_CONF, context, collector); 68 | spout.nextTuple(); 69 | 70 | verify(collector, times(1)).emit(any(Values.class), anyInt()); 71 | } 72 | 73 | @Test 74 | public void shouldEmitValueOnlyWhenAckDisabled() { 75 | PageViewSpout spout = new PageViewSpout(false, generator); 76 | 77 | spout.open(ANY_CONF, context, collector); 78 | spout.nextTuple(); 79 | 80 | verify(collector, times(1)).emit(any(Values.class)); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/spout/RandomMessageSpout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout; 20 | 21 | import backtype.storm.spout.SpoutOutputCollector; 22 | import backtype.storm.task.TopologyContext; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.topology.base.BaseRichSpout; 25 | import backtype.storm.tuple.Fields; 26 | import backtype.storm.tuple.Values; 27 | 28 | import java.util.Map; 29 | import java.util.Random; 30 | 31 | public class RandomMessageSpout extends BaseRichSpout { 32 | 33 | private static final long serialVersionUID = -4100642374496292646L; 34 | public static final String FIELDS = "message"; 35 | public static final String MESSAGE_SIZE = "message.size"; 36 | public static final int DEFAULT_MESSAGE_SIZE = 100; 37 | 38 | private final int sizeInBytes; 39 | private long messageCount = 0; 40 | private SpoutOutputCollector collector; 41 | private String [] messages = null; 42 | private final boolean ackEnabled; 43 | private Random rand = null; 44 | 45 | public RandomMessageSpout(boolean ackEnabled) { 46 | this(DEFAULT_MESSAGE_SIZE, ackEnabled); 47 | } 48 | 49 | public RandomMessageSpout(int sizeInBytes, boolean ackEnabled) { 50 | this.sizeInBytes = sizeInBytes; 51 | this.ackEnabled = ackEnabled; 52 | } 53 | 54 | public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { 55 | this.rand = new Random(); 56 | this.collector = collector; 57 | final int differentMessages = 100; 58 | this.messages = new String[differentMessages]; 59 | for(int i = 0; i < differentMessages; i++) { 60 | StringBuilder sb = new StringBuilder(sizeInBytes); 61 | for(int j = 0; j < sizeInBytes; j++) { 62 | sb.append(rand.nextInt(9)); 63 | } 64 | messages[i] = sb.toString(); 65 | } 66 | } 67 | 68 | 69 | @Override 70 | public void nextTuple() { 71 | final String message = messages[rand.nextInt(messages.length)]; 72 | if(ackEnabled) { 73 | collector.emit(new Values(message), messageCount); 74 | messageCount++; 75 | } else { 76 | collector.emit(new Values(message)); 77 | } 78 | } 79 | 80 | 81 | @Override 82 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 83 | declarer.declare(new Fields(FIELDS)); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/DataClean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.spout.SchemeAsMultiScheme; 24 | import backtype.storm.topology.IRichSpout; 25 | import backtype.storm.topology.TopologyBuilder; 26 | import backtype.storm.tuple.Fields; 27 | import storm.benchmark.lib.bolt.FilterBolt; 28 | import storm.benchmark.lib.bolt.PageViewBolt; 29 | import storm.benchmark.benchmarks.common.StormBenchmark; 30 | import storm.benchmark.util.BenchmarkUtils; 31 | import storm.benchmark.util.KafkaUtils; 32 | import storm.kafka.KafkaSpout; 33 | import storm.kafka.StringScheme; 34 | 35 | import static storm.benchmark.lib.spout.pageview.PageView.Item; 36 | 37 | public class DataClean extends StormBenchmark { 38 | public final static String SPOUT_ID = "spout"; 39 | public final static String SPOUT_NUM = "component.spout_num"; 40 | public final static String VIEW_ID = "view"; 41 | public final static String VIEW_NUM = "component.view_bolt_num"; 42 | public final static String FILTER_ID = "filter"; 43 | public final static String FILTER_NUM = "component.filter_bolt_num"; 44 | 45 | public static final int DEFAULT_SPOUT_NUM = 4; 46 | public static final int DEFAULT_PV_BOLT_NUM = 4; 47 | public static final int DEFAULT_FILTER_BOLT_NUM = 4; 48 | 49 | private IRichSpout spout; 50 | 51 | 52 | @Override 53 | public StormTopology getTopology(Config config) { 54 | final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); 55 | final int pvBoltNum = BenchmarkUtils.getInt(config, VIEW_NUM, DEFAULT_PV_BOLT_NUM); 56 | final int filterBoltNum = BenchmarkUtils.getInt(config, FILTER_NUM, DEFAULT_FILTER_BOLT_NUM); 57 | spout = new KafkaSpout(KafkaUtils.getSpoutConfig( 58 | config, new SchemeAsMultiScheme(new StringScheme()))); 59 | 60 | TopologyBuilder builder = new TopologyBuilder(); 61 | builder.setSpout(SPOUT_ID, spout, spoutNum); 62 | builder.setBolt(VIEW_ID, new PageViewBolt(Item.STATUS, Item.ALL), pvBoltNum) 63 | .localOrShuffleGrouping(SPOUT_ID); 64 | builder.setBolt(FILTER_ID, new FilterBolt(404), filterBoltNum) 65 | .fieldsGrouping(VIEW_ID, new Fields(Item.STATUS.toString())); 66 | return builder.createTopology(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/PageViewCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.spout.SchemeAsMultiScheme; 24 | import backtype.storm.topology.IRichSpout; 25 | import backtype.storm.topology.TopologyBuilder; 26 | import backtype.storm.tuple.Fields; 27 | import storm.benchmark.lib.bolt.PageViewBolt; 28 | import storm.benchmark.benchmarks.common.StormBenchmark; 29 | import storm.benchmark.benchmarks.common.WordCount; 30 | import storm.benchmark.util.BenchmarkUtils; 31 | import storm.benchmark.util.KafkaUtils; 32 | import storm.kafka.KafkaSpout; 33 | import storm.kafka.StringScheme; 34 | 35 | import static storm.benchmark.lib.spout.pageview.PageView.Item; 36 | 37 | public class PageViewCount extends StormBenchmark { 38 | public static final String SPOUT_ID = "spout"; 39 | public static final String SPOUT_NUM = "component.spout_num"; 40 | public static final String VIEW_ID = "view"; 41 | public static final String VIEW_NUM = "component.view_bolt_num"; 42 | public static final String COUNT_ID = "count"; 43 | public static final String COUNT_NUM = "component.count_bolt_num"; 44 | 45 | public static final int DEFAULT_SPOUT_NUM = 4; 46 | public static final int DEFAULT_VIEW_BOLT_NUM = 4; 47 | public static final int DEFAULT_COUNT_BOLT_NUM = 4; 48 | 49 | private IRichSpout spout; 50 | 51 | @Override 52 | public StormTopology getTopology(Config config) { 53 | 54 | final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); 55 | final int viewBoltNum = BenchmarkUtils.getInt(config, VIEW_NUM, DEFAULT_VIEW_BOLT_NUM); 56 | final int cntBoltNum = BenchmarkUtils.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM); 57 | 58 | spout = new KafkaSpout(KafkaUtils.getSpoutConfig( 59 | config, new SchemeAsMultiScheme(new StringScheme()))); 60 | 61 | TopologyBuilder builder = new TopologyBuilder(); 62 | builder.setSpout(SPOUT_ID, spout, spoutNum); 63 | builder.setBolt(VIEW_ID, new PageViewBolt(Item.URL, Item.ONE), viewBoltNum) 64 | .localOrShuffleGrouping(SPOUT_ID); 65 | builder.setBolt(COUNT_ID, new WordCount.Count(), cntBoltNum) 66 | .fieldsGrouping(VIEW_ID, new Fields(Item.URL.toString())); 67 | return builder.createTopology(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/bolt/UniqueVisitorBolt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.OutputFieldsDeclarer; 23 | import backtype.storm.tuple.Fields; 24 | import backtype.storm.tuple.Tuple; 25 | import backtype.storm.tuple.Values; 26 | import storm.benchmark.lib.reducer.SetReducer; 27 | import storm.benchmark.tools.SlidingWindow; 28 | 29 | import java.util.HashMap; 30 | import java.util.HashSet; 31 | import java.util.Map; 32 | import java.util.Set; 33 | 34 | public class UniqueVisitorBolt extends RollingBolt { 35 | 36 | private static final long serialVersionUID = -6518481724698629167L; 37 | public static final String FIELDS_URL = "url"; 38 | public static final String FIELD_UV = "unique_visitor"; 39 | 40 | private final SlidingWindow> window; 41 | private final Map> cached; 42 | 43 | public UniqueVisitorBolt() { 44 | this(DEFAULT_SLIDING_WINDOW_IN_SECONDS, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); 45 | } 46 | 47 | public UniqueVisitorBolt(int winLen, int emitFreq) { 48 | super(winLen, emitFreq); 49 | window = new SlidingWindow>(new SetReducer(), getWindowChunks()); 50 | cached = new HashMap>(); 51 | } 52 | 53 | @Override 54 | public void emitCurrentWindow(BasicOutputCollector collector) { 55 | dumpCache(); 56 | Map> urlToVisitors = window.reduceThenAdvanceWindow(); 57 | for (String url : urlToVisitors.keySet()) { 58 | collector.emit(new Values(url, urlToVisitors.get(url).size())); 59 | } 60 | } 61 | 62 | @Override 63 | public void updateCurrentWindow(Tuple tuple) { 64 | String url = tuple.getString(0); 65 | int userId = tuple.getInteger(1); 66 | Set users = cached.get(url); 67 | if (null == users) { 68 | users = new HashSet(); 69 | cached.put(url, users); 70 | } 71 | users.add(userId); 72 | } 73 | 74 | private void dumpCache() { 75 | for (String url : cached.keySet()) { 76 | window.add(url, cached.get(url)); 77 | } 78 | cached.clear(); 79 | } 80 | 81 | @Override 82 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 83 | declarer.declare(new Fields(FIELDS_URL, FIELD_UV)); 84 | } 85 | 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/benchmarks/common/WordCountTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks.common; 20 | 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.OutputFieldsDeclarer; 23 | import backtype.storm.tuple.Fields; 24 | import backtype.storm.tuple.Tuple; 25 | import backtype.storm.tuple.Values; 26 | import org.testng.annotations.Test; 27 | 28 | import static org.fest.assertions.api.Assertions.assertThat; 29 | import static org.mockito.Mockito.*; 30 | 31 | public class WordCountTest { 32 | 33 | @Test 34 | public void splitSentenceBoltShouldDeclareOutputFields() { 35 | WordCount.SplitSentence bolt = new WordCount.SplitSentence(); 36 | OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); 37 | 38 | bolt.declareOutputFields(declarer); 39 | 40 | verify(declarer, times(1)).declare(any(Fields.class)); 41 | } 42 | 43 | @Test 44 | public void splitSentenceBoltShouldEmitEveryWord() { 45 | final String sentence = "This is a sentence"; 46 | WordCount.SplitSentence bolt = new WordCount.SplitSentence(); 47 | Tuple tuple = mock(Tuple.class); 48 | BasicOutputCollector collector = mock(BasicOutputCollector.class); 49 | 50 | when(tuple.getString(0)).thenReturn(sentence); 51 | 52 | bolt.execute(tuple, collector); 53 | 54 | verify(collector, times(4)).emit(any(Values.class)); 55 | 56 | } 57 | 58 | @Test 59 | public void countBoltShouldDeclareOutputFields() { 60 | WordCount.Count bolt = new WordCount.Count(); 61 | OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); 62 | 63 | bolt.declareOutputFields(declarer); 64 | 65 | verify(declarer, times(1)).declare(any(Fields.class)); 66 | } 67 | 68 | @Test 69 | public void countBoltShouldCountAndEmitNumberOfEveryWord() { 70 | String[] words = { "word", "word", "count"}; 71 | WordCount.Count bolt = new WordCount.Count(); 72 | Tuple tuple = mock(Tuple.class); 73 | BasicOutputCollector collector = mock(BasicOutputCollector.class); 74 | 75 | for (String w : words) { 76 | when(tuple.getString(0)).thenReturn(w); 77 | bolt.execute(tuple, collector); 78 | } 79 | verify(collector, times(3)).emit(any(Values.class)); 80 | 81 | assertThat(bolt.counts.get("word")).isEqualTo(2); 82 | assertThat(bolt.counts.get("count")).isEqualTo(1); 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/bolt/RollingCountBolt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.topology.BasicOutputCollector; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Tuple; 26 | import backtype.storm.tuple.Values; 27 | import org.apache.log4j.Logger; 28 | import storm.benchmark.lib.reducer.LongSummer; 29 | import storm.benchmark.tools.SlidingWindow; 30 | 31 | import java.util.HashMap; 32 | import java.util.Map; 33 | import java.util.Map.Entry; 34 | 35 | /** 36 | * forked from RollingCountBolt in storm-starter 37 | */ 38 | 39 | public class RollingCountBolt extends RollingBolt { 40 | 41 | private static final long serialVersionUID = -903093673694769540L; 42 | private static final Logger LOG = Logger.getLogger(RollingCountBolt.class); 43 | public static final String FIELDS_OBJ = "obj"; 44 | public static final String FIELDS_CNT = "count"; 45 | 46 | private final SlidingWindow window; 47 | 48 | public RollingCountBolt() { 49 | this(DEFAULT_SLIDING_WINDOW_IN_SECONDS, DEFAULT_EMIT_FREQUENCY_IN_SECONDS); 50 | } 51 | 52 | public RollingCountBolt(int winLen, int emitFreq) { 53 | super(winLen, emitFreq); 54 | window = new SlidingWindow(new LongSummer(), getWindowChunks()); 55 | } 56 | 57 | @Override 58 | public void emitCurrentWindow(BasicOutputCollector collector) { 59 | emitCurrentWindowCounts(collector); 60 | } 61 | 62 | @Override 63 | public void updateCurrentWindow(Tuple tuple) { 64 | countObj(tuple); 65 | } 66 | 67 | private void emitCurrentWindowCounts(BasicOutputCollector collector) { 68 | Map counts = window.reduceThenAdvanceWindow(); 69 | for (Entry entry : counts.entrySet()) { 70 | Object obj = entry.getKey(); 71 | Long count = entry.getValue(); 72 | LOG.info(String.format("get %d %s in last %d seconds", count, obj, windowLengthInSeconds)); 73 | collector.emit(new Values(obj, count)); 74 | } 75 | } 76 | 77 | private void countObj(Tuple tuple) { 78 | Object obj = tuple.getValue(0); 79 | window.add(obj, (long) 1); 80 | } 81 | 82 | @Override 83 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 84 | declarer.declare(new Fields(FIELDS_OBJ, FIELDS_CNT)); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/spout/TridentFileReadSpout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | package storm.benchmark.lib.spout; 19 | 20 | import backtype.storm.task.TopologyContext; 21 | import backtype.storm.tuple.Fields; 22 | import backtype.storm.tuple.Values; 23 | import storm.benchmark.tools.FileReader; 24 | import storm.trident.operation.TridentCollector; 25 | import storm.trident.spout.IBatchSpout; 26 | 27 | import java.util.ArrayList; 28 | import java.util.HashMap; 29 | import java.util.List; 30 | import java.util.Map; 31 | 32 | 33 | public class TridentFileReadSpout implements IBatchSpout { 34 | 35 | private static final long serialVersionUID = -3538746749629409899L; 36 | private static final String DEFAULT_FILE = "/resources/A_Tale_of_Two_City.txt"; 37 | public static final String FIELDS = "sentence"; 38 | 39 | private final int maxBatchSize; 40 | private final FileReader reader; 41 | private final HashMap> batches = new HashMap>(); 42 | 43 | public TridentFileReadSpout(int maxBatchSize) { 44 | this(maxBatchSize, DEFAULT_FILE); 45 | } 46 | 47 | public TridentFileReadSpout(int maxBatchSize, String file) { 48 | this.maxBatchSize = maxBatchSize; 49 | this.reader = new FileReader(file); 50 | } 51 | 52 | public TridentFileReadSpout(int maxBatchSize, FileReader reader) { 53 | this.maxBatchSize = maxBatchSize; 54 | this.reader = reader; 55 | } 56 | 57 | @Override 58 | public void open(Map conf, TopologyContext context) { 59 | } 60 | 61 | @Override 62 | public void emitBatch(long batchId, TridentCollector collector) { 63 | List batch = batches.get(batchId); 64 | if (batch == null) { 65 | batch = new ArrayList(); 66 | for (int i = 0; i < maxBatchSize; i++) { 67 | batch.add(reader.nextLine()); 68 | } 69 | batches.put(batchId, batch); 70 | } 71 | for(String line : batch){ 72 | collector.emit(new Values(line)); 73 | } 74 | } 75 | 76 | @Override 77 | public void ack(long batchId) { 78 | batches.remove(batchId); 79 | } 80 | 81 | @Override 82 | public void close() { 83 | } 84 | 85 | @Override 86 | public Map getComponentConfiguration() { 87 | return null; 88 | } 89 | 90 | @Override 91 | public Fields getOutputFields() { 92 | return new Fields(FIELDS); 93 | } 94 | 95 | Map> getBatches() { 96 | return batches; 97 | } 98 | 99 | 100 | } -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/metrics/MetricsCollectorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.metrics; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.utils.Utils; 23 | import org.apache.log4j.Logger; 24 | import storm.benchmark.benchmarks.common.StormBenchmark; 25 | import storm.benchmark.util.BenchmarkUtils; 26 | 27 | import java.io.PrintWriter; 28 | import java.util.Map; 29 | import java.util.TreeMap; 30 | 31 | public class MetricsCollectorConfig { 32 | private static final Logger LOG = Logger.getLogger(MetricsCollectorConfig.class); 33 | 34 | public static final String CONF_FILE_FORMAT = "%s/%s_metrics_%d.yaml"; 35 | public static final String DATA_FILE_FORMAT = "%s/%s_metrics_%d.csv"; 36 | 37 | public static final String METRICS_ENABLED = "metrics.enabled"; 38 | public static final String METRICS_POLL_INTERVAL = "metrics.poll"; 39 | public static final String METRICS_TOTAL_TIME = "metrics.time"; 40 | public static final String METRICS_PATH = "metrics.path"; 41 | 42 | public static final int DEFAULT_POLL_INTERVAL = 30 * 1000; // 30 secs 43 | public static final int DEFAULT_TOTAL_TIME = 5 * 60 * 1000; // 5 mins 44 | public static final String DEFAULT_PATH = "/root/"; 45 | 46 | // storm configuration 47 | public final Config stormConfig; 48 | // storm benchmarks name 49 | public final String name; 50 | // How often should metrics be collected 51 | public final int pollInterval; 52 | // How long should the benchmark run for 53 | public final int totalTime; 54 | // metrics file path 55 | public final String path; 56 | 57 | public MetricsCollectorConfig(Config stormConfig) { 58 | this.stormConfig = stormConfig; 59 | name = (String) Utils.get( 60 | stormConfig, Config.TOPOLOGY_NAME, StormBenchmark.DEFAULT_TOPOLOGY_NAME); 61 | pollInterval = BenchmarkUtils.getInt( 62 | stormConfig, METRICS_POLL_INTERVAL, DEFAULT_POLL_INTERVAL); 63 | totalTime = BenchmarkUtils.getInt( 64 | stormConfig, METRICS_TOTAL_TIME, DEFAULT_TOTAL_TIME); 65 | path = (String) Utils.get(stormConfig, METRICS_PATH, DEFAULT_PATH); 66 | } 67 | 68 | 69 | public void writeStormConfig(PrintWriter writer) { 70 | LOG.info("writing out storm config into .yaml file"); 71 | if (writer != null) { 72 | Map sorted = new TreeMap(); 73 | sorted.putAll(stormConfig); 74 | for (Object key : sorted.keySet()) { 75 | writer.println(key + ": " + stormConfig.get(key)); 76 | } 77 | writer.flush(); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/RollingCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.topology.IRichSpout; 24 | import backtype.storm.topology.TopologyBuilder; 25 | import backtype.storm.tuple.Fields; 26 | import storm.benchmark.benchmarks.common.WordCount; 27 | import storm.benchmark.lib.bolt.RollingCountBolt; 28 | import storm.benchmark.lib.bolt.RollingBolt; 29 | import storm.benchmark.lib.spout.FileReadSpout; 30 | import storm.benchmark.benchmarks.common.StormBenchmark; 31 | import storm.benchmark.util.BenchmarkUtils; 32 | 33 | public class RollingCount extends StormBenchmark { 34 | 35 | private static final String WINDOW_LENGTH = "window.length"; 36 | private static final String EMIT_FREQ = "emit.frequency"; 37 | 38 | public static final String SPOUT_ID = "spout"; 39 | public static final String SPOUT_NUM = "component.spout_num"; 40 | public static final String SPLIT_ID = "split"; 41 | public static final String SPLIT_NUM = "component.split_bolt_num"; 42 | public static final String COUNTER_ID = "rolling_count"; 43 | public static final String COUNTER_NUM = "component.rolling_count_bolt_num"; 44 | 45 | public static final int DEFAULT_SPOUT_NUM = 4; 46 | public static final int DEFAULT_SP_BOLT_NUM = 8; 47 | public static final int DEFAULT_RC_BOLT_NUM = 8; 48 | 49 | private IRichSpout spout; 50 | 51 | @Override 52 | public StormTopology getTopology(Config config) { 53 | 54 | final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); 55 | final int spBoltNum = BenchmarkUtils.getInt(config, SPLIT_NUM, DEFAULT_SP_BOLT_NUM); 56 | final int rcBoltNum = BenchmarkUtils.getInt(config, COUNTER_NUM, DEFAULT_RC_BOLT_NUM); 57 | final int windowLength = BenchmarkUtils.getInt(config, WINDOW_LENGTH, 58 | RollingBolt.DEFAULT_SLIDING_WINDOW_IN_SECONDS); 59 | final int emitFreq = BenchmarkUtils.getInt(config, EMIT_FREQ, 60 | RollingBolt.DEFAULT_EMIT_FREQUENCY_IN_SECONDS); 61 | 62 | spout = new FileReadSpout(BenchmarkUtils.ifAckEnabled(config)); 63 | 64 | TopologyBuilder builder = new TopologyBuilder(); 65 | 66 | builder.setSpout(SPOUT_ID, spout, spoutNum); 67 | builder.setBolt(SPLIT_ID, new WordCount.SplitSentence(), spBoltNum) 68 | .localOrShuffleGrouping(SPOUT_ID); 69 | builder.setBolt(COUNTER_ID, new RollingCountBolt(windowLength, emitFreq), rcBoltNum) 70 | .fieldsGrouping(SPLIT_ID, new Fields(WordCount.SplitSentence.FIELDS)); 71 | return builder.createTopology(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/SOL.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.topology.IRichSpout; 24 | import backtype.storm.topology.TopologyBuilder; 25 | import com.google.common.collect.Sets; 26 | import storm.benchmark.lib.bolt.ConstBolt; 27 | import storm.benchmark.lib.spout.RandomMessageSpout; 28 | import storm.benchmark.metrics.BasicMetricsCollector; 29 | import storm.benchmark.metrics.IMetricsCollector; 30 | import storm.benchmark.benchmarks.common.StormBenchmark; 31 | import storm.benchmark.util.BenchmarkUtils; 32 | 33 | import static storm.benchmark.metrics.IMetricsCollector.MetricsItem; 34 | 35 | /** 36 | * forked from https://github.com/yahoo/storm-perf-test 37 | */ 38 | 39 | public class SOL extends StormBenchmark { 40 | 41 | public static final String TOPOLOGY_LEVEL = "topology.level"; 42 | public static final String SPOUT_ID = "spout"; 43 | public static final String SPOUT_NUM = "component.spout_num"; 44 | public static final String BOLT_ID = "bolt"; 45 | public static final String BOLT_NUM = "component.bolt_num"; 46 | 47 | public static final int DEFAULT_NUM_LEVELS = 2; 48 | public static final int DEFAULT_SPOUT_NUM = 4; 49 | public static final int DEFAULT_BOLT_NUM = 4; 50 | private IRichSpout spout; 51 | 52 | @Override 53 | public StormTopology getTopology(Config config) { 54 | final int numLevels = BenchmarkUtils.getInt(config, TOPOLOGY_LEVEL, DEFAULT_NUM_LEVELS); 55 | final int msgSize = BenchmarkUtils.getInt(config, RandomMessageSpout.MESSAGE_SIZE, 56 | RandomMessageSpout.DEFAULT_MESSAGE_SIZE); 57 | final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); 58 | final int boltNum = BenchmarkUtils.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM); 59 | 60 | spout = new RandomMessageSpout(msgSize, BenchmarkUtils.ifAckEnabled(config)); 61 | 62 | TopologyBuilder builder = new TopologyBuilder(); 63 | 64 | builder.setSpout(SPOUT_ID, spout, spoutNum); 65 | builder.setBolt(BOLT_ID + 1, new ConstBolt(), boltNum) 66 | .shuffleGrouping(SPOUT_ID); 67 | for (int levelNum = 2; levelNum <= numLevels - 1; levelNum++) { 68 | builder.setBolt(BOLT_ID + levelNum, new ConstBolt(), boltNum) 69 | .shuffleGrouping(BOLT_ID + (levelNum - 1)); 70 | } 71 | return builder.createTopology(); 72 | } 73 | 74 | @Override 75 | public IMetricsCollector getMetricsCollector(Config config, StormTopology topology) { 76 | return new BasicMetricsCollector(config, topology, 77 | Sets.newHashSet(MetricsItem.ALL)); 78 | } 79 | 80 | } 81 | 82 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/util/BenchmarkUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.Config; 22 | import com.google.common.collect.Lists; 23 | import com.google.common.collect.Maps; 24 | import com.google.common.collect.Sets; 25 | import org.testng.annotations.DataProvider; 26 | import org.testng.annotations.Test; 27 | 28 | import java.util.Map; 29 | 30 | import static org.fest.assertions.api.Assertions.assertThat; 31 | 32 | public class BenchmarkUtilsTest { 33 | 34 | @Test(dataProvider = "getAvgs") 35 | public void testAvg(Iterable numbers, double expected) { 36 | assertThat(BenchmarkUtils.avg(numbers)).isEqualTo(expected); 37 | } 38 | 39 | @DataProvider 40 | private Object[][] getAvgs() { 41 | return new Object[][] { 42 | { Lists.newArrayList(1.0, 3.0, 8.0), 4.0 }, 43 | { Sets.newHashSet(2.5, 4.5, 9.5), 5.5 } 44 | }; 45 | } 46 | 47 | @Test(dataProvider = "getMaxes") 48 | public void testMax(Iterable numbers, double expected) { 49 | assertThat(BenchmarkUtils.max(numbers)).isEqualTo(expected); 50 | } 51 | 52 | @DataProvider 53 | private Object[][] getMaxes() { 54 | return new Object[][] { 55 | { Lists.newArrayList(1.0, 3.0, 3.0), 3.0 }, 56 | { Sets.newHashSet(2.5, 4.5, 6.5), 6.5 } 57 | }; 58 | } 59 | 60 | @Test 61 | public void testPutIfAbsent() { 62 | Map map = Maps.newHashMap(); 63 | BenchmarkUtils.putIfAbsent(map, "key1", 1); 64 | assertThat(map.get("key1")).isEqualTo(1); 65 | BenchmarkUtils.putIfAbsent(map, "key1", 2); 66 | assertThat(map.get("key1")).isEqualTo(1); 67 | BenchmarkUtils.putIfAbsent(map, "key2", 3); 68 | assertThat(map.get("key2")).isEqualTo(3); 69 | } 70 | 71 | @Test 72 | public void testGetInt() { 73 | Map map = Maps.newHashMap(); 74 | assertThat(BenchmarkUtils.getInt(map, "key1", 1)).isEqualTo(1); 75 | map.put("key1", 2); 76 | assertThat(BenchmarkUtils.getInt(map, "key1", 1)).isEqualTo(2); 77 | map.put("key2", 3L); 78 | assertThat(BenchmarkUtils.getInt(map, "key2", 1)).isEqualTo(3); 79 | } 80 | 81 | @Test(dataProvider = "getNumberOfAckers") 82 | public void ifAckEnabledShouldReturnTrueForOneOrMoreAckers(int ackers, boolean expected) { 83 | Config config = new Config(); 84 | 85 | assertThat(BenchmarkUtils.ifAckEnabled(config)).isFalse(); 86 | config.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackers); 87 | assertThat(BenchmarkUtils.ifAckEnabled(config)).isEqualTo(expected); 88 | } 89 | 90 | @DataProvider 91 | private Object[][] getNumberOfAckers() { 92 | return new Object[][] { 93 | { 0, false }, 94 | { 1, true }, 95 | { 2, true } 96 | }; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/util/KafkaUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.spout.MultiScheme; 22 | import backtype.storm.utils.Utils; 23 | import storm.kafka.BrokerHosts; 24 | import storm.kafka.SpoutConfig; 25 | import storm.kafka.ZkHosts; 26 | import storm.kafka.trident.TridentKafkaConfig; 27 | 28 | import java.util.ArrayList; 29 | import java.util.Map; 30 | 31 | public final class KafkaUtils { 32 | 33 | public static final String ZOOKEEPER_SERVERS = "zookeeper.servers"; 34 | public static final String KAFKA_ROOT_PATH = "kafka.root.path"; 35 | public static final String TOPIC = "topic"; 36 | public static final String CLIENT_ID = "client_id"; 37 | 38 | public static final String DEFAULT_TOPIC = "storm"; 39 | 40 | private KafkaUtils() { 41 | } 42 | 43 | public static SpoutConfig getSpoutConfig(Map options, MultiScheme scheme) throws IllegalArgumentException { 44 | String zkServers = (String) Utils.get(options, ZOOKEEPER_SERVERS, "localhost:2181"); 45 | String kafkaRoot = (String) Utils.get(options, KAFKA_ROOT_PATH, "/kafka"); 46 | String connectString = zkServers + kafkaRoot; 47 | 48 | BrokerHosts hosts = new ZkHosts(connectString); 49 | String topic = (String) Utils.get(options, TOPIC, DEFAULT_TOPIC); 50 | String zkRoot = kafkaRoot + "/" + "storm-consumer-states"; 51 | String appId = (String) Utils.get(options, CLIENT_ID, "storm-app"); 52 | 53 | SpoutConfig config = new SpoutConfig(hosts, topic, zkRoot, appId); 54 | config.zkServers = new ArrayList(); 55 | 56 | String [] servers = zkServers.split(","); 57 | 58 | for (int i = 0; i < servers.length; i++) { 59 | String[] serverAndPort = servers[0].split(":"); 60 | config.zkServers.add(serverAndPort[0]); 61 | int port = Integer.parseInt(serverAndPort[1]); 62 | if (i == 0) { 63 | config.zkPort = port; 64 | } 65 | 66 | if (config.zkPort != port) { 67 | throw new IllegalArgumentException("The zookeeper port on all server must be same"); 68 | } 69 | } 70 | config.scheme = scheme; 71 | return config; 72 | } 73 | 74 | public static TridentKafkaConfig getTridentKafkaConfig(Map options, MultiScheme scheme) { 75 | String zkServers = (String) Utils.get(options, ZOOKEEPER_SERVERS, "localhost:2181") ; 76 | String kafkaRoot = (String) Utils.get(options, KAFKA_ROOT_PATH, "/kafka"); 77 | String connectString = zkServers + kafkaRoot; 78 | 79 | BrokerHosts hosts = new ZkHosts(connectString); 80 | String topic = (String) Utils.get(options, TOPIC, DEFAULT_TOPIC); 81 | String appId = (String) Utils.get(options, CLIENT_ID, "storm-app"); 82 | 83 | TridentKafkaConfig config = new TridentKafkaConfig(hosts, topic, appId); 84 | config.scheme = scheme; 85 | return config; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/UniqueVisitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.spout.SchemeAsMultiScheme; 24 | import backtype.storm.topology.IRichSpout; 25 | import backtype.storm.topology.TopologyBuilder; 26 | import backtype.storm.tuple.Fields; 27 | import storm.benchmark.lib.bolt.PageViewBolt; 28 | import storm.benchmark.lib.bolt.UniqueVisitorBolt; 29 | import storm.benchmark.benchmarks.common.StormBenchmark; 30 | import storm.benchmark.util.BenchmarkUtils; 31 | import storm.benchmark.util.KafkaUtils; 32 | import storm.kafka.KafkaSpout; 33 | import storm.kafka.StringScheme; 34 | 35 | import static storm.benchmark.lib.spout.pageview.PageView.Item; 36 | 37 | public class UniqueVisitor extends StormBenchmark { 38 | public final static String SPOUT_ID = "spout"; 39 | public final static String SPOUT_NUM = "component.spout_num"; 40 | public final static String VIEW_ID = "view"; 41 | public final static String VIEW_NUM = "component.view_bolt_num"; 42 | public final static String UNIQUER_ID = "uniquer"; 43 | public final static String UNIQUER_NUM = "component.uniquer_bolt_num"; 44 | public final static String WINDOW_LENGTH = "window.length"; 45 | public final static String EMIT_FREQ = "emit.frequency"; 46 | 47 | public static final int DEFAULT_SPOUT_NUM = 4; 48 | public static final int DEFAULT_PV_BOLT_NUM = 4; 49 | public static final int DEFAULT_UV_BOLT_NUM = 4; 50 | public static final int DEFAULT_WINDOW_LENGTH_IN_SEC = 9; // 9s 51 | public static final int DEFAULT_EMIT_FREQ_IN_SEC = 3; // 3s 52 | 53 | protected IRichSpout spout; 54 | 55 | @Override 56 | public StormTopology getTopology(Config config) { 57 | 58 | final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); 59 | final int pvBoltNum = BenchmarkUtils.getInt(config, VIEW_NUM, DEFAULT_PV_BOLT_NUM); 60 | final int uvBoltNum = BenchmarkUtils.getInt(config, UNIQUER_NUM, DEFAULT_UV_BOLT_NUM); 61 | final int winLen = BenchmarkUtils.getInt(config, WINDOW_LENGTH, DEFAULT_WINDOW_LENGTH_IN_SEC); 62 | final int emitFreq = BenchmarkUtils.getInt(config, EMIT_FREQ, DEFAULT_EMIT_FREQ_IN_SEC); 63 | spout = new KafkaSpout(KafkaUtils.getSpoutConfig( 64 | config, new SchemeAsMultiScheme(new StringScheme()))); 65 | 66 | TopologyBuilder builder = new TopologyBuilder(); 67 | builder.setSpout(SPOUT_ID, spout, spoutNum); 68 | builder.setBolt(VIEW_ID, new PageViewBolt(Item.URL, Item.USER), pvBoltNum) 69 | .localOrShuffleGrouping(SPOUT_ID); 70 | builder.setBolt(UNIQUER_ID, new UniqueVisitorBolt(winLen, emitFreq), uvBoltNum) 71 | .fieldsGrouping(VIEW_ID, new Fields(Item.URL.toString())); 72 | return builder.createTopology(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/lib/spout/pageview/PageView.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.spout.pageview; 20 | 21 | import org.apache.log4j.Logger; 22 | import storm.trident.operation.BaseFunction; 23 | import storm.trident.operation.TridentCollector; 24 | import storm.trident.tuple.TridentTuple; 25 | 26 | import java.util.ArrayList; 27 | import java.util.List; 28 | 29 | public class PageView { 30 | private static Logger LOG = Logger.getLogger(PageView.class); 31 | public final String url; 32 | public final int status; 33 | public final int zipCode; 34 | public final int userID; 35 | 36 | public PageView(String url, int status, int zipCode, int userID) { 37 | this.url = url; 38 | this.status = status; 39 | this.zipCode = zipCode; 40 | this.userID = userID; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return String.format("%s\t%d\t%d\t%d", url, status, zipCode, userID); 46 | } 47 | 48 | public static PageView fromString(String pv) { 49 | LOG.debug("get string '" + pv + "'"); 50 | String[] parts = pv.split("\t"); 51 | if (parts.length < 4) { 52 | return null; 53 | } 54 | for (int i = 0; i < parts.length; i++) { 55 | parts[i] = parts[i].trim(); 56 | } 57 | return new PageView(parts[0], 58 | Integer.parseInt(parts[1]), 59 | Integer.parseInt(parts[2]), 60 | Integer.parseInt(parts[3])); 61 | } 62 | 63 | public Object getValue(Item field) { 64 | switch (field) { 65 | case URL: 66 | return url; 67 | case STATUS: 68 | return status; 69 | case ZIP: 70 | return zipCode; 71 | case USER: 72 | return userID; 73 | case ONE: 74 | return 1; 75 | default: 76 | return toString(); 77 | } 78 | } 79 | 80 | public static enum Item { 81 | ALL("page_view"), 82 | URL("url"), 83 | STATUS("http_status"), 84 | ZIP("zip_code"), 85 | USER("user_id"), 86 | ONE("count_one"); 87 | 88 | private final String name; 89 | 90 | Item(String name) { 91 | this.name = name; 92 | } 93 | 94 | @Override 95 | public String toString() { 96 | return name; 97 | } 98 | } 99 | 100 | public static class Extract extends BaseFunction { 101 | 102 | private final List fields; 103 | 104 | public Extract(List fields) { 105 | this.fields = fields; 106 | } 107 | 108 | @Override 109 | public void execute(TridentTuple tuple, TridentCollector collector) { 110 | String pvString = tuple.getString(0); 111 | PageView pageView = PageView.fromString(pvString); 112 | if (null == pageView) { 113 | LOG.error("invalid pageview string '" + pvString + "'"); 114 | return; 115 | } 116 | List values = new ArrayList(); 117 | for (Item field : fields) { 118 | values.add(pageView.getValue(field)); 119 | } 120 | collector.emit(values); 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/util/MetricsUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.util; 20 | 21 | import backtype.storm.generated.ClusterSummary; 22 | import backtype.storm.generated.ExecutorStats; 23 | import backtype.storm.generated.SpoutStats; 24 | import backtype.storm.generated.TopologySummary; 25 | 26 | import java.util.LinkedList; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class MetricsUtils { 31 | 32 | private static final int MB = 1000 * 1000; 33 | 34 | private MetricsUtils() { 35 | } 36 | 37 | public static void addLatency(Map> stats, String id, double lat) { 38 | List latList = stats.get(id); 39 | if (null == latList) { 40 | latList = new LinkedList(); 41 | stats.put(id, latList); 42 | } 43 | latList.add(lat); 44 | } 45 | 46 | 47 | public static TopologySummary getTopologySummary(ClusterSummary cs, String name) { 48 | for (TopologySummary ts : cs.get_topologies()) { 49 | if (name.equals(ts.get_name())) { 50 | return ts; 51 | } 52 | } 53 | return null; 54 | } 55 | 56 | public static double getSpoutCompleteLatency(SpoutStats stats, String window, String stream) { 57 | Map> latAll = stats.get_complete_ms_avg(); 58 | if (latAll != null) { 59 | // latency in a time window 60 | Map latWin = latAll.get(window); 61 | if (latWin != null) { 62 | // latency in a stream 63 | Double latStr = latWin.get(stream); 64 | if (latStr != null) { 65 | return latStr; 66 | } 67 | } 68 | } 69 | return 0.0; 70 | } 71 | 72 | public static long getSpoutAcked(SpoutStats stats, String window, String stream) { 73 | Map> ackedAll = stats.get_acked(); 74 | if (ackedAll != null) { 75 | Map ackedWin = ackedAll.get(window); 76 | if (ackedWin != null) { 77 | Long ackedStr = ackedWin.get(stream); 78 | if (ackedStr != null) { 79 | return ackedStr; 80 | } 81 | } 82 | } 83 | return 0; 84 | } 85 | 86 | public static long getTransferred(ExecutorStats stats, String window, String stream) { 87 | Map> transAll = stats.get_transferred(); 88 | if (transAll != null) { 89 | Map transWin = transAll.get(window); 90 | if (transWin != null) { 91 | Long transStr = transWin.get(stream); 92 | if (transStr != null) { 93 | return transStr; 94 | } 95 | } 96 | } 97 | return 0; 98 | } 99 | 100 | // messages per second 101 | public static double getThroughput(double throughputDiff, long timeDiff) { 102 | return (0 == timeDiff) ? 0.0 : (throughputDiff / (timeDiff / 1000.0)); 103 | } 104 | 105 | // MB per second 106 | public static double getThroughputMB(double throughputDiff, long timeDiff, int msgSize) { 107 | return (0 == timeDiff) ? 0.0 : ((throughputDiff * msgSize) / (timeDiff / 1000.0) / MB); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/bolt/PageViewBoltTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt; 20 | 21 | import backtype.storm.topology.BasicOutputCollector; 22 | import backtype.storm.topology.OutputFieldsDeclarer; 23 | import backtype.storm.tuple.Fields; 24 | import backtype.storm.tuple.Tuple; 25 | import backtype.storm.tuple.Values; 26 | import org.testng.annotations.DataProvider; 27 | import org.testng.annotations.Test; 28 | import storm.benchmark.lib.spout.pageview.PageView; 29 | import storm.benchmark.util.MockTupleHelpers; 30 | 31 | import static org.junit.Assert.assertEquals; 32 | import static org.mockito.Mockito.*; 33 | import static storm.benchmark.lib.spout.pageview.PageView.Item; 34 | 35 | public class PageViewBoltTest { 36 | 37 | @Test(dataProvider = "getAnyFields") 38 | public void shouldEmitOnExecute(Item field1, Item field2) { 39 | PageViewBolt bolt = new PageViewBolt(field1, field2); 40 | BasicOutputCollector collector = mock(BasicOutputCollector.class); 41 | Tuple input = MockTupleHelpers.mockAnyTuple(); 42 | when(input.getString(0)).thenReturn("http://view1\t200\t100000\t100"); 43 | bolt.execute(input, collector); 44 | verify(collector, times(1)).emit(any(Values.class)); 45 | } 46 | 47 | @Test(dataProvider = "getAnyFields") 48 | public void shouldDeclareOutputFields(Item field1, Item field2) { 49 | PageViewBolt bolt = new PageViewBolt(field1, field2); 50 | OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); 51 | bolt.declareOutputFields(declarer); 52 | verify(declarer, times(1)).declare(any(Fields.class)); 53 | } 54 | 55 | @Test(dataProvider = "getAnyFields") 56 | public void getValueShouldReturnRightField(Item field1, Item field2) { 57 | PageViewBolt bolt = new PageViewBolt(field1, field2); 58 | 59 | PageView view1 = new PageView("http://view1", 200, 100000, 100); 60 | assertEquals(view1.toString(), view1.getValue(Item.ALL)); 61 | assertEquals("http://view1", view1.getValue(Item.URL)); 62 | assertEquals(200, view1.getValue(Item.STATUS)); 63 | assertEquals(100000, view1.getValue(Item.ZIP)); 64 | assertEquals(100, view1.getValue(Item.USER)); 65 | assertEquals(1, view1.getValue(Item.ONE)); 66 | 67 | PageView view2 = new PageView("http://view2", 400, 200000, 200); 68 | assertEquals(view2.toString(), view2.getValue(Item.ALL)); 69 | assertEquals("http://view2", view2.getValue(Item.URL)); 70 | assertEquals(400, view2.getValue(Item.STATUS)); 71 | assertEquals(200000, view2.getValue(Item.ZIP)); 72 | assertEquals(200, view2.getValue(Item.USER)); 73 | assertEquals(1, view2.getValue(Item.ONE)); 74 | } 75 | 76 | @DataProvider 77 | private Object[][] getAnyFields() { 78 | final Item[] items = Item.values(); 79 | final int num = items.length; 80 | final Item[][] fields = new Item[(num - 1) * num][2]; 81 | int index = 0; 82 | for (int i = 0; i < num; i++) { 83 | for (int j = 0; j < num; j++) { 84 | if (j != i) { 85 | fields[index][0] = items[i]; 86 | fields[index][1] = items[j]; 87 | index++; 88 | } 89 | } 90 | } 91 | return fields; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/lib/bolt/common/RollingBoltTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.lib.bolt.common; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.topology.BasicOutputCollector; 23 | import backtype.storm.topology.OutputFieldsDeclarer; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.tuple.Tuple; 26 | import backtype.storm.tuple.Values; 27 | import org.testng.annotations.DataProvider; 28 | import org.testng.annotations.Test; 29 | import storm.benchmark.lib.bolt.RollingBolt; 30 | import storm.benchmark.lib.bolt.RollingCountBolt; 31 | import storm.benchmark.lib.bolt.UniqueVisitorBolt; 32 | import storm.benchmark.util.MockTupleHelpers; 33 | 34 | import java.util.Map; 35 | 36 | import static org.fest.assertions.api.Assertions.assertThat; 37 | import static org.mockito.Matchers.any; 38 | import static org.mockito.Mockito.*; 39 | 40 | public class RollingBoltTest { 41 | private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; 42 | private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; 43 | 44 | private Tuple mockNormalTuple(Object obj) { 45 | Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); 46 | when(tuple.getValue(0)).thenReturn(obj); 47 | return tuple; 48 | } 49 | 50 | @Test(dataProvider = "getRollingBolt") 51 | public void shouldEmitNothingIfNoObjectHasBeenCountedYetAndTickTupleIsReceived(RollingBolt bolt) { 52 | Tuple tickTuple = MockTupleHelpers.mockTickTuple(); 53 | BasicOutputCollector collector = mock(BasicOutputCollector.class); 54 | 55 | bolt.execute(tickTuple, collector); 56 | 57 | verifyZeroInteractions(collector); 58 | } 59 | 60 | @Test(dataProvider = "getRollingBolt") 61 | public void shouldEmitSomethingIfAtLeastOneObjectWasCountedAndTickTupleIsReceived(RollingBolt bolt) { 62 | Tuple normalTuple = mockNormalTuple(new Object()); 63 | Tuple tickTuple = MockTupleHelpers.mockTickTuple(); 64 | BasicOutputCollector collector = mock(BasicOutputCollector.class); 65 | 66 | bolt.execute(normalTuple, collector); 67 | bolt.execute(tickTuple, collector); 68 | 69 | verify(collector).emit(any(Values.class)); 70 | } 71 | 72 | @Test(dataProvider = "getRollingBolt") 73 | public void shouldDeclareOutputFields(RollingBolt bolt) { 74 | OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class); 75 | 76 | bolt.declareOutputFields(declarer); 77 | 78 | verify(declarer, times(1)).declare(any(Fields.class)); 79 | } 80 | 81 | 82 | @Test(dataProvider = "getRollingBolt") 83 | public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue(RollingBolt bolt) { 84 | Map componentConfig = bolt.getComponentConfiguration(); 85 | 86 | assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); 87 | Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); 88 | assertThat(emitFrequencyInSeconds).isGreaterThan(0); 89 | } 90 | 91 | @DataProvider 92 | private Object[][] getRollingBolt() { 93 | return new Object[][] { 94 | { new RollingCountBolt() }, 95 | { new UniqueVisitorBolt() } 96 | }; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/test/java/storm/benchmark/tools/SlidingWindowTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools; 20 | 21 | import org.testng.annotations.DataProvider; 22 | import org.testng.annotations.Test; 23 | import storm.benchmark.lib.reducer.LongSummer; 24 | import storm.benchmark.lib.reducer.Reducer; 25 | import storm.benchmark.tools.SlidingWindow; 26 | 27 | import java.util.Map; 28 | 29 | import static org.fest.assertions.api.Assertions.assertThat; 30 | import static org.mockito.Mockito.mock; 31 | 32 | 33 | /** forked from SlidingWindowCounterTest in storm-starter */ 34 | 35 | public class SlidingWindowTest { 36 | private static final Object ANY_OBJECT = "ANY_OBJECT"; 37 | 38 | @DataProvider 39 | public Object[][] illegalWindowLengths() { 40 | return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } }; 41 | } 42 | 43 | @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths") 44 | public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) { 45 | Reducer reducer = mock(Reducer.class); 46 | new SlidingWindow(reducer, windowLengthInSlots); 47 | } 48 | 49 | @DataProvider 50 | public Object[][] legalWindowLengths() { 51 | return new Object[][]{ { 2 }, { 3 }, { 20 } }; 52 | } 53 | 54 | @Test(dataProvider = "legalWindowLengths") 55 | public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) { 56 | Reducer reducer = mock(Reducer.class); 57 | new SlidingWindow(reducer, windowLengthInSlots); 58 | } 59 | 60 | @DataProvider 61 | public Object[][] simulatedCounterIterations() { 62 | return new Object[][]{ 63 | { 2, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 2, 0, 1, 1, 0, 0 } }, 64 | { 3, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 2, 1, 1, 1, 0 } }, 65 | { 4, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 3, 1, 1, 1 } }, 66 | { 5, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 6, 3, 1, 1 } }, 67 | { 5, new int[]{ 3, 11, 5, 13, 7, 17, 0, 3, 50, 600, 7000 }, 68 | new long[]{ 3, 14, 19, 32, 39, 53, 42, 40, 77, 670, 7653 } } 69 | }; 70 | } 71 | 72 | 73 | 74 | 75 | @Test(dataProvider = "simulatedCounterIterations") 76 | public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration, 77 | long[] expCountsPerIteration) { 78 | // given 79 | SlidingWindow counter = new SlidingWindow(new LongSummer(), windowLengthInSlots); 80 | int numIterations = incrementsPerIteration.length; 81 | 82 | for (int i = 0; i < numIterations; i++) { 83 | int numIncrements = incrementsPerIteration[i]; 84 | long expCounts = expCountsPerIteration[i]; 85 | 86 | // given (for this iteration) 87 | if (0 == numIncrements) { 88 | counter.add(ANY_OBJECT, (long) 0); 89 | } 90 | for (int j = 0; j < numIncrements; j++) { 91 | counter.add(ANY_OBJECT, (long) 1); 92 | } 93 | 94 | // when (for this iteration) 95 | Map counts = counter.reduceThenAdvanceWindow(); 96 | 97 | // then (for this iteration) 98 | assertThat(counts.get(ANY_OBJECT)).isEqualTo(expCounts); 99 | } 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/tools/Runner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.tools; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.StormSubmitter; 23 | import backtype.storm.generated.AlreadyAliveException; 24 | import backtype.storm.generated.InvalidTopologyException; 25 | import backtype.storm.generated.StormTopology; 26 | import backtype.storm.utils.Utils; 27 | import org.apache.log4j.Logger; 28 | import storm.benchmark.api.IApplication; 29 | import storm.benchmark.api.IBenchmark; 30 | import storm.benchmark.api.IProducer; 31 | import storm.benchmark.metrics.IMetricsCollector; 32 | import storm.benchmark.metrics.MetricsCollectorConfig; 33 | 34 | /** 35 | * Runner is the main class of storm benchmark 36 | * It instantiates an IBenchmark from passed-in name and then run it 37 | */ 38 | public class Runner { 39 | private static final Logger LOG = Logger.getLogger(Runner.class); 40 | 41 | private static Config config = new Config(); 42 | private static StormTopology topology; 43 | 44 | public static void main(String[] args) throws Exception { 45 | if (null == args || args.length < 1) { 46 | throw new IllegalArgumentException("no benchmark is set"); 47 | } 48 | 49 | run(args[0]); 50 | } 51 | 52 | public static void run(String name) 53 | throws ClassNotFoundException, IllegalAccessException, 54 | InstantiationException, AlreadyAliveException, InvalidTopologyException { 55 | if (name.startsWith("storm.benchmark.benchmarks")) { 56 | LOG.info("running benchmark " + name); 57 | runBenchmark((IBenchmark) getApplicationFromName(name)); 58 | } else if (name.startsWith("storm.benchmark.tools.producer")) { 59 | LOG.info("running producer " + name); 60 | runProducer((IProducer) getApplicationFromName(name)); 61 | } else { 62 | throw new RuntimeException(name + " is neither benchmark nor producer"); 63 | } 64 | } 65 | 66 | public static void runBenchmark(IBenchmark benchmark) 67 | throws AlreadyAliveException, InvalidTopologyException, 68 | ClassNotFoundException, IllegalAccessException, InstantiationException { 69 | runApplication(benchmark); 70 | if (isMetricsEnabled()) { 71 | IMetricsCollector collector = benchmark.getMetricsCollector(config, topology); 72 | collector.run(); 73 | } 74 | } 75 | 76 | public static void runProducer(IProducer producer) 77 | throws AlreadyAliveException, InvalidTopologyException, 78 | ClassNotFoundException, IllegalAccessException, InstantiationException { 79 | runApplication(producer); 80 | } 81 | 82 | 83 | public static IApplication getApplicationFromName(String name) 84 | throws ClassNotFoundException, IllegalAccessException, InstantiationException { 85 | return (IApplication) Class.forName(name).newInstance(); 86 | } 87 | 88 | private static void runApplication(IApplication app) 89 | throws AlreadyAliveException, InvalidTopologyException { 90 | config.putAll(Utils.readStormConfig()); 91 | String name = (String) config.get(Config.TOPOLOGY_NAME); 92 | topology = app.getTopology(config); 93 | StormSubmitter.submitTopology(name, config, topology); 94 | } 95 | 96 | private static boolean isMetricsEnabled() { 97 | return (Boolean) config.get(MetricsCollectorConfig.METRICS_ENABLED); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/storm/benchmark/benchmarks/TridentWordCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License 17 | */ 18 | 19 | package storm.benchmark.benchmarks; 20 | 21 | import backtype.storm.Config; 22 | import backtype.storm.generated.StormTopology; 23 | import backtype.storm.spout.SchemeAsMultiScheme; 24 | import backtype.storm.tuple.Fields; 25 | import backtype.storm.utils.Utils; 26 | import org.jgrapht.graph.DefaultDirectedGraph; 27 | import storm.benchmark.lib.operation.WordSplit; 28 | import storm.benchmark.benchmarks.common.StormBenchmark; 29 | import storm.benchmark.util.BenchmarkUtils; 30 | import storm.benchmark.util.KafkaUtils; 31 | import storm.kafka.StringScheme; 32 | import storm.kafka.trident.TransactionalTridentKafkaSpout; 33 | import storm.trident.TridentTopology; 34 | import storm.trident.operation.builtin.Count; 35 | import storm.trident.planner.*; 36 | import storm.trident.planner.processor.AggregateProcessor; 37 | import storm.trident.planner.processor.EachProcessor; 38 | import storm.trident.planner.processor.PartitionPersistProcessor; 39 | import storm.trident.spout.IPartitionedTridentSpout; 40 | import storm.trident.testing.MemoryMapState; 41 | import storm.trident.util.IndexedEdge; 42 | 43 | import java.io.FileNotFoundException; 44 | import java.io.FileOutputStream; 45 | import java.io.OutputStreamWriter; 46 | import java.io.PrintWriter; 47 | import java.util.HashMap; 48 | import java.util.HashSet; 49 | import java.util.Map; 50 | import java.util.Set; 51 | 52 | 53 | public class TridentWordCount extends StormBenchmark { 54 | 55 | public static final String SPOUT_ID = "spout"; 56 | public static final String SPOUT_NUM = "component.spout_num"; 57 | public static final String SPLIT_ID = "split"; 58 | public static final String SPLIT_NUM = "component.split_bolt_num"; 59 | public static final String COUNT_ID = "count"; 60 | public static final String COUNT_NUM = "component.count_bolt_num"; 61 | 62 | public static final int DEFAULT_SPOUT_NUM = 8; 63 | public static final int DEFAULT_SPLIT_BOLT_NUM = 4; 64 | public static final int DEFAULT_COUNT_BOLT_NUM = 4; 65 | 66 | private IPartitionedTridentSpout spout; 67 | 68 | @Override 69 | public StormTopology getTopology(Config config) { 70 | final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); 71 | final int splitNum = BenchmarkUtils.getInt(config, SPLIT_NUM, DEFAULT_SPLIT_BOLT_NUM); 72 | final int countNum = BenchmarkUtils.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM); 73 | 74 | spout = new TransactionalTridentKafkaSpout( 75 | KafkaUtils.getTridentKafkaConfig(config, new SchemeAsMultiScheme(new StringScheme()))); 76 | 77 | TridentTopology trident = new TridentTopology(); 78 | 79 | trident.newStream("wordcount", spout).name("sentence").parallelismHint(spoutNum).shuffle() 80 | .each(new Fields(StringScheme.STRING_SCHEME_KEY), new WordSplit(), new Fields("word")) 81 | .parallelismHint(splitNum) 82 | .groupBy(new Fields("word")) 83 | .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) 84 | .parallelismHint(countNum); 85 | /* trident.newStream("wordcount", spout) 86 | .each(new Fields(StringScheme.STRING_SCHEME_KEY), new WordSplit(), new Fields("word")) 87 | .groupBy(new Fields("word")) 88 | .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"));*/ 89 | 90 | 91 | return trident.build(); 92 | } 93 | 94 | 95 | 96 | 97 | } 98 | --------------------------------------------------------------------------------