├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CODEOWNERS ├── LICENSE.txt ├── README.md ├── pom.xml ├── script ├── checkstyle-ruleset.xml ├── generateDocs.sh └── header.txt └── src ├── main └── java │ └── com │ └── salesforce │ └── storm │ └── spout │ ├── documentation │ ├── ClassSpec.java │ ├── ConfigDocumentation.java │ ├── DocGenerator.java │ └── MetricDocumentation.java │ ├── dynamic │ ├── ConsumerPartition.java │ ├── DefaultVirtualSpoutIdentifier.java │ ├── DelegateSpout.java │ ├── DelegateSpoutFactory.java │ ├── DynamicSpout.java │ ├── FactoryManager.java │ ├── JSON.java │ ├── Message.java │ ├── MessageBus.java │ ├── MessageId.java │ ├── SpoutMessageBus.java │ ├── Tools.java │ ├── VirtualSpout.java │ ├── VirtualSpoutFactory.java │ ├── VirtualSpoutIdentifier.java │ ├── VirtualSpoutMessageBus.java │ ├── buffer │ │ ├── FifoBuffer.java │ │ ├── MessageBuffer.java │ │ ├── RatioMessageBuffer.java │ │ ├── RoundRobinBuffer.java │ │ └── ThrottledMessageBuffer.java │ ├── config │ │ ├── DocTask.java │ │ └── SpoutConfig.java │ ├── consumer │ │ ├── Consumer.java │ │ ├── ConsumerPeerContext.java │ │ ├── ConsumerState.java │ │ ├── PartitionDistributor.java │ │ ├── PartitionOffsetManager.java │ │ ├── PartitionOffsetsManager.java │ │ └── Record.java │ ├── coordinator │ │ ├── SpoutContext.java │ │ ├── SpoutCoordinator.java │ │ ├── SpoutPartitionProgressMonitor.java │ │ ├── SpoutRunner.java │ │ └── ThreadContext.java │ ├── exception │ │ ├── SpoutAlreadyExistsException.java │ │ ├── SpoutDoesNotExistException.java │ │ └── SpoutNotOpenedException.java │ ├── filter │ │ ├── DefaultFilterChainStepIdentifier.java │ │ ├── FilterChain.java │ │ ├── FilterChainStep.java │ │ ├── FilterChainStepIdentifier.java │ │ ├── InvalidFilterChainStepException.java │ │ ├── NegatingFilterChainStep.java │ │ ├── StaticMessageFilter.java │ │ └── StopFilterChainException.java │ ├── handler │ │ ├── NoopSpoutHandler.java │ │ ├── NoopVirtualSpoutHandler.java │ │ ├── SpoutHandler.java │ │ └── VirtualSpoutHandler.java │ ├── kafka │ │ ├── Consumer.java │ │ ├── DocTask.java │ │ ├── KafkaConsumerConfig.java │ │ ├── KafkaMetrics.java │ │ └── deserializer │ │ │ ├── Deserializer.java │ │ │ ├── Utf8StringDeserializer.java │ │ │ └── compat │ │ │ └── AbstractScheme.java │ ├── metrics │ │ ├── ClassMetric.java │ │ ├── CustomMetric.java │ │ ├── DropwizardRecorder.java │ │ ├── KeyBuilder.java │ │ ├── LogRecorder.java │ │ ├── MetricDefinition.java │ │ ├── MetricsRecorder.java │ │ ├── MultiAssignableMetric.java │ │ ├── SpoutMetrics.java │ │ ├── StormRecorder.java │ │ └── TimerManager.java │ ├── persistence │ │ ├── InMemoryPersistenceAdapter.java │ │ ├── PersistenceAdapter.java │ │ ├── ZookeeperPersistenceAdapter.java │ │ └── zookeeper │ │ │ ├── CuratorFactory.java │ │ │ └── CuratorHelper.java │ └── retry │ │ ├── ExponentialBackoffRetryManager.java │ │ ├── FailedTuplesFirstRetryManager.java │ │ ├── NeverRetryManager.java │ │ └── RetryManager.java │ └── sideline │ ├── SidelineSpout.java │ ├── SidelineVirtualSpoutIdentifier.java │ ├── config │ ├── DocTask.java │ └── SidelineConfig.java │ ├── handler │ ├── SidelineController.java │ ├── SidelineSpoutHandler.java │ └── SidelineVirtualSpoutHandler.java │ ├── metrics │ └── SidelineMetrics.java │ ├── persistence │ ├── FilterChainStepSerializer.java │ ├── InMemoryPersistenceAdapter.java │ ├── PersistenceAdapter.java │ ├── SidelinePayload.java │ └── ZookeeperPersistenceAdapter.java │ ├── recipes │ └── trigger │ │ ├── KeyFilter.java │ │ ├── TriggerEvent.java │ │ ├── TriggerEventHelper.java │ │ └── zookeeper │ │ ├── Config.java │ │ └── ZookeeperWatchTrigger.java │ └── trigger │ ├── SidelineRequest.java │ ├── SidelineRequestIdentifier.java │ ├── SidelineTrigger.java │ └── SidelineType.java └── test ├── java ├── BadLoggersTest.java ├── DisallowUsingShadedImports.java └── com │ └── salesforce │ └── storm │ └── spout │ ├── dynamic │ ├── DefaultVirtualSpoutIdentifierTest.java │ ├── DynamicSpoutTest.java │ ├── FactoryManagerTest.java │ ├── JSONTest.java │ ├── KafkaConsumerSpoutTest.java │ ├── MessageBusTest.java │ ├── MessageIdTest.java │ ├── MessageTest.java │ ├── ToolsTest.java │ ├── VirtualSpoutFactoryTest.java │ ├── VirtualSpoutTest.java │ ├── buffer │ │ ├── FifoBufferTest.java │ │ ├── MessageBufferTest.java │ │ ├── RatioMessageBufferTest.java │ │ ├── RoundRobinBufferTest.java │ │ └── ThrottledMessageBufferTest.java │ ├── consumer │ │ ├── ConsumerPeerContextTest.java │ │ ├── ConsumerStateTest.java │ │ ├── PartitionDistributorTest.java │ │ └── PartitionOffsetManagerTest.java │ ├── coordinator │ │ ├── SpoutCoordinatorTest.java │ │ └── SpoutRunnerTest.java │ ├── filter │ │ ├── FilterChainTest.java │ │ └── NumberFilter.java │ ├── kafka │ │ ├── ConsumerTest.java │ │ ├── KafkaConsumerConfigTest.java │ │ └── deserializer │ │ │ ├── NullDeserializer.java │ │ │ ├── Utf8StringDeserializerTest.java │ │ │ └── compat │ │ │ └── AbstractSchemeTest.java │ ├── metrics │ │ ├── KeyBuilderTest.java │ │ ├── StormRecorderTest.java │ │ └── TimerManagerTest.java │ ├── mocks │ │ ├── MockConsumer.java │ │ ├── MockDelegateSpout.java │ │ ├── MockSpoutHandler.java │ │ ├── MockTopologyContext.java │ │ └── output │ │ │ ├── Emission.java │ │ │ ├── MockOutputCollector.java │ │ │ ├── MockSpoutOutputCollector.java │ │ │ └── SpoutEmission.java │ ├── persistence │ │ ├── ZookeeperPersistenceAdapterTest.java │ │ └── zookeeper │ │ │ └── CuratorHelperTest.java │ ├── retry │ │ ├── ExponentialBackoffRetryManagerTest.java │ │ ├── FailedMsgRetryManagerPerformanceTest.java │ │ ├── FailedTuplesFirstRetryManagerTest.java │ │ └── NeverRetryManagerTest.java │ └── test │ │ └── TestHelper.java │ └── sideline │ ├── SidelineSpoutTest.java │ ├── SidelineVirtualSpoutIdentifierTest.java │ ├── handler │ ├── SidelineSpoutHandlerTest.java │ └── SidelineVirtualSpoutHandlerTest.java │ ├── persistence │ └── ZookeeperPersistenceAdapterTest.java │ ├── recipes │ └── trigger │ │ ├── KeyFilterTest.java │ │ ├── TriggerEventHelperTest.java │ │ └── zookeeper │ │ └── ZookeeperWatchTriggerTest.java │ └── trigger │ ├── SidelineRequestTest.java │ ├── SidelineTypeTest.java │ └── StaticTrigger.java └── resources └── log4j2.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .DS_Store 3 | *.iml 4 | .classpath 5 | .project 6 | .settings/ 7 | target/ 8 | README.md.bak 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk8 4 | - openjdk11 5 | sudo: false 6 | cache: 7 | directories: 8 | - $HOME/.m2 9 | install: true 10 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Comment line immediately above ownership line is reserved for related gus information. Please be careful while editing. 2 | #ECCN:Open Source 3 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017, 2018, Salesforce.com, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 5 | following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 8 | disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 11 | disclaimer in the documentation and/or other materials provided with the distribution. 12 | 13 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 17 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 22 | USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /script/generateDocs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## Generally speaking, you should instead run `mvn clean site` to generate the documentation. 4 | 5 | ## Fail if any command fails 6 | set -e 7 | 8 | ## Build package 9 | mvn package -DskipTests=true 10 | 11 | ## Build DynamicSpout docs 12 | java -cp target/*jar-with-dependencies.jar com.salesforce.storm.spout.dynamic.config.DocTask 13 | 14 | ## Build KafkaConsumer docs 15 | java -cp target/*jar-with-dependencies.jar com.salesforce.storm.spout.dynamic.kafka.DocTask 16 | 17 | ## Build Sideline docs 18 | java -cp target/*jar-with-dependencies.jar com.salesforce.storm.spout.sideline.config.DocTask 19 | -------------------------------------------------------------------------------- /script/header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/documentation/ClassSpec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.documentation; 27 | 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | 31 | /** 32 | * Define a Class instance to generate documentation from. 33 | */ 34 | public class ClassSpec { 35 | private final Class clazz; 36 | private final Map defaults; 37 | 38 | /** 39 | * Constructor. 40 | * @param clazz class to generate documentation for. 41 | * @param defaults default values for each option. 42 | */ 43 | public ClassSpec(final Class clazz, final Map defaults) { 44 | this.clazz = clazz; 45 | this.defaults = defaults; 46 | } 47 | 48 | /** 49 | * Constructor. 50 | * @param clazz class to generate documentation for. 51 | */ 52 | public ClassSpec(final Class clazz) { 53 | this(clazz, new HashMap<>()); 54 | } 55 | 56 | /** 57 | * Get the configured class instance. 58 | * @return class instance. 59 | */ 60 | public Class getClazz() { 61 | return clazz; 62 | } 63 | 64 | public Map getDefaults() { 65 | return defaults; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/documentation/ConfigDocumentation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.documentation; 27 | 28 | import java.lang.annotation.Documented; 29 | import java.lang.annotation.ElementType; 30 | import java.lang.annotation.Retention; 31 | import java.lang.annotation.RetentionPolicy; 32 | import java.lang.annotation.Target; 33 | 34 | /** 35 | * Annotation for documenting spout configuration options. 36 | */ 37 | @Documented 38 | @Retention(RetentionPolicy.RUNTIME) 39 | @Target(ElementType.FIELD) 40 | public @interface ConfigDocumentation { 41 | 42 | /** 43 | * Enum of the categories for the configuration setting. 44 | */ 45 | enum Category { 46 | NONE(""), 47 | DYNAMIC_SPOUT("Dynamic Spout"), 48 | KAFKA("Kafka Consumer"), 49 | PERSISTENCE("Persistence"), 50 | PERSISTENCE_ZOOKEEPER("Zookeeper Persistence"), 51 | SIDELINE("Sideline"); 52 | 53 | private final String value; 54 | 55 | Category(final String value) { 56 | this.value = value; 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return value; 62 | } 63 | } 64 | 65 | /** 66 | * Get the description of the configuration setting. 67 | * @return description of the configuration setting. 68 | */ 69 | String description() default ""; 70 | 71 | /** 72 | * Get whether or not the configuration setting is required. 73 | * @return whether or not the configuration setting is required. 74 | */ 75 | boolean required() default false; 76 | 77 | /** 78 | * Get the category of the configuration setting. 79 | * @return category of the configuration setting. 80 | */ 81 | Category category() default Category.NONE; 82 | 83 | /** 84 | * Get the type of the value for the configuration setting. 85 | * @return type of the value for the configuration setting. 86 | */ 87 | Class type() default Default.class; 88 | 89 | /** 90 | * Default class type for use on the type field. 91 | */ 92 | final class Default { 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/documentation/MetricDocumentation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.documentation; 27 | 28 | import java.lang.annotation.Documented; 29 | import java.lang.annotation.ElementType; 30 | import java.lang.annotation.Retention; 31 | import java.lang.annotation.RetentionPolicy; 32 | import java.lang.annotation.Target; 33 | 34 | /** 35 | * Document metric information. 36 | * 37 | * Used to auto-generate metric documentation in README files. 38 | */ 39 | @Documented 40 | @Retention(RetentionPolicy.RUNTIME) 41 | @Target(ElementType.FIELD) 42 | public @interface MetricDocumentation { 43 | 44 | /** 45 | * Types of metrics. 46 | */ 47 | enum Type { 48 | AVERAGE, 49 | COUNTER, 50 | GAUGE, 51 | TIMER; 52 | 53 | public String toString() { 54 | return name(); 55 | } 56 | } 57 | 58 | /** 59 | * Categories for a metric. 60 | */ 61 | enum Category { 62 | DYNAMIC_SPOUT("Dynamic Spout"), 63 | KAFKA("Kafka"), 64 | SIDELINE("Sideline"); 65 | 66 | final String value; 67 | 68 | Category(final String value) { 69 | this.value = value; 70 | } 71 | 72 | public String toString() { 73 | return value; 74 | } 75 | } 76 | 77 | /** 78 | * Unit for a metric. 79 | */ 80 | enum Unit { 81 | UNKNOWN("Unknown"), 82 | NUMBER("Number"), 83 | PERCENT("Percent 0.0 to 1.0"), 84 | TIME_MILLISECONDS("Time in milliseconds"), 85 | TIME_SECONDS("Time in seconds"); 86 | 87 | final String value; 88 | 89 | Unit(final String value) { 90 | this.value = value; 91 | } 92 | 93 | public String toString() { 94 | return value; 95 | } 96 | } 97 | 98 | /** 99 | * Get the description of the metric. 100 | * @return description of the metric 101 | */ 102 | String description() default ""; 103 | 104 | /** 105 | * Get the values that should be replaced in the key. 106 | * @return values that should be replaced in the key 107 | */ 108 | String[] dynamicValues() default {}; 109 | 110 | /** 111 | * Get the unit of measurement for the metric. 112 | * @return unit of measurement for the metric 113 | */ 114 | Unit unit() default Unit.UNKNOWN; 115 | 116 | /** 117 | * Get the category of the metric. 118 | * @return category of the metric 119 | */ 120 | Category category(); 121 | 122 | /** 123 | * Get the type of the metric. 124 | * @return type of the metric 125 | */ 126 | Type type(); 127 | } 128 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/ConsumerPartition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | /** 29 | * Namespace and partition number DTO object. 30 | */ 31 | public final class ConsumerPartition { 32 | private int hash = 0; 33 | private final String namespace; 34 | private final int partition; 35 | 36 | /** 37 | * Constructor. 38 | * @param namespace Set the namespace for the Consumer. 39 | * @param partition Set the partition within the namespace. 40 | */ 41 | public ConsumerPartition(final String namespace, final int partition) { 42 | this.namespace = namespace; 43 | this.partition = partition; 44 | } 45 | 46 | public int partition() { 47 | return partition; 48 | } 49 | 50 | public String namespace() { 51 | return namespace; 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | if (hash != 0) { 57 | return hash; 58 | } 59 | final int prime = 31; 60 | int result = 1; 61 | result = prime * result + partition; 62 | result = prime * result + ((namespace == null) ? 0 : namespace.hashCode()); 63 | this.hash = result; 64 | return result; 65 | } 66 | 67 | @Override 68 | public boolean equals(Object obj) { 69 | if (this == obj) { 70 | return true; 71 | } 72 | if (obj == null) { 73 | return false; 74 | } 75 | if (getClass() != obj.getClass()) { 76 | return false; 77 | } 78 | ConsumerPartition other = (ConsumerPartition) obj; 79 | if (partition != other.partition) { 80 | return false; 81 | } 82 | if (namespace == null) { 83 | if (other.namespace != null) { 84 | return false; 85 | } 86 | } else if (!namespace.equals(other.namespace)) { 87 | return false; 88 | } 89 | return true; 90 | } 91 | 92 | @Override 93 | public String toString() { 94 | return namespace + "-" + partition; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/DefaultVirtualSpoutIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | import com.google.common.base.Preconditions; 29 | import com.google.common.base.Strings; 30 | 31 | /** 32 | * Identifier for a virtual spout. 33 | */ 34 | public final class DefaultVirtualSpoutIdentifier implements VirtualSpoutIdentifier { 35 | 36 | /** 37 | * The actual identifier. 38 | */ 39 | private final String id; 40 | 41 | /** 42 | * Create a new virtual spout identifier from a string. 43 | * @param id String of the id 44 | */ 45 | public DefaultVirtualSpoutIdentifier(final String id) { 46 | Preconditions.checkArgument(!Strings.isNullOrEmpty(id), "You must provide something in order to create an identifier!"); 47 | this.id = id; 48 | } 49 | 50 | /** 51 | * Get the string form of the identifier. 52 | * @return A string of the identifier 53 | */ 54 | @Override 55 | public String toString() { 56 | return id; 57 | } 58 | 59 | /** 60 | * Is this identifier equal to another. 61 | * @param obj The other identifier 62 | * @return Whether or not the two identifiers are equal 63 | */ 64 | @Override 65 | public boolean equals(Object obj) { 66 | if (this == obj) { 67 | return true; 68 | } 69 | if (obj == null || getClass() != obj.getClass()) { 70 | return false; 71 | } 72 | 73 | DefaultVirtualSpoutIdentifier that = (DefaultVirtualSpoutIdentifier) obj; 74 | 75 | return id != null ? id.equals(that.id) : that.id == null; 76 | } 77 | 78 | /** 79 | * Make a hash code for this object. 80 | * @return Hash code 81 | */ 82 | @Override 83 | public int hashCode() { 84 | return id.hashCode(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/DelegateSpoutFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | import com.salesforce.storm.spout.dynamic.consumer.ConsumerState; 29 | import com.salesforce.storm.spout.dynamic.metrics.MetricsRecorder; 30 | 31 | /** 32 | * Factory for easily creating {@link DelegateSpout} instances. 33 | * 34 | * Handy in things like {@link com.salesforce.storm.spout.dynamic.handler.SpoutHandler} where we want to abstract away particulars, such 35 | * as the things passed down from {@link DynamicSpout} such as the {@link MetricsRecorder} as an example. 36 | */ 37 | public interface DelegateSpoutFactory { 38 | 39 | /** 40 | * Create a {@link DelegateSpout} instance. 41 | * @param identifier identifier to use for this instance. 42 | * @return instance of {@link DelegateSpout}. 43 | */ 44 | DelegateSpout create( 45 | final VirtualSpoutIdentifier identifier 46 | ); 47 | 48 | /** 49 | * Create a {@link DelegateSpout} instance. 50 | * @param identifier identifier to use for this instance. 51 | * @param startingState starting consumer state for this instance. 52 | * @return instance of {@link DelegateSpout}. 53 | */ 54 | DelegateSpout create( 55 | final VirtualSpoutIdentifier identifier, 56 | final ConsumerState startingState 57 | ); 58 | 59 | /** 60 | * Create a {@link DelegateSpout} instance. 61 | * @param identifier identifier to use for this instance. 62 | * @param startingState starting consumer state for this instance. 63 | * @param endingState ending consumer state for this instance. 64 | * @return instance of {@link DelegateSpout}. 65 | */ 66 | DelegateSpout create( 67 | final VirtualSpoutIdentifier identifier, 68 | final ConsumerState startingState, 69 | final ConsumerState endingState 70 | ); 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/JSON.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | import com.google.gson.Gson; 29 | import com.google.gson.GsonBuilder; 30 | import com.salesforce.storm.spout.dynamic.filter.FilterChainStep; 31 | import com.salesforce.storm.spout.sideline.persistence.FilterChainStepSerializer; 32 | 33 | import java.util.HashMap; 34 | import java.util.Map; 35 | 36 | /** 37 | * Thin wrapper around JSON parsing. 38 | * 39 | * Intended to hide the particular JSON implementation of the day. 40 | */ 41 | @SuppressWarnings("checkstyle:AbbreviationAsWordInName") 42 | public class JSON { 43 | 44 | /** 45 | * JSON parser. 46 | * 47 | * Includes a special handler for serialized FilterChainSteps, which we hope to now always have to handle this way. 48 | */ 49 | private final Gson gson; 50 | 51 | /** 52 | * Create JSON serializer/deserializer instance. 53 | */ 54 | public JSON() { 55 | this(new HashMap<>()); 56 | } 57 | 58 | /** 59 | * Create JSON serializer/deserializer instance with default configuration. 60 | * @param config configuration. 61 | */ 62 | public JSON(final Map config) { 63 | final GsonBuilder gsonBuilder = new GsonBuilder() 64 | .setDateFormat("yyyy-MM-dd HH:mm:ss"); 65 | 66 | if (!config.isEmpty()) { 67 | gsonBuilder.registerTypeAdapter(FilterChainStep.class, new FilterChainStepSerializer(config)); 68 | } 69 | 70 | this.gson = gsonBuilder.create(); 71 | } 72 | 73 | /** 74 | * Convert an object to a string of JSON. 75 | * @param value object to be converted. 76 | * @return string of JSON. 77 | */ 78 | public String to(final Object value) { 79 | return gson.toJson(value); 80 | } 81 | 82 | /** 83 | * Convert a string of JSON to an object. 84 | * @param value string of JSON to be converted. 85 | * @param clazz class of object to convert the JSON to. 86 | * @param type to be used for returns. 87 | * @return object converted from JSON. 88 | */ 89 | public T from(final String value, final Class clazz) { 90 | return gson.fromJson(value, clazz); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/SpoutMessageBus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | /** 29 | * Facade in front of MessageBus reducing available scope down to only the methods 30 | * that should be available to the main Spout/DynamicSpout instance. 31 | */ 32 | public interface SpoutMessageBus { 33 | 34 | /** 35 | * Get the next error that should be reported up to the topology, or NULL if none exists. 36 | * @return next error that should be reported up to the topology, or NULL if none exists 37 | */ 38 | Throwable nextReportedError(); 39 | 40 | /** 41 | * Get the next available Message to be emitted into the topology, or NULL if none exists. 42 | * @return next available Message to be emitted into the topology, or NULL if none exists 43 | */ 44 | Message nextMessage(); 45 | 46 | /** 47 | * Acks a tuple on the spout that it belongs to. 48 | * @param id tuple message id to ack 49 | */ 50 | void ack(final MessageId id); 51 | 52 | /** 53 | * Fails a tuple on the spout that it belongs to. 54 | * @param id tuple message id to fail 55 | */ 56 | void fail(final MessageId id); 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/VirtualSpoutIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | /** 29 | * Identifier for a virtual spout. 30 | */ 31 | public interface VirtualSpoutIdentifier { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/VirtualSpoutMessageBus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | /** 29 | * Facade in front of MessageBus reducing available scope down to only the methods 30 | * that should be available to virtual spouts. 31 | */ 32 | public interface VirtualSpoutMessageBus { 33 | 34 | /** 35 | * Registers a new VirtualSpout with the bus. 36 | * @param virtualSpoutIdentifier identifier to register 37 | */ 38 | void registerVirtualSpout(final VirtualSpoutIdentifier virtualSpoutIdentifier); 39 | 40 | /** 41 | * Publish a new message onto the bus. 42 | * Depending on the implementation, this *may* be blocking. 43 | * @param message message to publish 44 | * @throws InterruptedException if the operation is interrupted 45 | */ 46 | void publishMessage(final Message message) throws InterruptedException; 47 | 48 | /** 49 | * Get the number of message on the bus. 50 | * @return number of message on the bus 51 | */ 52 | int messageSize(); 53 | 54 | /** 55 | * Publish an error to the bus. 56 | * @param throwable error to publish 57 | */ 58 | void publishError(final Throwable throwable); 59 | 60 | /** 61 | * Get next acked messageId for the given VirtualSpout. 62 | * This method should never block, but instead return NULL if none exists. 63 | * 64 | * @param virtualSpoutIdentifier Identifier to retrieve the next messageId for. 65 | * @return next available MessageId, or NULL 66 | */ 67 | MessageId getAckedMessage(final VirtualSpoutIdentifier virtualSpoutIdentifier); 68 | 69 | /** 70 | * Get next failed messageId for the given VirtualSpout. 71 | * This method should never block, but instead return a NULL if none exists. 72 | * 73 | * @param virtualSpoutIdentifier Identifier to retrieve the next messageId for. 74 | * @return next available MessageId, or NULL 75 | */ 76 | MessageId getFailedMessage(final VirtualSpoutIdentifier virtualSpoutIdentifier); 77 | 78 | /** 79 | * Called to unregister a VirtualSpout. 80 | * @param virtualSpoutIdentifier identifier to unregister. 81 | */ 82 | void unregisterVirtualSpout(final VirtualSpoutIdentifier virtualSpoutIdentifier); 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/buffer/MessageBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.buffer; 27 | 28 | import com.salesforce.storm.spout.dynamic.Message; 29 | import com.salesforce.storm.spout.dynamic.VirtualSpoutIdentifier; 30 | 31 | import java.util.Map; 32 | 33 | /** 34 | * This interface defines an abstraction around essentially a concurrent queue. 35 | * Abstracting this instead of using directly a queue object allows us to do things like 36 | * implement a "fairness" algorithm on the poll() method for pulling off of the queue. 37 | * Using a straight ConcurrentQueue would give us FIFO semantics (see {@link FifoBuffer}) 38 | * but with an abstraction we could implement round robin (see {@link RoundRobinBuffer}) across 39 | * VirtualSpouts or any scheduling algorithm that we'd like. 40 | */ 41 | public interface MessageBuffer { 42 | 43 | /** 44 | * Called prior to utilizing the instance. 45 | * @param spoutConfig copy of the storm topology config. 46 | */ 47 | void open(Map spoutConfig); 48 | 49 | /** 50 | * Let the Implementation know that we're adding a new VirtualSpoutId. 51 | * @param virtualSpoutId identifier of new Virtual Spout 52 | */ 53 | void addVirtualSpoutId(final VirtualSpoutIdentifier virtualSpoutId); 54 | 55 | /** 56 | * Let the Implementation know that we're removing/cleaning up from closing a VirtualSpout. 57 | * @param virtualSpoutId identifier of Virtual Spout to be cleaned up 58 | */ 59 | void removeVirtualSpoutId(final VirtualSpoutIdentifier virtualSpoutId); 60 | 61 | /** 62 | * Put a new message onto the queue. This method is blocking if the queue buffer is full. 63 | * @param message message to be added to the queue 64 | * @throws InterruptedException thrown if a thread is interrupted while blocked adding to the queue 65 | */ 66 | void put(final Message message) throws InterruptedException; 67 | 68 | /** 69 | * Get the size of the buffer. 70 | * @return size of the buffer 71 | */ 72 | int size(); 73 | 74 | /** 75 | * Get the next message to be processed out of the queue. 76 | * @return next message to be processed out of the queue 77 | */ 78 | Message poll(); 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/config/DocTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.config; 27 | 28 | import com.salesforce.storm.spout.documentation.ClassSpec; 29 | import com.salesforce.storm.spout.documentation.DocGenerator; 30 | import com.salesforce.storm.spout.dynamic.metrics.SpoutMetrics; 31 | 32 | import java.io.IOException; 33 | import java.nio.file.Path; 34 | import java.nio.file.Paths; 35 | import java.util.ArrayList; 36 | import java.util.HashMap; 37 | import java.util.List; 38 | 39 | /** 40 | * Generates Configuration Values for DynamicSpout. 41 | */ 42 | public class DocTask { 43 | /** 44 | * Entry point into Doc generating task. 45 | * @param args not used. 46 | * @throws IOException on error writing file. 47 | */ 48 | public static void main(final String[] args) throws IOException { 49 | final Path inputPath = Paths.get("README.md"); 50 | generateConfigDocs(inputPath); 51 | generateMetricDocs(inputPath); 52 | } 53 | 54 | private static void generateMetricDocs(final Path inputPath) throws IOException { 55 | final String tagArg = "DYNAMIC_SPOUT_METRICS"; 56 | final List classSpecs = new ArrayList<>(); 57 | classSpecs.add(new ClassSpec(SpoutMetrics.class)); 58 | 59 | final DocGenerator docGenerator = new DocGenerator(inputPath, tagArg, classSpecs); 60 | docGenerator.generateMetricDocs(); 61 | } 62 | 63 | private static void generateConfigDocs(final Path inputPath) throws IOException { 64 | final String tagArg = "DYNAMIC_SPOUT_CONFIGURATION"; 65 | final List classSpecs = new ArrayList<>(); 66 | classSpecs.add(new ClassSpec(SpoutConfig.class, SpoutConfig.setDefaults(new HashMap<>()))); 67 | 68 | final DocGenerator docGenerator = new DocGenerator(inputPath, tagArg, classSpecs); 69 | docGenerator.generateConfigDocs(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/consumer/ConsumerPeerContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.consumer; 27 | 28 | /** 29 | * This class presents information to a Consumer about how many total instances of it are running, along with 30 | * the unique instance number the consumer is. 31 | */ 32 | public class ConsumerPeerContext { 33 | /** 34 | * This represents how many consumer instances are running. 35 | * This should always be AT LEAST 1. 36 | */ 37 | private final int totalInstances; 38 | 39 | /** 40 | * This defines what instance number the current instance is out of the total number of instances. 41 | * This number is 0 indexed. 42 | * 43 | * Example, if there are 10 instances total, this will have a value ranging from [0-9] 44 | */ 45 | private final int instanceNumber; 46 | 47 | /** 48 | * Constructor. 49 | * @param totalInstances Total number of consumer instances running. 50 | * @param instanceNumber The instance number. 51 | */ 52 | public ConsumerPeerContext(int totalInstances, int instanceNumber) { 53 | this.totalInstances = totalInstances; 54 | this.instanceNumber = instanceNumber; 55 | } 56 | 57 | /** 58 | * Get the total number of consumer instances. 59 | * @return total number of consumer instances. 60 | */ 61 | public int getTotalInstances() { 62 | return totalInstances; 63 | } 64 | 65 | /** 66 | * Get the instance number for the current instance. 67 | * 68 | * This is take out of the total number of instances. Indexing begins at 0. 69 | * 70 | * @return instance number for the current instance. 71 | */ 72 | public int getInstanceNumber() { 73 | return instanceNumber; 74 | } 75 | 76 | @Override 77 | public String toString() { 78 | return "ConsumerPeerContext{" 79 | + "totalInstances=" + totalInstances 80 | + ", instanceNumber=" + instanceNumber 81 | + '}'; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/consumer/Record.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.consumer; 27 | 28 | import org.apache.storm.tuple.Values; 29 | 30 | /** 31 | * Represents the next 'Record' coming from a Consumer instance. 32 | */ 33 | public class Record { 34 | private final String namespace; 35 | private final int partition; 36 | private final long offset; 37 | private final Values values; 38 | 39 | /** 40 | * Consumer Record. 41 | * @param namespace namespace of the record. 42 | * @param partition partition of the record. 43 | * @param offset offset of the record. 44 | * @param values values of the record. 45 | */ 46 | public Record(final String namespace, final int partition, final long offset, final Values values) { 47 | this.namespace = namespace; 48 | this.partition = partition; 49 | this.offset = offset; 50 | this.values = values; 51 | } 52 | 53 | public String getNamespace() { 54 | return namespace; 55 | } 56 | 57 | public int getPartition() { 58 | return partition; 59 | } 60 | 61 | public long getOffset() { 62 | return offset; 63 | } 64 | 65 | public Values getValues() { 66 | return values; 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | return "Record{" 72 | + "namespace='" + namespace + '\'' 73 | + ", partition=" + partition 74 | + ", offset=" + offset 75 | + ", values=" + values 76 | + '}'; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/coordinator/SpoutContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.coordinator; 27 | 28 | import java.util.concurrent.CompletableFuture; 29 | 30 | /** 31 | * Contains a reference to a SpoutRunner and its associated CompletableFuture. 32 | */ 33 | class SpoutContext { 34 | private final SpoutRunner spoutRunner; 35 | private final CompletableFuture completableFuture; 36 | 37 | /** 38 | * Constructor. 39 | * @param spoutRunner SpoutRunner instance. 40 | * @param completableFuture CompletableFuture instance. 41 | */ 42 | SpoutContext(final SpoutRunner spoutRunner, final CompletableFuture completableFuture) { 43 | this.spoutRunner = spoutRunner; 44 | this.completableFuture = completableFuture; 45 | } 46 | 47 | SpoutRunner getSpoutRunner() { 48 | return spoutRunner; 49 | } 50 | 51 | CompletableFuture getCompletableFuture() { 52 | return completableFuture; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/coordinator/ThreadContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.coordinator; 27 | 28 | /** 29 | * ThreadContext provides details about the context an instance 30 | * is running within. 31 | */ 32 | public class ThreadContext { 33 | /** 34 | * Represents the name of the Storm component that it's executing within. 35 | */ 36 | private final String componentId; 37 | 38 | /** 39 | * Represents the instance number of the Storm component it's executing within. 40 | */ 41 | private final int componentIndex; 42 | 43 | /** 44 | * Constructor. 45 | * @param componentId Name of the Storm component that it's executing within. 46 | * @param componentIndex Instance number of the Storm component it's executing within. 47 | */ 48 | public ThreadContext(final String componentId, final int componentIndex) { 49 | this.componentId = componentId; 50 | this.componentIndex = componentIndex; 51 | } 52 | 53 | public String getComponentId() { 54 | return componentId; 55 | } 56 | 57 | public int getComponentIndex() { 58 | return componentIndex; 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return componentId + ':' + componentIndex; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/exception/SpoutAlreadyExistsException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.exception; 27 | 28 | import com.salesforce.storm.spout.dynamic.DelegateSpout; 29 | 30 | /** 31 | * Thrown when attempting to add a spout to the coordinator and that spout already exists. This doesn't necessarily 32 | * mean that the instance is the same, but that the identifier on the instance matches one that the coordinator was 33 | * already running. 34 | */ 35 | public class SpoutAlreadyExistsException extends RuntimeException { 36 | 37 | /** 38 | * The spout with an identifier that already exists in the coordinator. 39 | */ 40 | private final DelegateSpout spout; 41 | 42 | /** 43 | * Thrown when attempting to add a spout to the coordinator and that spout already exists. 44 | * @param message specific message about the already existing spout. 45 | * @param spout specific spout that appears to already exist in the coordinator. 46 | */ 47 | public SpoutAlreadyExistsException(String message, DelegateSpout spout) { 48 | super(message); 49 | this.spout = spout; 50 | } 51 | 52 | /** 53 | * Spout that caused this exception to be thrown. 54 | * @return spout that caused this exception to be thrown. 55 | */ 56 | public DelegateSpout getSpout() { 57 | return spout; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/exception/SpoutDoesNotExistException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.exception; 27 | 28 | import com.salesforce.storm.spout.dynamic.VirtualSpoutIdentifier; 29 | 30 | /** 31 | * Thrown when attempting to remove a spout from the coordinator and that spout does not exist. 32 | */ 33 | public class SpoutDoesNotExistException extends RuntimeException { 34 | 35 | /** 36 | * Spout with this identifier does not exist in the coordinator. 37 | */ 38 | private final VirtualSpoutIdentifier virtualSpoutIdentifier; 39 | 40 | /** 41 | * Thrown when attempting to remove a spout from the coordinator and that spout does not exist. 42 | * @param message specific message about the already existing spout. 43 | * @param virtualSpoutIdentifier identifer of the virtual spout that does not exist 44 | */ 45 | public SpoutDoesNotExistException(final String message, final VirtualSpoutIdentifier virtualSpoutIdentifier) { 46 | super(message); 47 | this.virtualSpoutIdentifier = virtualSpoutIdentifier; 48 | } 49 | 50 | /** 51 | * Spout that caused this exception to be thrown. 52 | * @return spout that caused this exception to be thrown. 53 | */ 54 | public VirtualSpoutIdentifier getVirtualSpoutIdentifier() { 55 | return virtualSpoutIdentifier; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/exception/SpoutNotOpenedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.exception; 27 | 28 | /** 29 | * Spout has not yet been opened, but needed to be for the performed operation. 30 | */ 31 | public class SpoutNotOpenedException extends RuntimeException { 32 | 33 | /** 34 | * Spout has not yet been opened, but needed to be for the performed operation. 35 | */ 36 | public SpoutNotOpenedException() { 37 | super("Spout has not yet been opened!"); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/filter/DefaultFilterChainStepIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | import com.google.common.base.Preconditions; 29 | import com.google.common.base.Strings; 30 | 31 | import java.util.Objects; 32 | 33 | /** 34 | * Identifier for a FilterChainStep. 35 | */ 36 | public class DefaultFilterChainStepIdentifier implements FilterChainStepIdentifier { 37 | 38 | private final String id; 39 | 40 | /** 41 | * Identifier for a FilterChainStep. 42 | * @param id identifier. 43 | */ 44 | public DefaultFilterChainStepIdentifier(final String id) { 45 | Preconditions.checkArgument(!Strings.isNullOrEmpty(id), "You must provide something in order to create an identifier!"); 46 | this.id = id; 47 | } 48 | 49 | @Override 50 | public boolean equals(Object obj) { 51 | if (this == obj) { 52 | return true; 53 | } 54 | if (obj == null || getClass() != obj.getClass()) { 55 | return false; 56 | } 57 | DefaultFilterChainStepIdentifier that = (DefaultFilterChainStepIdentifier) obj; 58 | return Objects.equals(id, that.id); 59 | } 60 | 61 | @Override 62 | public int hashCode() { 63 | return Objects.hash(id); 64 | } 65 | 66 | @Override 67 | public String toString() { 68 | return id; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/filter/FilterChainStep.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | import com.salesforce.storm.spout.dynamic.Message; 29 | 30 | import java.io.Serializable; 31 | 32 | /** 33 | * A step in a chain for processing records, these steps must be serializable and should include 34 | * an equals() method. 35 | */ 36 | public interface FilterChainStep extends Serializable { 37 | 38 | /** 39 | * Inputs an object, performs some business logic on it and then returns the result. 40 | * 41 | * @param message The filter to be processed by this step of the chain 42 | * @return The resulting filter after being processed 43 | */ 44 | boolean filter(Message message); 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/filter/FilterChainStepIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | /** 29 | * Identifier to a FilterChainStep. 30 | * 31 | * This object marks a given step as unique, and provides a way to identify it. You can implement your FilterChainStep however 32 | * you see fit, say a descriptive string of the filter or an md5 hash of it, etc. It is important that you implement a toString() 33 | * and equals() method and that these should be deterministic when identifying a FilterChainStep instance.. 34 | */ 35 | public interface FilterChainStepIdentifier { 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/filter/InvalidFilterChainStepException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | /** 29 | * Thrown when a filter chain step cannot be serialized/unserialized correctly. 30 | */ 31 | public class InvalidFilterChainStepException extends RuntimeException { 32 | 33 | /** 34 | * Thrown when a filter chain step cannot be serialized/unserialized correctly. 35 | * @param message message. 36 | * @param cause underlying exception causing this one to be thrown. 37 | */ 38 | public InvalidFilterChainStepException(final String message, final Throwable cause) { 39 | super(message, cause); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/filter/NegatingFilterChainStep.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | import com.salesforce.storm.spout.dynamic.Message; 29 | 30 | /** 31 | * Negates a {@link FilterChainStep} basically doing the opposite of the intended filtering. 32 | */ 33 | public class NegatingFilterChainStep implements FilterChainStep { 34 | 35 | private final FilterChainStep step; 36 | 37 | public NegatingFilterChainStep(FilterChainStep step) { 38 | this.step = step; 39 | } 40 | 41 | public boolean filter(Message message) { 42 | return !this.step.filter(message); 43 | } 44 | 45 | @Override 46 | public boolean equals(Object other) { 47 | if (this == other) { 48 | return true; 49 | } 50 | if (other == null || getClass() != other.getClass()) { 51 | return false; 52 | } 53 | 54 | NegatingFilterChainStep that = (NegatingFilterChainStep) other; 55 | 56 | return step != null ? step.equals(that.step) : that.step == null; 57 | } 58 | 59 | @Override 60 | public int hashCode() { 61 | return step != null ? step.hashCode() : 0; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/filter/StaticMessageFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | import com.salesforce.storm.spout.dynamic.Message; 29 | 30 | import java.util.UUID; 31 | 32 | /** 33 | * We use this filter in tests because it allows us an easy way to define 34 | * how a filter behaves. 35 | */ 36 | public class StaticMessageFilter implements FilterChainStep { 37 | 38 | /** 39 | * We need a way to make this instance unique from others, so we use a UUID. 40 | */ 41 | private String id = UUID.randomUUID().toString(); 42 | 43 | public StaticMessageFilter() { 44 | } 45 | 46 | public StaticMessageFilter(final String id) { 47 | this.id = id; 48 | } 49 | 50 | @Override 51 | public boolean filter(Message message) { 52 | return true; 53 | } 54 | 55 | public String getId() { 56 | return id; 57 | } 58 | 59 | @Override 60 | public boolean equals(Object other) { 61 | if (this == other) { 62 | return true; 63 | } 64 | if (other == null || getClass() != other.getClass()) { 65 | return false; 66 | } 67 | 68 | StaticMessageFilter that = (StaticMessageFilter) other; 69 | 70 | return id.equals(that.id); 71 | } 72 | 73 | @Override 74 | public int hashCode() { 75 | return id.hashCode(); 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | return id; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/filter/StopFilterChainException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | /** 29 | * An exception that can be thrown to abort the processing of steps in a chain. 30 | */ 31 | public class StopFilterChainException extends RuntimeException { 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/handler/NoopSpoutHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.handler; 27 | 28 | /** 29 | * No-op Spout Handler, used by default to avoid null pointers. 30 | */ 31 | public class NoopSpoutHandler implements SpoutHandler { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/handler/NoopVirtualSpoutHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.handler; 27 | 28 | /** 29 | * No-op Virtual Spout Handler, used by default to avoid null pointers. 30 | */ 31 | public class NoopVirtualSpoutHandler implements VirtualSpoutHandler { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/handler/SpoutHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.handler; 27 | 28 | import com.salesforce.storm.spout.dynamic.DelegateSpout; 29 | import com.salesforce.storm.spout.dynamic.DelegateSpoutFactory; 30 | import com.salesforce.storm.spout.dynamic.DynamicSpout; 31 | import org.apache.storm.task.TopologyContext; 32 | 33 | import java.util.Map; 34 | 35 | /** 36 | * Handlers (or callbacks) used by the DynamicSpout during it's lifecycle. Integrations can hook into the DynamicSpout 37 | * by creating a SpoutHanlder implementation. 38 | */ 39 | public interface SpoutHandler { 40 | 41 | /** 42 | * Open the handler. 43 | * @param spoutConfig Spout configuration. 44 | * @param delegateSpoutFactory Factory for creating {@link DelegateSpout} instances. 45 | */ 46 | default void open(Map spoutConfig, DelegateSpoutFactory delegateSpoutFactory) { 47 | 48 | } 49 | 50 | /** 51 | * Close the handler. 52 | */ 53 | default void close() { 54 | 55 | } 56 | 57 | /** 58 | * Called when the DynamicSpout is opened. 59 | * @param spout DynamicSpout instance. 60 | * @param topologyConfig Topology configuration. 61 | * @param topologyContext Topology context. 62 | */ 63 | default void onSpoutOpen(DynamicSpout spout, Map topologyConfig, TopologyContext topologyContext) { 64 | 65 | } 66 | 67 | /** 68 | * Called when the DynamicSpout is activated. 69 | * @param spout DynamicSpout instance. 70 | */ 71 | default void onSpoutActivate(DynamicSpout spout) { 72 | 73 | } 74 | 75 | /** 76 | * Called when the DynamicSpout is deactivated. 77 | * @param spout DynamicSpout instance. 78 | */ 79 | default void onSpoutDeactivate(DynamicSpout spout) { 80 | 81 | } 82 | 83 | /** 84 | * Called when the DynamicSpout is closed. 85 | * @param spout DynamicSpout instance. 86 | */ 87 | default void onSpoutClose(DynamicSpout spout) { 88 | 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/handler/VirtualSpoutHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.handler; 27 | 28 | import com.salesforce.storm.spout.dynamic.DelegateSpout; 29 | 30 | import java.util.Map; 31 | 32 | /** 33 | * Handlers (or callbacks) used by the VirtualSpout during it's lifecycle. Integrations can hook into the VirtualSpout 34 | * by creating a VirtualSpoutHandler implementation. 35 | */ 36 | public interface VirtualSpoutHandler { 37 | 38 | /** 39 | * Open the handler. 40 | * @param spoutConfig Spout configuration. 41 | */ 42 | default void open(Map spoutConfig) { 43 | 44 | } 45 | 46 | /** 47 | * Close the handler. 48 | */ 49 | default void close() { 50 | 51 | } 52 | 53 | /** 54 | * Called when the VirtualSpout is opened. 55 | * @param virtualSpout VirtualSpout instance. 56 | */ 57 | default void onVirtualSpoutOpen(DelegateSpout virtualSpout) { 58 | 59 | } 60 | 61 | /** 62 | * Called when the VirtualSpout is activated. 63 | * @param virtualSpout VirtualSpout instance. 64 | */ 65 | default void onVirtualSpoutClose(DelegateSpout virtualSpout) { 66 | 67 | } 68 | 69 | /** 70 | * Called when the VirtualSpout is completed, this happens before the call to onVirtualSpoutClose(), but only when 71 | * the spout is closing and has reached it's endingOffset. 72 | * @param virtualSpout VirtualSpout instance. 73 | */ 74 | default void onVirtualSpoutCompletion(DelegateSpout virtualSpout) { 75 | 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/kafka/DocTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.kafka; 27 | 28 | import com.salesforce.storm.spout.documentation.ClassSpec; 29 | import com.salesforce.storm.spout.documentation.DocGenerator; 30 | 31 | import java.io.IOException; 32 | import java.nio.file.Path; 33 | import java.nio.file.Paths; 34 | import java.util.ArrayList; 35 | import java.util.List; 36 | 37 | /** 38 | * Generates Configuration and Metric README documentation for Kafka Consumer. 39 | */ 40 | public class DocTask { 41 | /** 42 | * Entry point into Doc generating task. 43 | * @param args not used. 44 | * @throws IOException on error writing file. 45 | */ 46 | public static void main(final String[] args) throws IOException { 47 | final Path inputPath = Paths.get("README.md"); 48 | generateConfigDocs(inputPath); 49 | generateMetricDocs(inputPath); 50 | } 51 | 52 | private static void generateMetricDocs(final Path inputPath) throws IOException { 53 | final String tagArg = "KAFKA_CONSUMER_METRICS"; 54 | final List classSpecs = new ArrayList<>(); 55 | classSpecs.add(new ClassSpec(KafkaMetrics.class)); 56 | 57 | final DocGenerator docGenerator = new DocGenerator(inputPath, tagArg, classSpecs); 58 | docGenerator.generateMetricDocs(); 59 | } 60 | 61 | private static void generateConfigDocs(final Path inputPath) throws IOException { 62 | final String tagArg = "KAFKA_CONSUMER_CONFIGURATION"; 63 | final List classSpecs = new ArrayList<>(); 64 | classSpecs.add(new ClassSpec(KafkaConsumerConfig.class)); 65 | 66 | final DocGenerator docGenerator = new DocGenerator(inputPath, tagArg, classSpecs); 67 | docGenerator.generateConfigDocs(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/kafka/KafkaMetrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.kafka; 27 | 28 | import com.salesforce.storm.spout.dynamic.metrics.ClassMetric; 29 | import com.salesforce.storm.spout.dynamic.metrics.MetricDefinition; 30 | import com.salesforce.storm.spout.documentation.MetricDocumentation; 31 | import org.apache.kafka.clients.consumer.KafkaConsumer; 32 | 33 | /** 34 | * Metrics for the {@link com.salesforce.storm.spout.dynamic.kafka.Consumer} and friends. 35 | */ 36 | public final class KafkaMetrics { 37 | @MetricDocumentation( 38 | type = MetricDocumentation.Type.GAUGE, 39 | unit = MetricDocumentation.Unit.NUMBER, 40 | category = MetricDocumentation.Category.KAFKA, 41 | description = "Offset consumer has processed.", 42 | dynamicValues = { "topic", "partition" } 43 | ) 44 | public static final MetricDefinition KAFKA_CURRENT_OFFSET = 45 | new ClassMetric(KafkaConsumer.class, "topic.{}.partition.{}.currentOffset"); 46 | 47 | @MetricDocumentation( 48 | type = MetricDocumentation.Type.GAUGE, 49 | unit = MetricDocumentation.Unit.NUMBER, 50 | category = MetricDocumentation.Category.KAFKA, 51 | description = "Offset for TAIL position in the partition.", 52 | dynamicValues = { "topic", "partition" } 53 | ) 54 | public static final MetricDefinition KAFKA_END_OFFSET = 55 | new ClassMetric(KafkaConsumer.class, "topic.{}.partition.{}.endOffset"); 56 | 57 | @MetricDocumentation( 58 | type = MetricDocumentation.Type.GAUGE, 59 | unit = MetricDocumentation.Unit.NUMBER, 60 | category = MetricDocumentation.Category.KAFKA, 61 | description = "Difference between endOffset and currentOffset metrics.", 62 | dynamicValues = { "topic", "partition" } 63 | ) 64 | public static final MetricDefinition KAFKA_LAG = new ClassMetric(KafkaConsumer.class, "topic.{}.partition.{}.lag"); 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/kafka/deserializer/Deserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.kafka.deserializer; 27 | 28 | import org.apache.storm.tuple.Values; 29 | 30 | import java.io.Serializable; 31 | 32 | /** 33 | * This interface allows you to define how to deserialize messages coming from Kafka into values 34 | * that get emitted as tuples into the topology. 35 | */ 36 | public interface Deserializer extends Serializable { 37 | 38 | /** 39 | * This is the method your implementation would need define. 40 | * A null return value from here will result in this message being ignored. 41 | * 42 | * @param topic - represents what topic this message came from. 43 | * @param partition - represents what partition this message came from. 44 | * @param offset - represents what offset this message came from. 45 | * @param key - byte array representing the key. 46 | * @param value - byte array representing the value. 47 | * @return Values that should be emitted by the spout to the topology. 48 | */ 49 | Values deserialize(final String topic, final int partition, final long offset, final byte[] key, final byte[] value); 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/kafka/deserializer/Utf8StringDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.kafka.deserializer; 27 | 28 | import org.apache.storm.tuple.Values; 29 | 30 | import java.nio.charset.StandardCharsets; 31 | 32 | /** 33 | * Simple deserializer that can deserialize the key and message fields as UTF8 Strings. 34 | */ 35 | public class Utf8StringDeserializer implements Deserializer { 36 | 37 | @Override 38 | public Values deserialize(String topic, int partition, long offset, byte[] key, byte[] value) { 39 | // If the value is null we wholesale abandon and don't return anything. 40 | if (value == null) { 41 | return null; 42 | } 43 | 44 | return new Values( 45 | // Kafka messages can have null keys. 46 | key == null ? null : new String(key, StandardCharsets.UTF_8), 47 | new String(value, StandardCharsets.UTF_8) 48 | ); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/kafka/deserializer/compat/AbstractScheme.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.kafka.deserializer.compat; 27 | 28 | import com.salesforce.storm.spout.dynamic.kafka.deserializer.Deserializer; 29 | import org.apache.storm.tuple.Fields; 30 | import org.apache.storm.tuple.Values; 31 | 32 | import java.nio.ByteBuffer; 33 | import java.util.List; 34 | 35 | /** 36 | * Provides a compatibility-like layer to Storm-Kafka Scheme interface. 37 | */ 38 | public abstract class AbstractScheme implements Deserializer { 39 | 40 | /** 41 | * Define the fields that you will deserialize into. 42 | * @return the fields that you will deserialize into. 43 | */ 44 | public abstract Fields getOutputFields(); 45 | 46 | /** 47 | * Implement deserialization logic and return List of objects/tuple values. 48 | * @param ser byte buffer to deserialize. 49 | * @return list of objects/tuple values. 50 | */ 51 | public abstract List deserialize(ByteBuffer ser); 52 | 53 | /** 54 | * Provides compatibility layer to 'Storm-Kafka' Scheme-like interface. 55 | */ 56 | public Values deserialize(String topic, int partition, long offset, byte[] key, byte[] value) { 57 | List list = deserialize(ByteBuffer.wrap(value)); 58 | if (list == null) { 59 | return null; 60 | } 61 | Values values = new Values(); 62 | values.addAll(list); 63 | return values; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/metrics/ClassMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.metrics; 27 | 28 | import com.google.common.base.Preconditions; 29 | 30 | import java.io.Serializable; 31 | 32 | /** 33 | * Class metric. 34 | * 35 | * Prefixes the metric by the name of the class given on the constructor. 36 | */ 37 | public final class ClassMetric implements MetricDefinition, Serializable { 38 | 39 | /** 40 | * Metric key. 41 | */ 42 | private final String key; 43 | 44 | /** 45 | * Defines a metric for a class. 46 | * @param clazz Context of the metric. 47 | * @param name Name of the metric. 48 | */ 49 | public ClassMetric(final Class clazz, final String name) { 50 | Preconditions.checkNotNull(clazz); 51 | Preconditions.checkNotNull(name); 52 | 53 | // Compute the key name once and store. 54 | this.key = clazz.getSimpleName().concat(".").concat(name); 55 | } 56 | 57 | @Override 58 | public String getKey() { 59 | return this.key; 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | return getKey(); 65 | } 66 | 67 | @Override 68 | public boolean equals(final Object other) { 69 | if (this == other) { 70 | return true; 71 | } 72 | if (other == null || getClass() != other.getClass()) { 73 | return false; 74 | } 75 | final ClassMetric that = (ClassMetric) other; 76 | return getKey().equals(that.getKey()); 77 | } 78 | 79 | @Override 80 | public int hashCode() { 81 | return key.hashCode(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/metrics/CustomMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.metrics; 27 | 28 | import java.io.Serializable; 29 | 30 | /** 31 | * Custom metric. 32 | * 33 | * Take a simple string and adhere to the definition used by the {@link MetricsRecorder}. 34 | */ 35 | public final class CustomMetric implements MetricDefinition, Serializable { 36 | private final String key; 37 | 38 | /** 39 | * Constructor. 40 | * @param key Metric key name. 41 | */ 42 | public CustomMetric(final String key) { 43 | this.key = key; 44 | } 45 | 46 | @Override 47 | public String getKey() { 48 | return key; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return key; 54 | } 55 | 56 | @Override 57 | public boolean equals(final Object other) { 58 | if (this == other) { 59 | return true; 60 | } 61 | if (other == null || getClass() != other.getClass()) { 62 | return false; 63 | } 64 | final CustomMetric that = (CustomMetric) other; 65 | return getKey().equals(that.getKey()); 66 | } 67 | 68 | @Override 69 | public int hashCode() { 70 | return key.hashCode(); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/metrics/KeyBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.metrics; 27 | 28 | import org.slf4j.helpers.MessageFormatter; 29 | 30 | /** 31 | * Takes a {@link MetricDefinition} and generates a String that can be used as a metric key. 32 | */ 33 | class KeyBuilder { 34 | 35 | private final String keyPrefix; 36 | 37 | /** 38 | * Create a new {@link KeyBuilder} with the specified prefix. 39 | * 40 | * Prefix is optional, if you don't need it simply pass in null to the builder. 41 | * 42 | * @param keyPrefix prefix to use when building keys from {@link MetricDefinition} instances 43 | */ 44 | KeyBuilder(final String keyPrefix) { 45 | this.keyPrefix = keyPrefix; 46 | } 47 | 48 | /** 49 | * Build metric keys from {@link MetricDefinition} instances with the optionally supplied parameters. 50 | * 51 | * Strings are formatted using {@link MessageFormatter}, so if you have parameters "foo" and "bar" for a metric with the name of "test" 52 | * than you could use a {@link CustomMetric} with "test.{}.{}" in the constructor and pass parameters "foo" and "bar" to make the full 53 | * metric key of "test.foo.bar". 54 | * 55 | * @return format string like: "prefix.metric_name.param1.param2" 56 | */ 57 | String build(final MetricDefinition metric, final Object[] parameters) { 58 | final StringBuilder keyBuilder = new StringBuilder(); 59 | 60 | // Conditionally add key prefix. 61 | if (getKeyPrefix() != null && !getKeyPrefix().isEmpty()) { 62 | keyBuilder 63 | .append(getKeyPrefix()) 64 | .append("."); 65 | } 66 | 67 | // Our default implementation should include the simple class name in the key 68 | keyBuilder.append( 69 | MessageFormatter.arrayFormat(metric.getKey(), parameters).getMessage() 70 | ); 71 | return keyBuilder.toString(); 72 | } 73 | 74 | /** 75 | * Get key prefix. 76 | * @return key prefix 77 | */ 78 | private String getKeyPrefix() { 79 | return keyPrefix; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/metrics/MetricDefinition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.metrics; 27 | 28 | /** 29 | * Defines a metric that can be used in {@link MetricsRecorder}. 30 | */ 31 | public interface MetricDefinition { 32 | 33 | /** 34 | * Key to use for the given metric. 35 | * @return key to use for the given metric. 36 | */ 37 | String getKey(); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/metrics/MultiAssignableMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.metrics; 27 | 28 | import org.apache.storm.metric.api.AssignableMetric; 29 | import org.apache.storm.metric.api.IMetric; 30 | 31 | import java.util.HashMap; 32 | import java.util.Map; 33 | import java.util.concurrent.ConcurrentHashMap; 34 | 35 | /** 36 | * Simple implementation of a MultiAssignableMetric. 37 | */ 38 | public class MultiAssignableMetric implements IMetric { 39 | private final Map values = new ConcurrentHashMap<>(); 40 | 41 | public MultiAssignableMetric() { 42 | } 43 | 44 | public AssignableMetric scope(String key) { 45 | return this.values.computeIfAbsent(key, k -> new AssignableMetric(null)); 46 | } 47 | 48 | /** 49 | * Get the values stored and resets internal values. 50 | * @return values stored and resets internal values 51 | */ 52 | public Object getValueAndReset() { 53 | Map ret = new HashMap<>(); 54 | 55 | for (Map.Entry entry : this.values.entrySet()) { 56 | ret.put(entry.getKey(), entry.getValue().getValueAndReset()); 57 | } 58 | 59 | return ret; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/persistence/PersistenceAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.persistence; 27 | 28 | import java.util.Map; 29 | 30 | /** 31 | * Interface that controls persistence of state. 32 | */ 33 | public interface PersistenceAdapter { 34 | /** 35 | * Performs any required initialization/connection/setup required for 36 | * the implementation. By contract, this will be called once prior to calling 37 | * persistState() or getState(). 38 | * @param spoutConfig - The storm topology config map. 39 | */ 40 | void open(Map spoutConfig); 41 | 42 | /** 43 | * Performs any cleanup required for the implementation on shutdown. 44 | */ 45 | void close(); 46 | 47 | /** 48 | * Pass in the consumer state that you'd like persisted. 49 | * @param consumerId The consumer's id. 50 | * @param partitionId The partition id 51 | * @param offset Offset of the consumer on the given partition 52 | */ 53 | void persistConsumerState(final String consumerId, final int partitionId, final long offset); 54 | 55 | /** 56 | * Retrieves the consumer state from the persistence layer. 57 | * @param consumerId The consumer's id. 58 | * @param partitionId The partition id 59 | * @return Offset of the consumer on the given partition 60 | */ 61 | Long retrieveConsumerState(final String consumerId, final int partitionId); 62 | 63 | /** 64 | * Removes consumer state from the persistence layer. 65 | * @param consumerId The consumer's id you'd like cleared. 66 | * @param partitionId The partition id 67 | */ 68 | void clearConsumerState(final String consumerId, final int partitionId); 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/retry/FailedTuplesFirstRetryManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.retry; 27 | 28 | import com.salesforce.storm.spout.dynamic.MessageId; 29 | 30 | import java.util.HashSet; 31 | import java.util.LinkedList; 32 | import java.util.Map; 33 | import java.util.Queue; 34 | import java.util.Set; 35 | 36 | /** 37 | * This implementation will always retry failed tuples at the earliest chance it can. 38 | * No back-off strategy, no maximum times a tuple can fail. 39 | */ 40 | public class FailedTuplesFirstRetryManager implements RetryManager { 41 | /** 42 | * This Set holds which Tuples are in flight. 43 | */ 44 | private Set messageIdsInFlight; 45 | 46 | /** 47 | * Our FIFO queue of failed messageIds. 48 | */ 49 | private Queue failedMessageIds; 50 | 51 | @Override 52 | public void open(Map spoutConfig) { 53 | messageIdsInFlight = new HashSet<>(); 54 | failedMessageIds = new LinkedList<>(); 55 | } 56 | 57 | @Override 58 | public void failed(MessageId messageId) { 59 | messageIdsInFlight.remove(messageId); 60 | failedMessageIds.add(messageId); 61 | } 62 | 63 | @Override 64 | public void acked(MessageId messageId) { 65 | messageIdsInFlight.remove(messageId); 66 | failedMessageIds.remove(messageId); 67 | } 68 | 69 | @Override 70 | public MessageId nextFailedMessageToRetry() { 71 | final MessageId nextMessageId = failedMessageIds.poll(); 72 | if (nextMessageId == null) { 73 | return null; 74 | } 75 | messageIdsInFlight.add(nextMessageId); 76 | return nextMessageId; 77 | } 78 | 79 | @Override 80 | public boolean retryFurther(MessageId messageId) { 81 | // We always retry. 82 | return true; 83 | } 84 | 85 | /** 86 | * Get the message ids currently in flight. 87 | * @return message ids currently in flight 88 | */ 89 | Set getMessageIdsInFlight() { 90 | return messageIdsInFlight; 91 | } 92 | 93 | /** 94 | * Get the message ids currently marked as having failed, excluding those in flight. 95 | * @return message ids currently marked as having failed, excluding those in flight 96 | */ 97 | Queue getFailedMessageIds() { 98 | return failedMessageIds; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/retry/NeverRetryManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.retry; 27 | 28 | import com.salesforce.storm.spout.dynamic.MessageId; 29 | 30 | import java.util.Map; 31 | 32 | /** 33 | * This implementation will never retry failed messages. One and done. 34 | */ 35 | public class NeverRetryManager implements RetryManager { 36 | @Override 37 | public void open(Map spoutConfig) { 38 | // Nothing to do 39 | } 40 | 41 | @Override 42 | public void failed(MessageId messageId) { 43 | // Nothing to do 44 | } 45 | 46 | @Override 47 | public void acked(MessageId messageId) { 48 | // Nothing to do 49 | } 50 | 51 | /** 52 | * The next failed message to retry is always null in this implementation. 53 | * @return always null 54 | */ 55 | @Override 56 | public MessageId nextFailedMessageToRetry() { 57 | return null; 58 | } 59 | 60 | /** 61 | * This implementation will never retry further. 62 | * @return always false 63 | */ 64 | @Override 65 | public boolean retryFurther(MessageId messageId) { 66 | return false; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/dynamic/retry/RetryManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.retry; 27 | 28 | import com.salesforce.storm.spout.dynamic.MessageId; 29 | 30 | import java.util.Map; 31 | 32 | /** 33 | * Interface for Tracking Failed messages. By creating an implementation of this interface 34 | * you can control how the DynamicSpout deals with tuples that have failed within the topology. 35 | */ 36 | public interface RetryManager { 37 | 38 | /** 39 | * Initialization. 40 | * @param spoutConfig spout configuration. 41 | */ 42 | void open(Map spoutConfig); 43 | 44 | /** 45 | * Called for MessageId's that have failed to process within the Topology. 46 | * @param messageId id corresponding to the Message that has failed. 47 | */ 48 | void failed(MessageId messageId); 49 | 50 | /** 51 | * Called for MessageId's that have successfully finished processing. 52 | * @param messageId id corresponding to the Message that has finished processing. 53 | */ 54 | void acked(MessageId messageId); 55 | 56 | /** 57 | * Expected to return a MessageId of a previously failed Message that we want to replay. 58 | * If the implementation has no such MessageId that it wants to replay yet, simply return null. 59 | * @return id of a Message that we want to replay, or null if it has none to replay. 60 | */ 61 | MessageId nextFailedMessageToRetry(); 62 | 63 | /** 64 | * The Spout will call this for any MessageId that has failed to be processed by the topology. 65 | * If the implementation will want to retry this MessageId in the future it should return true here. 66 | * If the implementation returns false, the Spout will clean up its state around that Message, mark it 67 | * has having been completed successfully, and will be unable to replay it later. 68 | * 69 | * @param messageId id the spout wants to know if it will want to be replayed in the future. 70 | * @return true if the message should be retried again in the future. False otherwise. 71 | */ 72 | boolean retryFurther(MessageId messageId); 73 | } -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/SidelineSpout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline; 27 | 28 | import com.salesforce.storm.spout.dynamic.DynamicSpout; 29 | import com.salesforce.storm.spout.dynamic.Tools; 30 | import com.salesforce.storm.spout.dynamic.config.SpoutConfig; 31 | import com.salesforce.storm.spout.sideline.handler.SidelineSpoutHandler; 32 | import com.salesforce.storm.spout.sideline.handler.SidelineVirtualSpoutHandler; 33 | 34 | import java.util.HashMap; 35 | import java.util.Map; 36 | 37 | /** 38 | * Spout that supports sidelining messages by filters. 39 | */ 40 | public class SidelineSpout extends DynamicSpout { 41 | 42 | /** 43 | * Used to overload and modify settings before passing them to the constructor. 44 | * @param spoutConfig Supplied configuration. 45 | * @return Resulting configuration. 46 | */ 47 | private static Map modifyConfig(Map spoutConfig) { 48 | Map config = new HashMap<>(); 49 | // Start by making a copy of our existing configuration map 50 | config.putAll(spoutConfig); 51 | // Add our opinionated configuration items 52 | config.put(SpoutConfig.SPOUT_HANDLER_CLASS, SidelineSpoutHandler.class.getName()); 53 | config.put(SpoutConfig.VIRTUAL_SPOUT_HANDLER_CLASS, SidelineVirtualSpoutHandler.class.getName()); 54 | // Return a copy of the config that cannot be modified. 55 | return Tools.immutableCopy(config); 56 | } 57 | 58 | /** 59 | * Spout that supports sidelining messages by filters. 60 | * @param config Spout configuration. 61 | */ 62 | @SuppressWarnings("unchecked") 63 | public SidelineSpout(Map config) { 64 | super(modifyConfig(config)); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/config/DocTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.config; 27 | 28 | import com.salesforce.storm.spout.documentation.ClassSpec; 29 | import com.salesforce.storm.spout.documentation.DocGenerator; 30 | import com.salesforce.storm.spout.sideline.metrics.SidelineMetrics; 31 | 32 | import java.io.IOException; 33 | import java.nio.file.Path; 34 | import java.nio.file.Paths; 35 | import java.util.ArrayList; 36 | import java.util.HashMap; 37 | import java.util.List; 38 | 39 | /** 40 | * Generates Configuration Values for Sideline. 41 | */ 42 | public class DocTask { 43 | /** 44 | * Entry point into Doc generating task. 45 | * @param args not used. 46 | * @throws IOException on error writing file. 47 | */ 48 | public static void main(String[] args) throws IOException { 49 | final Path inputPath = Paths.get("README.md"); 50 | generateConfigDocs(inputPath); 51 | generateMetricDocs(inputPath); 52 | } 53 | 54 | private static void generateMetricDocs(final Path inputPath) throws IOException { 55 | final String tagArg = "SIDELINE_METRICS"; 56 | final List classSpecs = new ArrayList<>(); 57 | classSpecs.add(new ClassSpec(SidelineMetrics.class)); 58 | 59 | final DocGenerator docGenerator = new DocGenerator(inputPath, tagArg, classSpecs); 60 | docGenerator.generateMetricDocs(); 61 | } 62 | 63 | private static void generateConfigDocs(final Path inputPath) throws IOException { 64 | final String tagArg = "SIDELINE_CONFIGURATION"; 65 | final List classSpecs = new ArrayList<>(); 66 | classSpecs.add(new ClassSpec(SidelineConfig.class, SidelineConfig.setDefaults(new HashMap<>()))); 67 | 68 | final DocGenerator docGenerator = new DocGenerator(inputPath, tagArg, classSpecs); 69 | docGenerator.generateConfigDocs(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/metrics/SidelineMetrics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.metrics; 27 | 28 | import com.salesforce.storm.spout.dynamic.metrics.ClassMetric; 29 | import com.salesforce.storm.spout.dynamic.metrics.MetricDefinition; 30 | import com.salesforce.storm.spout.documentation.MetricDocumentation; 31 | import com.salesforce.storm.spout.sideline.handler.SidelineSpoutHandler; 32 | 33 | /** 34 | * Sideline specific metrics. 35 | */ 36 | public final class SidelineMetrics { 37 | 38 | @MetricDocumentation( 39 | type = MetricDocumentation.Type.COUNTER, 40 | unit = MetricDocumentation.Unit.NUMBER, 41 | category = MetricDocumentation.Category.SIDELINE, 42 | description = "Total number of started sidelines." 43 | ) 44 | public static final MetricDefinition START = new ClassMetric(SidelineSpoutHandler.class, "start"); 45 | 46 | @MetricDocumentation( 47 | type = MetricDocumentation.Type.COUNTER, 48 | unit = MetricDocumentation.Unit.NUMBER, 49 | category = MetricDocumentation.Category.SIDELINE, 50 | description = "Total number of resumed sidelines." 51 | ) 52 | public static final MetricDefinition RESUME = new ClassMetric(SidelineSpoutHandler.class, "resume"); 53 | 54 | @MetricDocumentation( 55 | type = MetricDocumentation.Type.COUNTER, 56 | unit = MetricDocumentation.Unit.NUMBER, 57 | category = MetricDocumentation.Category.SIDELINE, 58 | description = "Total number of resolving sidelines." 59 | ) 60 | public static final MetricDefinition RESOLVE = new ClassMetric(SidelineSpoutHandler.class, "resolve"); 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/recipes/trigger/KeyFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.recipes.trigger; 27 | 28 | import com.salesforce.storm.spout.dynamic.Message; 29 | import com.salesforce.storm.spout.dynamic.filter.FilterChainStep; 30 | 31 | import java.util.List; 32 | 33 | /** 34 | * Filter for filtering messages by their key. 35 | */ 36 | public class KeyFilter implements FilterChainStep { 37 | 38 | private final List filteredKeys; 39 | 40 | public KeyFilter(final List filteredKeys) { 41 | this.filteredKeys = filteredKeys; 42 | } 43 | 44 | @Override 45 | public boolean filter(Message message) { 46 | final String key = String.valueOf(message.getValues().get(0)); 47 | // If the key for this message is contained in our list we filter it out 48 | return filteredKeys.contains(key); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/trigger/SidelineRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.trigger; 27 | 28 | import com.salesforce.storm.spout.dynamic.filter.FilterChainStep; 29 | 30 | /** 31 | * A request to sideline. 32 | */ 33 | public class SidelineRequest { 34 | 35 | /** 36 | * Id of the sideline request. 37 | */ 38 | public final SidelineRequestIdentifier id; 39 | /** 40 | * Filter chain step for this sideline. 41 | */ 42 | public final FilterChainStep step; 43 | 44 | /** 45 | * A request to sideline. 46 | * @param id id of the sideline request. 47 | * @param step filter chain step for this sideline. 48 | */ 49 | public SidelineRequest(final SidelineRequestIdentifier id, final FilterChainStep step) { 50 | this.id = id; 51 | this.step = step; 52 | } 53 | 54 | @Override 55 | public boolean equals(Object other) { 56 | if (this == other) { 57 | return true; 58 | } 59 | if (other == null || getClass() != other.getClass()) { 60 | return false; 61 | } 62 | 63 | SidelineRequest that = (SidelineRequest) other; 64 | 65 | return step != null ? step.equals(that.step) : that.step == null; 66 | } 67 | 68 | @Override 69 | public int hashCode() { 70 | return step != null ? step.hashCode() : 0; 71 | } 72 | 73 | @Override 74 | public String toString() { 75 | return "SidelineRequest{" 76 | + "id=" 77 | + id 78 | + ", step=" 79 | + step 80 | + '}'; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/trigger/SidelineRequestIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.trigger; 27 | 28 | import com.salesforce.storm.spout.dynamic.filter.FilterChainStepIdentifier; 29 | 30 | /** 31 | * Identifies a sideline request, this should be unique to the request. 32 | */ 33 | public class SidelineRequestIdentifier implements FilterChainStepIdentifier { 34 | 35 | private String id; 36 | 37 | public SidelineRequestIdentifier(final String id) { 38 | this.id = id; 39 | } 40 | 41 | /** 42 | * Override toString to return the id. 43 | */ 44 | @Override 45 | public String toString() { 46 | return id; 47 | } 48 | 49 | @Override 50 | public boolean equals(Object other) { 51 | if (this == other) { 52 | return true; 53 | } 54 | if (other == null || getClass() != other.getClass()) { 55 | return false; 56 | } 57 | 58 | SidelineRequestIdentifier that = (SidelineRequestIdentifier) other; 59 | 60 | return id != null ? id.equals(that.id) : that.id == null; 61 | } 62 | 63 | @Override 64 | public int hashCode() { 65 | return id != null ? id.hashCode() : 0; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/trigger/SidelineTrigger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.trigger; 27 | 28 | import com.salesforce.storm.spout.sideline.handler.SidelineController; 29 | 30 | import java.util.Map; 31 | 32 | /** 33 | * A trigger is a class that can start and stop a sideline by constructing sideline requests. 34 | */ 35 | public interface SidelineTrigger { 36 | 37 | 38 | /** 39 | * Set the sideline controller instance. 40 | * @param sidelineController sideline controller instance. 41 | */ 42 | default void setSidelineController(SidelineController sidelineController) { 43 | // TODO: Drop the default here when we're ready to switch over to this method. 44 | } 45 | 46 | /** 47 | * Open the trigger. 48 | * @param spoutConfig Spout configuration. 49 | */ 50 | default void open(Map spoutConfig) { 51 | } 52 | 53 | /** 54 | * Close the trigger. 55 | */ 56 | default void close() { 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/salesforce/storm/spout/sideline/trigger/SidelineType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.trigger; 27 | 28 | /** 29 | * Type of sideline request, either a start of stop operation. 30 | */ 31 | public enum SidelineType { 32 | 33 | /** 34 | * Start means that a filter is being applied to the firehose and the current offset is captured so it can be used later 35 | * during the resume phase. 36 | */ 37 | START, 38 | /** 39 | * Resume means that a filter is on the firehose, and a new {@link com.salesforce.storm.spout.dynamic.VirtualSpout} is being 40 | * spun up with the negation of that filter. In other words, whatever that filter was filtering will now be processed 41 | * in it's own thread where it can be throttled independently of the firehose. 42 | */ 43 | RESUME, 44 | /** 45 | * Resolve means that an ending offset is being applied to the {@link com.salesforce.storm.spout.dynamic.VirtualSpout} and it 46 | * cease to run whenever it reached that {@link com.salesforce.storm.spout.dynamic.consumer.ConsumerState}. The filter is also 47 | * removed from the firehose, so that future record's are processed in real time. 48 | */ 49 | RESOLVE; 50 | 51 | /** 52 | * Get a {@link SidelineType} from a provided string. 53 | * @param value string that maps to a specific sideline type. 54 | * @return sideline type from the provided string. 55 | */ 56 | public static SidelineType fromValue(final String value) { 57 | if (value == null) { 58 | throw new IllegalArgumentException("Provided value cannot be null."); 59 | } 60 | 61 | switch (value.toLowerCase()) { 62 | case "start": 63 | return SidelineType.START; 64 | case "resume": 65 | return SidelineType.RESUME; 66 | case "resolve": 67 | return SidelineType.RESOLVE; 68 | default: 69 | throw new IllegalArgumentException("Provided \"" + value + "\" is not a valid sideline type."); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/DisallowUsingShadedImports.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | import org.junit.jupiter.api.Test; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import java.io.File; 31 | import java.io.FileNotFoundException; 32 | import java.util.ArrayList; 33 | import java.util.List; 34 | import java.util.Scanner; 35 | import java.util.regex.Matcher; 36 | import java.util.regex.Pattern; 37 | 38 | import static org.junit.jupiter.api.Assertions.assertTrue; 39 | 40 | 41 | /** 42 | * Using imports from shaded libraries is a bad practice and makes upgrading difficult. 43 | */ 44 | public class DisallowUsingShadedImports { 45 | private static final Logger logger = LoggerFactory.getLogger(DisallowUsingShadedImports.class); 46 | 47 | // Find any imports using .shade. 48 | private static final Pattern regexPattern = Pattern.compile("import .*(\\.storm\\.shade\\.).*;"); 49 | 50 | private List failedFiles = new ArrayList<>(); 51 | 52 | @Test 53 | public void doTest() throws FileNotFoundException { 54 | // Hacky way to determine root path 55 | final File currentPath = new File(getClass().getProtectionDomain().getCodeSource().getLocation().getPath()); 56 | final File projectRootPath = currentPath.getParentFile().getParentFile(); 57 | 58 | // Walk all the files in the path 59 | walk(projectRootPath); 60 | } 61 | 62 | private void walk(File root) throws FileNotFoundException { 63 | File[] list = root.listFiles(); 64 | 65 | if (list == null) { 66 | return; 67 | } 68 | 69 | for (File f : list) { 70 | if (f.isDirectory()) { 71 | walk(f); 72 | } else { 73 | // Skip non java source files 74 | if (!f.getAbsoluteFile().getPath().endsWith(".java")) { 75 | continue; 76 | } 77 | testFile(f); 78 | } 79 | } 80 | 81 | failedFiles.forEach(logger::error); 82 | 83 | assertTrue(failedFiles.isEmpty(), "Should have not found any files"); 84 | } 85 | 86 | private void testFile(File myFile) throws FileNotFoundException { 87 | String fileData = new Scanner(myFile).useDelimiter("\\Z").next(); 88 | 89 | // Look for our pattern 90 | Matcher matches = regexPattern.matcher(fileData); 91 | 92 | // If we didn't find a match 93 | if (!matches.find()) { 94 | return; 95 | } 96 | 97 | // Found shade import usage! 98 | failedFiles.add("Found instance of logger using wrong class? " + myFile.getPath() + " Using " + matches.toMatchResult().toString()); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/DefaultVirtualSpoutIdentifierTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | import org.junit.jupiter.api.Assertions; 29 | import org.junit.jupiter.api.Test; 30 | 31 | import static org.junit.jupiter.api.Assertions.assertEquals; 32 | import static org.junit.jupiter.api.Assertions.assertNotEquals; 33 | 34 | /** 35 | * Test that {@link DefaultVirtualSpoutIdentifier}'s can be created correctly. 36 | */ 37 | public class DefaultVirtualSpoutIdentifierTest { 38 | 39 | /** 40 | * Test that two identifiers created with the same string match. 41 | * @throws Exception Bad identifier 42 | */ 43 | @Test 44 | public void test_toString_and_equals() throws Exception { 45 | final DefaultVirtualSpoutIdentifier virtualSpoutIdentifier = new DefaultVirtualSpoutIdentifier("FooBar"); 46 | 47 | assertEquals(virtualSpoutIdentifier, new DefaultVirtualSpoutIdentifier("FooBar")); 48 | assertEquals(virtualSpoutIdentifier.toString(), new DefaultVirtualSpoutIdentifier("FooBar").toString()); 49 | } 50 | 51 | /** 52 | * Test that two identifiers that are different do not match. 53 | * @throws Exception Bad identifier 54 | */ 55 | @Test 56 | public void test_not_toString_and_equals() throws Exception { 57 | final DefaultVirtualSpoutIdentifier virtualSpoutIdentifier1 = new DefaultVirtualSpoutIdentifier("Foo"); 58 | final DefaultVirtualSpoutIdentifier virtualSpoutIdentifier2 = new DefaultVirtualSpoutIdentifier("Bar"); 59 | 60 | assertNotEquals(virtualSpoutIdentifier1, virtualSpoutIdentifier2); 61 | assertNotEquals(virtualSpoutIdentifier1.toString(), virtualSpoutIdentifier2.toString()); 62 | } 63 | 64 | /** 65 | * Test that supplying null will throw an exception. 66 | */ 67 | @Test 68 | public void test_nullIdentifier() { 69 | Assertions.assertThrows(IllegalArgumentException.class, () -> 70 | new DefaultVirtualSpoutIdentifier(null) 71 | ); 72 | } 73 | 74 | /** 75 | * Test that supplying an empty string will throw an exception. 76 | */ 77 | @Test 78 | public void test_emptyIdentifier() { 79 | Assertions.assertThrows(IllegalArgumentException.class, () -> 80 | new DefaultVirtualSpoutIdentifier("") 81 | ); 82 | } 83 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/JSONTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic; 27 | 28 | import org.junit.jupiter.api.Test; 29 | 30 | import java.util.HashMap; 31 | import java.util.Map; 32 | 33 | import static org.junit.jupiter.api.Assertions.assertEquals; 34 | 35 | /** 36 | * Tests that the JSON abstraction converts to and from JSON correctly. 37 | */ 38 | @SuppressWarnings("checkstyle:AbbreviationAsWordInName") 39 | class JSONTest { 40 | 41 | // Why the heck am I double wrapping the map? Because GSON can't handle inner classes... 42 | private final Map jsonMap = new HashMap<>(new HashMap() { 43 | { 44 | put("key1", "value1"); 45 | put("key2", 2.0); 46 | put("key3", true); 47 | } 48 | }); 49 | 50 | // Note GSON favors doubles, so if I gave it just "2" I'd still get 2.0 when it converts to JSON 51 | private final String json = "{\"key1\":\"value1\",\"key2\":2.0,\"key3\":true}"; 52 | 53 | /** 54 | * Test that given a Map we get a valid string of JSON back. 55 | */ 56 | @Test 57 | void testTo() { 58 | assertEquals( 59 | json, 60 | new JSON(new HashMap<>()).to(jsonMap) 61 | ); 62 | } 63 | 64 | /** 65 | * Test that given a string of JSON we get a valid HashMap back. 66 | */ 67 | @Test 68 | void testFrom() { 69 | assertEquals( 70 | jsonMap, 71 | new JSON(new HashMap<>()).from(json, HashMap.class) 72 | ); 73 | } 74 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/consumer/ConsumerPeerContextTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.consumer; 27 | 28 | import org.junit.jupiter.api.Test; 29 | 30 | import static org.junit.jupiter.api.Assertions.assertEquals; 31 | 32 | /** 33 | * Test that {@link ConsumerPeerContext} stored information about it's peers correctly. 34 | */ 35 | public class ConsumerPeerContextTest { 36 | 37 | /** 38 | * Verifies total instances property. 39 | */ 40 | @Test 41 | public void getTotalInstances() { 42 | final int expectedInstances = 23; 43 | final int expectedInstanceNumber = 45; 44 | final ConsumerPeerContext cpc = new ConsumerPeerContext(expectedInstances, expectedInstanceNumber); 45 | assertEquals(expectedInstances, cpc.getTotalInstances()); 46 | } 47 | 48 | /** 49 | * Verifies instance number property. 50 | */ 51 | @Test 52 | public void getInstanceNumber() { 53 | final int expectedInstances = 23; 54 | final int expectedInstanceNumber = 45; 55 | final ConsumerPeerContext cpc = new ConsumerPeerContext(expectedInstances, expectedInstanceNumber); 56 | assertEquals(expectedInstanceNumber, cpc.getInstanceNumber()); 57 | } 58 | 59 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/filter/NumberFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.filter; 27 | 28 | import com.salesforce.storm.spout.dynamic.Message; 29 | 30 | class NumberFilter implements FilterChainStep { 31 | 32 | private final int number; 33 | 34 | NumberFilter(final int number) { 35 | this.number = number; 36 | } 37 | 38 | /** 39 | * Filter a message. 40 | * @param message The filter to be processed by this step of the chain. 41 | * @return true if the message should be filtered, false otherwise. 42 | */ 43 | @Override 44 | public boolean filter(Message message) { 45 | Integer messageNumber = (Integer) message.getValues().get(0); 46 | // Filter them if they don't match, in other words "not" equals 47 | return messageNumber.equals(number); 48 | } 49 | 50 | /** 51 | * Get the number used in the filter. 52 | * @return number used in the filter. 53 | */ 54 | int getNumber() { 55 | return number; 56 | } 57 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/kafka/KafkaConsumerConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.kafka; 27 | 28 | import org.apache.kafka.common.serialization.ByteArrayDeserializer; 29 | import org.junit.jupiter.api.Test; 30 | 31 | import java.util.Arrays; 32 | import java.util.List; 33 | 34 | import static org.junit.jupiter.api.Assertions.assertEquals; 35 | 36 | /** 37 | * Validates that we have some sane default settings. 38 | */ 39 | public class KafkaConsumerConfigTest { 40 | 41 | /** 42 | * Validates that we have some sane default settings. 43 | */ 44 | @Test 45 | public void testForSaneDefaultKafkaConsumerSettings() { 46 | final List brokerHosts = Arrays.asList( 47 | "broker1:9092", "broker2:9093" 48 | ); 49 | final String consumerId = "myConsumerId"; 50 | final String topic = "myTopic"; 51 | 52 | // Create config 53 | final KafkaConsumerConfig config = new KafkaConsumerConfig(brokerHosts, consumerId, topic); 54 | 55 | // now do validation on constructor arguments 56 | assertEquals(topic, config.getTopic(), "Topic set"); 57 | assertEquals(consumerId, config.getConsumerId(), "ConsumerId set"); 58 | assertEquals("broker1:9092,broker2:9093", config.getKafkaConsumerProperty("bootstrap.servers"), "BrokerHosts set"); 59 | 60 | // Check defaults are sane. 61 | assertEquals(consumerId, config.getKafkaConsumerProperty("group.id"), "group.id set"); 62 | assertEquals("none", config.getKafkaConsumerProperty("auto.offset.reset"), "auto.offset.reset set to none"); 63 | assertEquals( 64 | ByteArrayDeserializer.class.getName(), 65 | config.getKafkaConsumerProperty("key.deserializer"), 66 | "Key Deserializer set to bytes deserializer" 67 | ); 68 | assertEquals( 69 | ByteArrayDeserializer.class.getName(), 70 | config.getKafkaConsumerProperty("value.deserializer"), 71 | "Value Deserializer set to bytes deserializer" 72 | ); 73 | } 74 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/kafka/deserializer/NullDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.kafka.deserializer; 27 | 28 | import org.apache.storm.tuple.Values; 29 | 30 | /** 31 | * Define a deserializer that always returns null. Only used for testing purposes. 32 | */ 33 | public class NullDeserializer implements Deserializer { 34 | @Override 35 | public Values deserialize(String topic, int partition, long offset, byte[] key, byte[] value) { 36 | return null; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/mocks/MockTopologyContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.mocks; 27 | 28 | import org.apache.storm.metric.api.IMetric; 29 | import org.apache.storm.task.TopologyContext; 30 | 31 | import java.util.Collections; 32 | import java.util.HashMap; 33 | import java.util.List; 34 | import java.util.Map; 35 | 36 | /** 37 | * Mock for Storm's TopologyContext.. 38 | */ 39 | public class MockTopologyContext extends TopologyContext { 40 | 41 | public Map mockRegisteredMetrics = new HashMap<>(); 42 | public int taskId = 0; 43 | public int taskIndex = 0; 44 | public List componentTasks = Collections.singletonList(1); 45 | 46 | /** 47 | * Mock for Storm's TopologyContext.. 48 | */ 49 | public MockTopologyContext() { 50 | super(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); 51 | } 52 | 53 | @Override 54 | public T registerMetric(String name, T metric, int timeBucketSizeInSecs) { 55 | mockRegisteredMetrics.put(name, metric); 56 | return metric; 57 | } 58 | 59 | @Override 60 | public IMetric getRegisteredMetricByName(String name) { 61 | return mockRegisteredMetrics.get(name); 62 | } 63 | 64 | @Override 65 | public int getThisTaskId() { 66 | return taskId; 67 | } 68 | 69 | public String getThisComponentId() { 70 | return "Mock"; 71 | } 72 | 73 | public List getComponentTasks(String componentId) { 74 | return componentTasks; 75 | } 76 | 77 | public int getThisTaskIndex() { 78 | return taskIndex; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/mocks/output/MockSpoutOutputCollector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.mocks.output; 27 | 28 | import org.apache.storm.spout.ISpoutOutputCollector; 29 | import org.apache.storm.spout.SpoutOutputCollector; 30 | 31 | import java.util.ArrayList; 32 | import java.util.Collections; 33 | import java.util.List; 34 | 35 | /** 36 | * Extension of SpoutOutputCollector to aide in testing. 37 | */ 38 | public class MockSpoutOutputCollector extends SpoutOutputCollector { 39 | /** 40 | * This contains all of the Tuples that were 'emitted' to our MockOutputCollector. 41 | */ 42 | private final List emissions = new ArrayList<>(); 43 | 44 | /** 45 | * This contains any errors that were reported. 46 | */ 47 | private final List reportedErrors = new ArrayList<>(); 48 | 49 | 50 | public MockSpoutOutputCollector() { 51 | super(null); 52 | } 53 | 54 | /** 55 | * Not used, but here to comply to SpoutOutputCollector interface. 56 | * @param delegate not used. 57 | */ 58 | public MockSpoutOutputCollector(final ISpoutOutputCollector delegate) { 59 | super(delegate); 60 | } 61 | 62 | @Override 63 | public List emit(String streamId, List tuple, Object messageId) { 64 | emissions.add(new SpoutEmission(messageId, streamId, tuple)); 65 | return new ArrayList<>(); 66 | } 67 | 68 | @Override 69 | public void emitDirect(int taskId, String streamId, List tuple, Object messageId) { 70 | emissions.add(new SpoutEmission(messageId, streamId, tuple, taskId)); 71 | } 72 | 73 | @Override 74 | public long getPendingCount() { 75 | // Dunno? Nothing yet. 76 | return 0; 77 | } 78 | 79 | @Override 80 | public void reportError(final Throwable error) { 81 | reportedErrors.add(error); 82 | } 83 | 84 | /** 85 | * Get a clone of our Emissions in an unmodifiable list. 86 | * @return clone of our Emissions in an unmodifiable list 87 | */ 88 | public List getEmissions() { 89 | return Collections.unmodifiableList(emissions); 90 | } 91 | 92 | /** 93 | * Get a clone of reported errors in an unmodifiable list. 94 | * @return clone of reported errors in an unmodifiable list 95 | */ 96 | public List getReportedErrors() { 97 | return Collections.unmodifiableList(reportedErrors); 98 | } 99 | 100 | /** 101 | * Resets the internal state of our mock output collector. 102 | */ 103 | public void reset() { 104 | emissions.clear(); 105 | reportedErrors.clear(); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/mocks/output/SpoutEmission.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.mocks.output; 27 | 28 | import java.util.List; 29 | 30 | /** 31 | * Wrapper for tracking emissions from a {@link com.salesforce.storm.spout.dynamic.DynamicSpout}. 32 | */ 33 | public class SpoutEmission { 34 | private final Object messageId; 35 | private final String streamId; 36 | private final List tuple; 37 | private final Integer taskId; 38 | 39 | /** 40 | * Wrapper for tracking emissions from a {@link com.salesforce.storm.spout.dynamic.DynamicSpout}. 41 | * @param messageId message id. 42 | * @param streamId stream id. 43 | * @param tuple tuple. 44 | */ 45 | public SpoutEmission(Object messageId, String streamId, List tuple) { 46 | this(messageId, streamId, tuple, null); 47 | } 48 | 49 | /** 50 | * Wrapper for tracking emissions from a {@link com.salesforce.storm.spout.dynamic.DynamicSpout}. 51 | * @param messageId message id. 52 | * @param streamId stream id. 53 | * @param tuple tuple. 54 | * @param taskId task id. 55 | */ 56 | public SpoutEmission(Object messageId, String streamId, List tuple, Integer taskId) { 57 | this.messageId = messageId; 58 | this.streamId = streamId; 59 | this.tuple = tuple; 60 | this.taskId = taskId; 61 | } 62 | 63 | public Object getMessageId() { 64 | return messageId; 65 | } 66 | 67 | public String getStreamId() { 68 | return streamId; 69 | } 70 | 71 | public List getTuple() { 72 | return tuple; 73 | } 74 | 75 | public Integer getTaskId() { 76 | return taskId; 77 | } 78 | 79 | @Override 80 | public String toString() { 81 | return "SpoutEmission{" 82 | + "messageId=" + messageId 83 | + ", streamId='" + streamId + '\'' 84 | + ", tuple=" + tuple 85 | + ", taskId=" + taskId 86 | + '}'; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/retry/NeverRetryManagerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.retry; 27 | 28 | import com.salesforce.storm.spout.dynamic.MessageId; 29 | import com.salesforce.storm.spout.dynamic.DefaultVirtualSpoutIdentifier; 30 | import org.junit.jupiter.api.Test; 31 | 32 | import java.util.HashMap; 33 | 34 | import static org.junit.jupiter.api.Assertions.assertFalse; 35 | import static org.junit.jupiter.api.Assertions.assertNull; 36 | 37 | /** 38 | * Test over the No Retry Failed Msg manager. 39 | */ 40 | public class NeverRetryManagerTest { 41 | 42 | /** 43 | * Mostly for lame test coverage. 44 | */ 45 | @Test 46 | public void testShouldReEmitMsg() { 47 | // Create instance. 48 | NeverRetryManager retryManager = new NeverRetryManager(); 49 | 50 | final DefaultVirtualSpoutIdentifier consumerId1 = new DefaultVirtualSpoutIdentifier("ConsumerId1"); 51 | final DefaultVirtualSpoutIdentifier consumerId2 = new DefaultVirtualSpoutIdentifier("ConsumerId2"); 52 | final DefaultVirtualSpoutIdentifier consumerId3 = new DefaultVirtualSpoutIdentifier("ConsumerId3"); 53 | final DefaultVirtualSpoutIdentifier consumerId4 = new DefaultVirtualSpoutIdentifier("ConsumerId4"); 54 | 55 | retryManager.open(new HashMap<>()); 56 | 57 | // retryFurther always returns false 58 | assertFalse(retryManager.retryFurther(new MessageId("MyTopic1", 1, 1L, consumerId1))); 59 | assertFalse(retryManager.retryFurther(new MessageId("MyTopic2", 2, 2L, consumerId2))); 60 | assertFalse(retryManager.retryFurther(new MessageId("MyTopic3", 3, 3L, consumerId3))); 61 | assertFalse(retryManager.retryFurther(new MessageId("MyTopic4", 4, 4L, consumerId4))); 62 | 63 | // Call fail 64 | retryManager.failed(new MessageId("MyTopic1", 1, 1L, consumerId1)); 65 | retryManager.failed(new MessageId("MyTopic2", 2, 2L, consumerId2)); 66 | retryManager.failed(new MessageId("MyTopic3", 3, 3L, consumerId3)); 67 | retryManager.failed(new MessageId("MyTopic4", 4, 4L, consumerId4)); 68 | 69 | // nextFailedMessageToRetry should always return null. 70 | assertNull(retryManager.nextFailedMessageToRetry()); 71 | assertNull(retryManager.nextFailedMessageToRetry()); 72 | assertNull(retryManager.nextFailedMessageToRetry()); 73 | assertNull(retryManager.nextFailedMessageToRetry()); 74 | 75 | // Call acked started 76 | retryManager.acked(new MessageId("MyTopic1", 1, 1L, consumerId1)); 77 | retryManager.acked(new MessageId("MyTopic2", 2, 2L, consumerId2)); 78 | retryManager.acked(new MessageId("MyTopic3", 3, 3L, consumerId3)); 79 | retryManager.acked(new MessageId("MyTopic4", 4, 4L, consumerId4)); 80 | } 81 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/dynamic/test/TestHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.dynamic.test; 27 | 28 | import com.salesforce.storm.spout.dynamic.FactoryManager; 29 | import com.salesforce.storm.spout.dynamic.VirtualSpout; 30 | import com.salesforce.storm.spout.dynamic.VirtualSpoutIdentifier; 31 | import com.salesforce.storm.spout.dynamic.consumer.ConsumerPeerContext; 32 | import com.salesforce.storm.spout.dynamic.consumer.ConsumerState; 33 | import com.salesforce.storm.spout.dynamic.metrics.LogRecorder; 34 | 35 | import java.util.Map; 36 | 37 | // TODO: Use the VirtualSpoutFactory in these methods 38 | 39 | /** 40 | * Helper for creating things in tests. 41 | */ 42 | public class TestHelper { 43 | 44 | /** 45 | * Create a {@link VirtualSpout} and use sane defaults for most things. 46 | * @param config spout configuration. 47 | * @param identifier identifier of the {@link VirtualSpout} 48 | * @return a {@link VirtualSpout} instance. 49 | */ 50 | public static VirtualSpout createVirtualSpout( 51 | final Map config, 52 | final VirtualSpoutIdentifier identifier 53 | ) { 54 | return createVirtualSpout(config, identifier, null, null); 55 | } 56 | 57 | /** 58 | * Create a {@link VirtualSpout} and use sane defaults for most things. 59 | * @param config spout configuration. 60 | * @param identifier identifier of the {@link VirtualSpout} 61 | * @param startingState state for the consumer to start at. 62 | * @param endingState state for the consumer to end at. 63 | * @return a {@link VirtualSpout} instance. 64 | */ 65 | public static VirtualSpout createVirtualSpout( 66 | final Map config, 67 | final VirtualSpoutIdentifier identifier, 68 | final ConsumerState startingState, 69 | final ConsumerState endingState 70 | ) { 71 | final FactoryManager factoryManager = new FactoryManager(config); 72 | 73 | return new VirtualSpout( 74 | identifier, 75 | config, 76 | new ConsumerPeerContext(1, 0), 77 | factoryManager, 78 | new LogRecorder(), 79 | startingState, 80 | endingState 81 | ); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/sideline/SidelineVirtualSpoutIdentifierTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline; 27 | 28 | import com.salesforce.storm.spout.dynamic.VirtualSpoutIdentifier; 29 | import com.salesforce.storm.spout.sideline.trigger.SidelineRequestIdentifier; 30 | import org.junit.jupiter.api.Test; 31 | 32 | import static org.junit.jupiter.api.Assertions.assertEquals; 33 | import static org.junit.jupiter.api.Assertions.assertTrue; 34 | import static org.junit.jupiter.api.Assertions.assertFalse; 35 | 36 | /** 37 | * Test {@link SidelineVirtualSpoutIdentifier} work correctly. 38 | */ 39 | public class SidelineVirtualSpoutIdentifierTest { 40 | 41 | /** 42 | * Test that toString creates correct identifiers. 43 | * @throws Exception Null or empty data provided. 44 | */ 45 | @Test 46 | public void test_toString() throws Exception { 47 | VirtualSpoutIdentifier virtualSpoutIdentifier1 = new SidelineVirtualSpoutIdentifier( 48 | "foo", 49 | new SidelineRequestIdentifier("bar") 50 | ); 51 | 52 | // Note that the identifier inserts 'sideline' in the middle of the consumer id and the request identifier 53 | assertEquals("foo:sideline:bar", virtualSpoutIdentifier1.toString()); 54 | 55 | VirtualSpoutIdentifier virtualSpoutIdentifier2 = new SidelineVirtualSpoutIdentifier( 56 | "foo", 57 | new SidelineRequestIdentifier("baz") 58 | ); 59 | 60 | // Note that the identifier inserts 'sideline' in the middle of the consumer id and the request identifier 61 | assertEquals("foo:sideline:baz", virtualSpoutIdentifier2.toString()); 62 | } 63 | 64 | /** 65 | *Test that equals checks correctly. 66 | * @throws Exception Null or empty data provided. 67 | */ 68 | @Test 69 | public void test_equals() throws Exception { 70 | VirtualSpoutIdentifier virtualSpoutIdentifier1 = new SidelineVirtualSpoutIdentifier( 71 | "foo", 72 | new SidelineRequestIdentifier("main") 73 | ); 74 | 75 | VirtualSpoutIdentifier virtualSpoutIdentifier2 = new SidelineVirtualSpoutIdentifier( 76 | "foo", 77 | new SidelineRequestIdentifier("main") 78 | ); 79 | 80 | VirtualSpoutIdentifier virtualSpoutIdentifier3 = new SidelineVirtualSpoutIdentifier( 81 | "foo", 82 | new SidelineRequestIdentifier("bar") 83 | ); 84 | 85 | assertTrue(virtualSpoutIdentifier1.equals(virtualSpoutIdentifier2)); 86 | 87 | assertFalse(virtualSpoutIdentifier1.equals(virtualSpoutIdentifier3));; 88 | } 89 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/sideline/recipes/trigger/KeyFilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.recipes.trigger; 27 | 28 | import com.salesforce.storm.spout.dynamic.DefaultVirtualSpoutIdentifier; 29 | import com.salesforce.storm.spout.dynamic.Message; 30 | import com.salesforce.storm.spout.dynamic.MessageId; 31 | import org.apache.storm.tuple.Values; 32 | import org.junit.jupiter.api.Test; 33 | 34 | 35 | import java.util.Collections; 36 | 37 | import static org.junit.jupiter.api.Assertions.assertFalse; 38 | import static org.junit.jupiter.api.Assertions.assertTrue; 39 | 40 | /** 41 | * Test that messages are filtered by their keys. 42 | */ 43 | class KeyFilterTest { 44 | 45 | /** 46 | * Test that messages are filtered by their keys. 47 | */ 48 | @Test 49 | void testFilter() { 50 | final KeyFilter keyFilter = new KeyFilter(Collections.singletonList("key1")); 51 | 52 | final MessageId messageId1 = new MessageId("namespace", 0, 0, new DefaultVirtualSpoutIdentifier("id")); 53 | final Message message1 = new Message(messageId1, new Values("key1", "value1")); 54 | 55 | assertTrue(keyFilter.filter(message1), "Message should be filtered"); 56 | 57 | final MessageId messageId2 = new MessageId("namespace", 0, 1, new DefaultVirtualSpoutIdentifier("id")); 58 | final Message message2 = new Message(messageId1, new Values("key2", "value2")); 59 | 60 | assertFalse(keyFilter.filter(message2), "Message should not be filterd"); 61 | 62 | } 63 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/sideline/trigger/SidelineRequestTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.trigger; 27 | 28 | import com.salesforce.storm.spout.dynamic.JSON; 29 | import com.salesforce.storm.spout.dynamic.filter.FilterChainStep; 30 | import com.salesforce.storm.spout.dynamic.filter.StaticMessageFilter; 31 | import com.salesforce.storm.spout.sideline.config.SidelineConfig; 32 | import org.junit.jupiter.api.Test; 33 | 34 | import java.util.HashMap; 35 | import java.util.Map; 36 | 37 | import static org.junit.jupiter.api.Assertions.assertEquals; 38 | import static org.junit.jupiter.api.Assertions.assertNotNull; 39 | 40 | /** 41 | * Test the behavior of a SidelineRequest object. 42 | */ 43 | class SidelineRequestTest { 44 | 45 | /** 46 | * Test that a sideline request with a FilterChainStep instance can serialize & deserialize properly. 47 | */ 48 | @Test 49 | void testSerializingAndDeserializing() { 50 | final Map config = new HashMap<>(); 51 | config.put(SidelineConfig.FILTER_CHAIN_STEP_CLASS, StaticMessageFilter.class.getName()); 52 | 53 | final JSON json = new JSON(config); 54 | 55 | final FilterChainStep filterChainStep = new StaticMessageFilter("fixedid"); 56 | final SidelineRequest sidelineRequest = new SidelineRequest( 57 | new SidelineRequestIdentifier("foo"), 58 | filterChainStep 59 | ); 60 | 61 | final String data = json.to(sidelineRequest); 62 | 63 | assertNotNull(data, "JSON data should not be null after serializing"); 64 | 65 | final SidelineRequest sidelineRequestRefresh = json.from(data, SidelineRequest.class); 66 | 67 | assertEquals(sidelineRequest, sidelineRequestRefresh, "Sideline request should match after deserializing"); 68 | } 69 | } -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/sideline/trigger/SidelineTypeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.trigger; 27 | 28 | import org.junit.jupiter.api.Test; 29 | 30 | import static org.junit.jupiter.api.Assertions.assertEquals; 31 | import static org.junit.jupiter.api.Assertions.assertThrows; 32 | 33 | /** 34 | * Test that helper methods on {@link SidelineType} work. 35 | */ 36 | class SidelineTypeTest { 37 | 38 | /** 39 | * Test that {@link SidelineType#fromValue(String)} handles strings to type in a case insensitive manner. 40 | */ 41 | @Test 42 | void testFromValue() { 43 | assertEquals( 44 | SidelineType.START, 45 | SidelineType.fromValue("start") 46 | ); 47 | 48 | assertEquals( 49 | SidelineType.START, 50 | SidelineType.fromValue("START") 51 | ); 52 | 53 | assertEquals( 54 | SidelineType.START, 55 | SidelineType.fromValue("sTarT") 56 | ); 57 | 58 | assertEquals( 59 | SidelineType.RESOLVE, 60 | SidelineType.fromValue("resolve") 61 | ); 62 | 63 | assertEquals( 64 | SidelineType.RESUME, 65 | SidelineType.fromValue("resume") 66 | ); 67 | } 68 | 69 | /** 70 | * Test that {@link SidelineType#fromValue(String)} throws an exception for strings that don't map to types. 71 | */ 72 | @Test 73 | void testFromUndefineValue() { 74 | assertThrows(IllegalArgumentException.class, () -> 75 | SidelineType.fromValue("notRealSidelineType") 76 | ); 77 | } 78 | 79 | /** 80 | * Test that {@link SidelineType#fromValue(String)} throws an exception for null. 81 | */ 82 | @Test 83 | void testFromNull() { 84 | assertThrows(IllegalArgumentException.class, () -> 85 | SidelineType.fromValue(null) 86 | ); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/test/java/com/salesforce/storm/spout/sideline/trigger/StaticTrigger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017, 2018, Salesforce.com, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 6 | * following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, this list of conditions and the following 9 | * disclaimer. 10 | * 11 | * * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided with the distribution. 13 | * 14 | * * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products 15 | * derived from this software without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 23 | * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | */ 25 | 26 | package com.salesforce.storm.spout.sideline.trigger; 27 | 28 | import com.salesforce.storm.spout.sideline.handler.SidelineController; 29 | 30 | import java.util.Map; 31 | 32 | /** 33 | * Static trigger, you should only ever use this for testing. 34 | */ 35 | public class StaticTrigger implements SidelineTrigger { 36 | 37 | /** 38 | * Sideline controller instance. 39 | */ 40 | private static SidelineController sidelineController; 41 | 42 | /** 43 | * Open the trigger. 44 | * @param config spout configuration. 45 | */ 46 | @Override 47 | public void open(final Map config) { 48 | } 49 | 50 | /** 51 | * Close the trigger. 52 | */ 53 | @Override 54 | public void close() { 55 | } 56 | 57 | /** 58 | * Set the sideline controller instance on this trigger. 59 | * @param sidelineController sideline controller instance. 60 | */ 61 | @Override 62 | public void setSidelineController(SidelineController sidelineController) { 63 | StaticTrigger.sidelineController = sidelineController; 64 | } 65 | 66 | /** 67 | * Send a start sideline request. 68 | * @param request sideline request. 69 | */ 70 | public static void sendStartRequest(final SidelineRequest request) { 71 | StaticTrigger.sidelineController.start(request); 72 | } 73 | 74 | /** 75 | * Send a resume sideline request. 76 | * @param request sideline request. 77 | */ 78 | public static void sendResumeRequest(final SidelineRequest request) { 79 | StaticTrigger.sidelineController.resume(request); 80 | } 81 | 82 | /** 83 | * Send a resolve sideline request. 84 | * @param request sideline request. 85 | */ 86 | public static void sendResolveRequest(final SidelineRequest request) { 87 | StaticTrigger.sidelineController.resolve(request); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | --------------------------------------------------------------------------------