├── .gitignore ├── DISCLAIMER ├── LICENSE ├── NOTICE.txt ├── README.md ├── RELEASE_NOTES.html ├── build.gradle ├── config ├── binrelease │ ├── LICENSE │ └── NOTICE.txt ├── checkstyle │ └── s4-checkstyle.xml └── eclipse │ └── s4-eclipse-format.xml ├── gradlew ├── gradlew.bat ├── lib ├── gradle-wrapper-1.0.jar └── gradle-wrapper-1.0.properties ├── s4 ├── settings.gradle ├── subprojects ├── s4-base │ ├── s4-base.gradle │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── s4 │ │ └── base │ │ ├── Emitter.java │ │ ├── Event.java │ │ ├── EventMessage.java │ │ ├── GenericKeyFinder.java │ │ ├── Hasher.java │ │ ├── Key.java │ │ ├── KeyFinder.java │ │ ├── Listener.java │ │ ├── RemoteEmitter.java │ │ ├── SerializerDeserializer.java │ │ ├── package-info.java │ │ └── util │ │ ├── S4RLoader.java │ │ ├── S4RLoaderFactory.java │ │ └── package-info.java ├── s4-comm │ ├── s4-comm.gradle │ └── src │ │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── s4 │ │ │ │ └── comm │ │ │ │ ├── DefaultCommModule.java │ │ │ │ ├── DefaultHasher.java │ │ │ │ ├── RemoteEmitterFactory.java │ │ │ │ ├── package-info.java │ │ │ │ ├── serialize │ │ │ │ ├── KryoSerDeser.java │ │ │ │ └── package-info.java │ │ │ │ ├── tcp │ │ │ │ ├── RemoteEmitters.java │ │ │ │ ├── TCPEmitter.java │ │ │ │ ├── TCPListener.java │ │ │ │ ├── TCPRemoteEmitter.java │ │ │ │ └── package-info.java │ │ │ │ ├── tools │ │ │ │ ├── TaskSetup.java │ │ │ │ └── package-info.java │ │ │ │ ├── topology │ │ │ │ ├── Assignment.java │ │ │ │ ├── AssignmentFromZK.java │ │ │ │ ├── Cluster.java │ │ │ │ ├── ClusterChangeListener.java │ │ │ │ ├── ClusterFromZK.java │ │ │ │ ├── ClusterNode.java │ │ │ │ ├── Clusters.java │ │ │ │ ├── ClustersFromZK.java │ │ │ │ ├── PhysicalCluster.java │ │ │ │ ├── RemoteCluster.java │ │ │ │ ├── RemoteStreams.java │ │ │ │ ├── StreamConsumer.java │ │ │ │ ├── ZNRecord.java │ │ │ │ ├── ZNRecordSerializer.java │ │ │ │ ├── ZkClient.java │ │ │ │ └── package-info.java │ │ │ │ └── udp │ │ │ │ ├── UDPEmitter.java │ │ │ │ ├── UDPListener.java │ │ │ │ ├── UDPRemoteEmitter.java │ │ │ │ └── package-info.java │ │ └── resources │ │ │ └── default.s4.comm.properties │ │ └── test │ │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── s4 │ │ │ ├── comm │ │ │ ├── DeliveryTestUtil.java │ │ │ ├── tcp │ │ │ │ ├── MultiPartitionDeliveryTest.java │ │ │ │ ├── SimpleDeliveryTest.java │ │ │ │ └── TCPCommTest.java │ │ │ ├── topology │ │ │ │ ├── AssignmentsFromZKTest1.java │ │ │ │ ├── AssignmentsFromZKTest2.java │ │ │ │ ├── ClustersFromZKTest.java │ │ │ │ ├── ZKBaseTest.java │ │ │ │ └── ZNRecordSerializerTest.java │ │ │ ├── udp │ │ │ │ ├── MultiPartitionDeliveryTest.java │ │ │ │ ├── SimpleDeliveryTest.java │ │ │ │ └── UDPCommTest.java │ │ │ └── util │ │ │ │ ├── PartitionInfo.java │ │ │ │ └── ProtocolTestUtil.java │ │ │ └── fixtures │ │ │ ├── CommTestUtils.java │ │ │ └── ZkBasedTest.java │ │ └── resources │ │ └── udp.s4.comm.properties ├── s4-core │ ├── s4-core.gradle │ └── src │ │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── s4 │ │ │ │ ├── core │ │ │ │ ├── App.java │ │ │ │ ├── DefaultCoreModule.java │ │ │ │ ├── Key.java │ │ │ │ ├── Main.java │ │ │ │ ├── ProcessingElement.java │ │ │ │ ├── Receiver.java │ │ │ │ ├── RemoteSender.java │ │ │ │ ├── RemoteSenders.java │ │ │ │ ├── RemoteStream.java │ │ │ │ ├── Sender.java │ │ │ │ ├── Server.java │ │ │ │ ├── Stream.java │ │ │ │ ├── Streamable.java │ │ │ │ ├── adapter │ │ │ │ │ ├── AdapterApp.java │ │ │ │ │ └── package-info.java │ │ │ │ ├── ft │ │ │ │ │ ├── CheckpointId.java │ │ │ │ │ ├── CheckpointingConfig.java │ │ │ │ │ ├── CheckpointingFramework.java │ │ │ │ │ ├── CheckpointingTask.java │ │ │ │ │ ├── DefaultFileSystemStateStorage.java │ │ │ │ │ ├── FetchTask.java │ │ │ │ │ ├── FileSystemBackendCheckpointingModule.java │ │ │ │ │ ├── LoggingStorageCallbackFactory.java │ │ │ │ │ ├── NoOpCheckpointingFramework.java │ │ │ │ │ ├── SafeKeeper.java │ │ │ │ │ ├── SaveStateTask.java │ │ │ │ │ ├── SerializeTask.java │ │ │ │ │ ├── StateStorage.java │ │ │ │ │ ├── StorageCallback.java │ │ │ │ │ ├── StorageCallbackFactory.java │ │ │ │ │ └── package-info.java │ │ │ │ ├── gen │ │ │ │ │ ├── OverloadDispatcher.java │ │ │ │ │ ├── OverloadDispatcherGenerator.java │ │ │ │ │ └── package-info.java │ │ │ │ ├── package-info.java │ │ │ │ ├── util │ │ │ │ │ ├── ParametersInjectionModule.java │ │ │ │ │ └── package-info.java │ │ │ │ └── window │ │ │ │ │ ├── AbstractSlidingWindowPE.java │ │ │ │ │ ├── DefaultAggregatingSlot.java │ │ │ │ │ ├── OHCLSlot.java │ │ │ │ │ ├── Slot.java │ │ │ │ │ ├── SlotFactory.java │ │ │ │ │ └── package-info.java │ │ │ │ └── deploy │ │ │ │ ├── DeploymentFailedException.java │ │ │ │ ├── DeploymentManager.java │ │ │ │ ├── DistributedDeploymentManager.java │ │ │ │ ├── FileSystemS4RFetcher.java │ │ │ │ ├── HttpS4RFetcher.java │ │ │ │ ├── NoOpDeploymentManager.java │ │ │ │ ├── S4RFetcher.java │ │ │ │ └── package-info.java │ │ └── resources │ │ │ ├── default.s4.core.properties │ │ │ └── logback.xml │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── s4 │ │ ├── base │ │ ├── EventAttributeTest.java │ │ └── SimpleEvent.java │ │ ├── core │ │ ├── TestCircularFifoBuffer.java │ │ ├── TriggerTest.java │ │ ├── ft │ │ │ ├── CheckpointingModuleWithUnrespondingFetchingStorageBackend.java │ │ │ ├── CheckpointingTest.java │ │ │ ├── FTWordCountApp.java │ │ │ ├── FTWordCountTest.java │ │ │ ├── FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.java │ │ │ ├── RecoveryTest.java │ │ │ ├── S4AppWithCountBasedCheckpointing.java │ │ │ ├── S4AppWithManualCheckpointing.java │ │ │ ├── S4AppWithTimeBasedCheckpointing.java │ │ │ ├── StatefulTestPE.java │ │ │ └── StorageWithUnrespondingFetching.java │ │ ├── overloadgen │ │ │ ├── A.java │ │ │ ├── B.java │ │ │ ├── C.java │ │ │ ├── D.java │ │ │ ├── Event1.java │ │ │ ├── Event1a.java │ │ │ ├── Event2.java │ │ │ └── OverloadDispatcherTest.java │ │ ├── timers │ │ │ └── MultithreadingTest.java │ │ ├── triggers │ │ │ ├── CountTriggerTest.java │ │ │ ├── NoTriggerTest.java │ │ │ ├── TimeTriggerTest.java │ │ │ ├── TriggerablePE.java │ │ │ └── TriggeredApp.java │ │ └── windowing │ │ │ ├── WindowingPE1.java │ │ │ └── WindowingPETest.java │ │ ├── deploy │ │ ├── AppConstants.java │ │ ├── TestAutomaticDeployment.java │ │ └── prodcon │ │ │ └── TestProducerConsumer.java │ │ ├── fixtures │ │ ├── AssignmentFromZKNoFailFast.java │ │ ├── ClusterFromZKNoFailFast.java │ │ ├── ClustersFromZKNoFailFast.java │ │ ├── CoreTestUtils.java │ │ ├── MockCommModule.java │ │ ├── MockCoreModule.java │ │ └── NonFailFastZookeeperClientsModule.java │ │ └── wordcount │ │ ├── KeyValueEvent.java │ │ ├── KeyValueKeyFinder.java │ │ ├── SentenceKeyFinder.java │ │ ├── StringEvent.java │ │ ├── Word.java │ │ ├── WordClassifierPE.java │ │ ├── WordCountApp.java │ │ ├── WordCountEvent.java │ │ ├── WordCountKeyFinder.java │ │ ├── WordCountModule.java │ │ ├── WordCountTest.java │ │ ├── WordCounterPE.java │ │ ├── WordSeenEvent.java │ │ ├── WordSeenKeyFinder.java │ │ └── WordSplitterPE.java ├── s4-edsl │ ├── s4-edsl.gradle │ └── src │ │ ├── main │ │ ├── diezel │ │ │ └── s4 │ │ │ │ ├── s4-impl.xml │ │ │ │ └── s4.xml │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── s4 │ │ │ └── edsl │ │ │ ├── AppBuilder.java │ │ │ └── StreamBuilder.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── s4 │ │ └── edsl │ │ ├── DurationKeyFinder.java │ │ ├── EventA.java │ │ ├── EventB.java │ │ ├── HeightKeyFinder.java │ │ ├── MyApp.java │ │ ├── PEX.java │ │ ├── PEY.java │ │ ├── PEZ.java │ │ ├── QueryKeyFinder.java │ │ └── TestEDSL.java ├── s4-example │ ├── s4-example.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── s4 │ │ │ └── example │ │ │ ├── counter │ │ │ ├── AgeKeyFinder.java │ │ │ ├── CountEvent.java │ │ │ ├── CountKeyFinder.java │ │ │ ├── CounterPE.java │ │ │ ├── GenderKeyFinder.java │ │ │ ├── GenerateUserEventPE.java │ │ │ ├── Module.java │ │ │ ├── MyApp.java │ │ │ ├── PrintPE.java │ │ │ ├── README.md │ │ │ ├── UserEvent.java │ │ │ └── UserIDKeyFinder.java │ │ │ └── edsl │ │ │ └── counter │ │ │ ├── AgeKeyFinder.java │ │ │ ├── CountEvent.java │ │ │ ├── CountKeyFinder.java │ │ │ ├── CounterApp.java │ │ │ ├── CounterPE.java │ │ │ ├── GenderKeyFinder.java │ │ │ ├── GenerateUserEventPE.java │ │ │ ├── PrintPE.java │ │ │ ├── UserEvent.java │ │ │ └── UserIDKeyFinder.java │ │ └── test │ │ └── io │ │ └── s4 │ │ └── model │ │ └── TestGaussianModel.java └── s4-tools │ ├── s4-tools.gradle │ └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── s4 │ │ └── tools │ │ ├── CreateApp.java │ │ ├── DefineCluster.java │ │ ├── Deploy.java │ │ ├── FileExistsValidator.java │ │ ├── Package.java │ │ ├── S4ArgsBase.java │ │ ├── Status.java │ │ ├── Tools.java │ │ ├── ZKServer.java │ │ └── package-info.java │ └── resources │ └── templates │ ├── HelloApp.java.txt │ ├── HelloInputAdapter.java.txt │ ├── HelloPE.java.txt │ ├── build.gradle │ ├── gradlew │ ├── newApp.README │ ├── s4 │ └── settings.gradle └── test-apps ├── consumer-app ├── build.gradle └── src │ └── main │ └── java │ └── s4app │ ├── ConsumerApp.java │ └── ConsumerPE.java ├── producer-app ├── build.gradle └── src │ └── main │ └── java │ └── s4app │ ├── ProducerApp.java │ └── ProducerPE.java ├── simple-deployable-app-1 ├── build.gradle └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── s4 │ │ └── deploy │ │ ├── A.java │ │ ├── AppConstants.java │ │ ├── SimplePE.java │ │ └── TestApp.java │ └── resources │ └── resource.txt ├── twitter-adapter ├── README.txt ├── build.gradle └── src │ └── main │ └── java │ └── org │ └── apache │ └── s4 │ └── example │ └── twitter │ └── TwitterInputAdapter.java └── twitter-counter ├── README.txt ├── build.gradle └── src └── main └── java └── org └── apache └── s4 └── example └── twitter ├── TopNTopicPE.java ├── TopicCountAndReportPE.java ├── TopicEvent.java ├── TopicExtractorPE.java └── TwitterCounterApp.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Eclipse meta-information 2 | .project 3 | .classpath 4 | .settings 5 | 6 | # Build directory 7 | target/ 8 | build/ 9 | bin/ 10 | doc/ 11 | tmp/ 12 | pepe/ 13 | 14 | # SVN 15 | .svn 16 | 17 | # Gradle files 18 | .gradle 19 | 20 | # Backup files 21 | *~ 22 | 23 | # Misc hidden files 24 | .DS_Store 25 | 26 | # Java 27 | *.class 28 | 29 | # apache RAT 30 | .rat-excludes -------------------------------------------------------------------------------- /DISCLAIMER: -------------------------------------------------------------------------------- 1 | Apache S4 is an effort undergoing incubation at the Apache Software 2 | Foundation (ASF), sponsored by the Apache Incubator PMC. 3 | 4 | Incubation is required of all newly accepted projects until a further review 5 | indicates that the infrastructure, communications, and decision making process 6 | have stabilized in a manner consistent with other successful ASF projects. 7 | 8 | While incubation status is not necessarily a reflection of the completeness 9 | or stability of the code, it does indicate that the project has yet to be 10 | fully endorsed by the ASF. 11 | 12 | For more information about the incubation status of the S4 project you 13 | can go to the following page: 14 | 15 | http://incubator.apache.org/s4 -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache S4 2 | Copyright 2012 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /config/binrelease/NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache S4 2 | Copyright 2012 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | This product includes concurrency annotations, 8 | developed by Brian Goetz and Tim Peierls (http://jcip.net/) -------------------------------------------------------------------------------- /lib/gradle-wrapper-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-retired-s4/53e0d34fb40df384efeb03d47c8a965f4bb4c0cc/lib/gradle-wrapper-1.0.jar -------------------------------------------------------------------------------- /lib/gradle-wrapper-1.0.properties: -------------------------------------------------------------------------------- 1 | #Sun Jul 01 12:59:07 CEST 2012 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-1.0-bin.zip 7 | -------------------------------------------------------------------------------- /s4: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # NOTE: "./gradlew s4-tools:installApp" will prepare/update the tools subproject and related startup scripts 4 | 5 | S4_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | S4_SCRIPT_PATH="$S4_DIR/s4" 7 | 8 | # JVM options for starting nodes and for other s4 tools can be configured here 9 | # export JAVA_OPTS=-Xmx1G 10 | 11 | subprojects/s4-tools/build/install/s4-tools/bin/s4-tools -s4ScriptPath=$S4_SCRIPT_PATH $@ 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | include 's4-base' 20 | include 's4-core' 21 | include 's4-comm' 22 | include 's4-edsl' 23 | include 's4-example' 24 | include 's4-tools' 25 | //include 's4-example' 26 | //include ':test-apps:simple-adapter-1' 27 | include ':test-apps:simple-deployable-app-1' 28 | include ':test-apps:producer-app' 29 | include ':test-apps:consumer-app' 30 | 31 | rootProject.name = 's4' 32 | rootProject.children.each {project -> 33 | if (project.name != 'test-apps') { 34 | String fileBaseName = project.name.replaceAll("\\p{Upper}") { "-${it.toLowerCase()}" } 35 | String projectDirName = "subprojects/$fileBaseName" 36 | project.projectDir = new File(settingsDir, projectDirName) 37 | project.buildFileName = "${fileBaseName}.gradle" 38 | assert project.projectDir.isDirectory() 39 | assert project.buildFile.isFile() 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-base/s4-base.gradle: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | description = 'Interfaces and most basic classes required by nultiple modules.' 21 | 22 | dependencies { 23 | } 24 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/Emitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | /** 22 | * Defines an event emitter, responsible for sending an event to a given partition of the cluster. 23 | * 24 | */ 25 | public interface Emitter { 26 | 27 | /** 28 | * @param partitionId 29 | * - destination partition 30 | * 31 | * @param message 32 | * - message payload that needs to be sent 33 | * 34 | * @return - true - if message is sent across successfully - false - if send fails 35 | */ 36 | boolean send(int partitionId, EventMessage message); 37 | 38 | int getPartitionCount(); 39 | 40 | void close(); 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/Hasher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | /** 22 | * Defines a transformation between a key and a hashcode. 23 | * 24 | */ 25 | public interface Hasher { 26 | long hash(String hashKey); 27 | } 28 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/Key.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | import java.util.List; 22 | 23 | import org.apache.commons.lang.StringUtils; 24 | 25 | /** 26 | * The Key class is used to get the value of the key on a specific type of event. This is done to abstract all the 27 | * complexity required to get the value. The method for getting the value is implemented in a method of an object of 28 | * type KeyFinder. 29 | * 30 | * The application programmer provides the events and the corresponding finders. The framework will use it to key on 31 | * events. 32 | */ 33 | public class Key { 34 | 35 | final private KeyFinder finder; 36 | final private String separator; 37 | 38 | public Key(KeyFinder finder, String separator) { 39 | this.finder = finder; 40 | this.separator = separator; 41 | } 42 | 43 | public List getList(T event) { 44 | return finder.get(event); 45 | } 46 | 47 | public String get(T event) { 48 | List keys = getList(event); 49 | 50 | return StringUtils.join(keys, separator); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/KeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | import java.util.List; 22 | 23 | /** 24 | * The KeyFinder returns a list with one or more String values (One value for single keys, and more than one value for 25 | * composite keys.) 26 | * */ 27 | public interface KeyFinder { 28 | 29 | public List get(T event); 30 | } 31 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/Listener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | /** 22 | * 23 | * Get a byte array received by a lower level layer. 24 | * 25 | */ 26 | public interface Listener { 27 | 28 | /** 29 | * Perform blocking receive on the appropriate communication channel 30 | * 31 | * @return
    32 | *
  • byte[] message returned by the channel
    • 33 | *
  • null if the associated blocking thread is interrupted
    • 34 | *
35 | */ 36 | byte[] recv(); 37 | 38 | public int getPartitionId(); 39 | 40 | void close(); 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/RemoteEmitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | /** 22 | * Marker interface for identifying emitters to remote S4 clusters. 23 | * 24 | */ 25 | public interface RemoteEmitter extends Emitter { 26 | 27 | } 28 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/SerializerDeserializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | /** 22 | * Defines serialization and deserialization methods used within the S4 platform, typically for events and PEs. 23 | * 24 | */ 25 | public interface SerializerDeserializer { 26 | public byte[] serialize(Object message); 27 | 28 | public Object deserialize(byte[] rawMessage); 29 | } 30 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Defines some of the basic elements of the S4 platforms. 21 | */ 22 | package org.apache.s4.base; -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/util/S4RLoader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base.util; 20 | 21 | import java.net.URL; 22 | import java.net.URLClassLoader; 23 | 24 | /** 25 | * A classloader that fetches and loads classes and resources from : 26 | *
    27 | *
  • Application classes in an S4R archive
    • 28 | *
  • Application dependencies from an S4R archive
    • 29 | *
  • Classes dynamically generated 30 | * 31 | */ 32 | public class S4RLoader extends URLClassLoader { 33 | 34 | public S4RLoader(URL[] urls) { 35 | super(urls); 36 | } 37 | 38 | public Class loadGeneratedClass(String name, byte[] bytes) { 39 | Class clazz = findLoadedClass(name); 40 | if (clazz == null) { 41 | return defineClass(name, bytes, 0, bytes.length); 42 | } else { 43 | return clazz; 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /subprojects/s4-base/src/main/java/org/apache/s4/base/util/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Utility classes, mostly related to classloading. 21 | */ 22 | package org.apache.s4.base.util; -------------------------------------------------------------------------------- /subprojects/s4-comm/s4-comm.gradle: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | description = 'Implementation-specific components of the communication layer.' 21 | 22 | dependencies { 23 | compile project(":s4-base") 24 | compile libraries.gson 25 | compile libraries.kryo 26 | compile libraries.netty 27 | compile (libraries.zkclient) 28 | } 29 | 30 | task testJar(type: Jar) { 31 | baseName = "test-${project.archivesBaseName}" 32 | from sourceSets.test.output 33 | } 34 | 35 | configurations { 36 | tests 37 | } 38 | 39 | artifacts { 40 | tests testJar 41 | } 42 | 43 | test { 44 | forkEvery=1 45 | } 46 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/DefaultHasher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm; 20 | 21 | import org.apache.s4.base.Hasher; 22 | 23 | /** 24 | * Simple hasher implementation. 25 | * 26 | */ 27 | public class DefaultHasher implements Hasher { 28 | 29 | @Override 30 | public long hash(String hashKey) { 31 | int b = 378551; 32 | int a = 63689; 33 | long hash = 0; 34 | 35 | for (int i = 0; i < hashKey.length(); i++) { 36 | hash = hash * a + hashKey.charAt(i); 37 | a = a * b; 38 | } 39 | 40 | return Math.abs(hash); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/RemoteEmitterFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm; 20 | 21 | import org.apache.s4.base.RemoteEmitter; 22 | import org.apache.s4.comm.topology.Cluster; 23 | 24 | /** 25 | * Used for creating RemoteEmitter instances depending on the cluster configuration. Follows the "assisted injection" 26 | * pattern from Guice 3. 27 | * 28 | */ 29 | public interface RemoteEmitterFactory { 30 | 31 | RemoteEmitter createRemoteEmitter(Cluster topology); 32 | 33 | } 34 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * This package and subpackages define mechanisms for low-level communications in the S4 platform. 21 | */ 22 | package org.apache.s4.comm; -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/serialize/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Serialization utilities for messages and PEs. 21 | */ 22 | package org.apache.s4.comm.serialize; -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/tcp/RemoteEmitters.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.tcp; 20 | 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | 24 | import org.apache.s4.base.RemoteEmitter; 25 | import org.apache.s4.comm.RemoteEmitterFactory; 26 | import org.apache.s4.comm.topology.Cluster; 27 | 28 | import com.google.inject.Inject; 29 | import com.google.inject.Singleton; 30 | 31 | /** 32 | * Manages the {@link RemoteEmitter} instances for sending messages to remote subclusters. 33 | * 34 | */ 35 | @Singleton 36 | public class RemoteEmitters { 37 | 38 | Map emitters = new HashMap(); 39 | 40 | @Inject 41 | RemoteEmitterFactory emitterFactory; 42 | 43 | public RemoteEmitter getEmitter(Cluster topology) { 44 | RemoteEmitter emitter = emitters.get(topology); 45 | if (emitter == null) { 46 | emitter = emitterFactory.createRemoteEmitter(topology); 47 | emitters.put(topology, emitter); 48 | } 49 | return emitter; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/tcp/TCPRemoteEmitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.tcp; 20 | 21 | import org.apache.s4.base.RemoteEmitter; 22 | import org.apache.s4.comm.topology.Cluster; 23 | 24 | import com.google.inject.Inject; 25 | import com.google.inject.assistedinject.Assisted; 26 | import com.google.inject.name.Named; 27 | 28 | /** 29 | * Emitter to remote subclusters. 30 | * 31 | */ 32 | public class TCPRemoteEmitter extends TCPEmitter implements RemoteEmitter { 33 | 34 | /** 35 | * Sends to remote subclusters. This is dynamically created, through an injected factory, when new subclusters are 36 | * discovered (as remote streams outputs) 37 | */ 38 | @Inject 39 | public TCPRemoteEmitter(@Assisted Cluster topology, @Named("s4.comm.timeout") int timeout) 40 | throws InterruptedException { 41 | super(topology, timeout); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/tcp/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * TCP-based communication layer implementation. 21 | */ 22 | package org.apache.s4.comm.tcp; -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/tools/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Utility classes. 21 | */ 22 | package org.apache.s4.comm.tools; -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/topology/Assignment.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | /** 22 | * 23 | * Upon startup an S4 process in a cluster must be assigned one and only one of the available cluster nodes. Cluster 24 | * nodes ({@link ClusterNode}) are defined using a configuration mechanism at startup. 25 | * 26 | * The Assignment implementation is responsible for coordinating how cluster nodes are uniquely assigned to processes. 27 | * 28 | */ 29 | public interface Assignment { 30 | 31 | /** 32 | * @return the ClusterNode associated assigned to this process. 33 | */ 34 | public ClusterNode assignClusterNode(); 35 | } 36 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/topology/Cluster.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | /** 22 | * Represents a logical cluster 23 | * 24 | */ 25 | 26 | public interface Cluster { 27 | public PhysicalCluster getPhysicalCluster(); 28 | 29 | public void addListener(ClusterChangeListener listener); 30 | 31 | public void removeListener(ClusterChangeListener listener); 32 | } 33 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/topology/ClusterChangeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | /** 22 | * Entities interested in changes occurring in topologies implement this listener and should register through the 23 | * {@link Cluster} interface 24 | * 25 | */ 26 | public interface ClusterChangeListener { 27 | public void onChange(); 28 | } 29 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/topology/ClusterNode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | /** 22 | * Represents an node. 23 | * 24 | */ 25 | public class ClusterNode { 26 | private int partition; 27 | private int port; 28 | private String machineName; 29 | private String taskId; 30 | 31 | public ClusterNode(int partition, int port, String machineName, String taskId) { 32 | this.partition = partition; 33 | this.port = port; 34 | this.machineName = machineName; 35 | this.taskId = taskId; 36 | } 37 | 38 | public int getPartition() { 39 | return partition; 40 | } 41 | 42 | public int getPort() { 43 | return port; 44 | } 45 | 46 | public String getMachineName() { 47 | return machineName; 48 | } 49 | 50 | public String getTaskId() { 51 | return taskId; 52 | } 53 | 54 | public String toString() { 55 | StringBuffer sb = new StringBuffer(); 56 | sb.append("{").append("partition=").append(partition).append(",port=").append(port).append(",machineName=") 57 | .append(machineName).append(",taskId=").append(taskId).append("}"); 58 | return sb.toString(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/topology/Clusters.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | /** 22 | * Represents clusters related to the current node (clusters to which this node belongs, and connected clusters that may 23 | * receive messages from this node) 24 | * 25 | */ 26 | public interface Clusters { 27 | 28 | Cluster getCluster(String clusterName); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/topology/RemoteCluster.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | /** 22 | * 23 | * Represents a logical cluster external to the current cluster 24 | * 25 | */ 26 | public interface RemoteCluster extends Cluster { 27 | 28 | } 29 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/topology/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Clusters configurations and partitions assignments to cluster nodes through Zookeeper. 21 | */ 22 | package org.apache.s4.comm.topology; -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/udp/UDPRemoteEmitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.udp; 20 | 21 | import org.apache.s4.comm.topology.Cluster; 22 | 23 | import com.google.inject.Inject; 24 | import com.google.inject.assistedinject.Assisted; 25 | 26 | /** 27 | * UDP-based emitter for sending events to remote clusters. 28 | * 29 | */ 30 | public class UDPRemoteEmitter extends UDPEmitter { 31 | 32 | /** 33 | * Sends to remote subclusters. This is dynamically created, through an injected factory, when new subclusters are 34 | * discovered (as remote streams outputs) 35 | */ 36 | @Inject 37 | public UDPRemoteEmitter(@Assisted Cluster topology) throws InterruptedException { 38 | super(topology); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/java/org/apache/s4/comm/udp/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * UDP-based implementation of the communication layer. 21 | */ 22 | package org.apache.s4.comm.udp; -------------------------------------------------------------------------------- /subprojects/s4-comm/src/main/resources/default.s4.comm.properties: -------------------------------------------------------------------------------- 1 | s4.comm.emitter.class=org.apache.s4.comm.tcp.TCPEmitter 2 | s4.comm.emitter.remote.class=org.apache.s4.comm.tcp.TCPRemoteEmitter 3 | s4.comm.listener.class=org.apache.s4.comm.tcp.TCPListener 4 | # I/O channel connection timeout, when applicable (e.g. used by netty) 5 | s4.comm.timeout=1000 6 | s4.cluster.zk_address = localhost:2181 7 | s4.cluster.zk_session_timeout = 10000 8 | s4.cluster.zk_connection_timeout = 10000 -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/java/org/apache/s4/comm/tcp/MultiPartitionDeliveryTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.tcp; 20 | 21 | import java.io.IOException; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | public class MultiPartitionDeliveryTest extends TCPCommTest { 27 | 28 | private static Logger logger = LoggerFactory.getLogger(MultiPartitionDeliveryTest.class); 29 | 30 | public MultiPartitionDeliveryTest() throws IOException { 31 | super(6); 32 | logger = LoggerFactory.getLogger(MultiPartitionDeliveryTest.class); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/java/org/apache/s4/comm/tcp/SimpleDeliveryTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.tcp; 20 | 21 | import java.io.IOException; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | public class SimpleDeliveryTest extends TCPCommTest { 27 | 28 | private static Logger logger = LoggerFactory.getLogger(SimpleDeliveryTest.class); 29 | 30 | public SimpleDeliveryTest() throws IOException { 31 | super(); 32 | logger = LoggerFactory.getLogger(SimpleDeliveryTest.class); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/java/org/apache/s4/comm/topology/AssignmentsFromZKTest2.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | import org.apache.s4.comm.tools.TaskSetup; 22 | import org.apache.s4.fixtures.CommTestUtils; 23 | import org.apache.s4.fixtures.ZkBasedTest; 24 | import org.junit.Test; 25 | 26 | /** 27 | * Separated from AssignmentsFromZKTest1 so that VM exit upon Zookeeper connection expiration does not affect the test 28 | * in that other class. 29 | * 30 | */ 31 | public class AssignmentsFromZKTest2 extends ZkBasedTest { 32 | 33 | @Test 34 | public void testAssignmentFor2Clusters() throws Exception { 35 | Thread.sleep(2000); 36 | TaskSetup taskSetup = new TaskSetup(CommTestUtils.ZK_STRING); 37 | final String topologyNames = "cluster2, cluster3"; 38 | AssignmentsFromZKTest1.testAssignment(taskSetup, topologyNames); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/java/org/apache/s4/comm/topology/ZKBaseTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.s4.fixtures.CommTestUtils; 24 | import org.apache.zookeeper.KeeperException; 25 | import org.apache.zookeeper.server.NIOServerCnxn.Factory; 26 | import org.junit.After; 27 | import org.junit.Before; 28 | 29 | public class ZKBaseTest { 30 | 31 | private Factory zkFactory; 32 | 33 | @Before 34 | public void setUp() throws IOException, InterruptedException, KeeperException { 35 | CommTestUtils.cleanupTmpDirs(); 36 | zkFactory = CommTestUtils.startZookeeperServer(); 37 | 38 | } 39 | 40 | @After 41 | public void tearDown() throws Exception { 42 | CommTestUtils.stopZookeeperServer(zkFactory); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/java/org/apache/s4/comm/topology/ZNRecordSerializerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.topology; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | public class ZNRecordSerializerTest { 25 | 26 | @Test 27 | public void testSerDeser() { 28 | 29 | ZNRecordSerializer serializer = new ZNRecordSerializer(); 30 | ZNRecord znRecord = new ZNRecord("test"); 31 | byte[] serialize = serializer.serialize(znRecord); 32 | System.out.println(new String(serialize)); 33 | 34 | ZNRecord newZNRecord = (ZNRecord) serializer.deserialize(serialize); 35 | System.out.println(newZNRecord.getId()); 36 | Assert.assertEquals(znRecord, newZNRecord); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/java/org/apache/s4/comm/udp/MultiPartitionDeliveryTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.udp; 20 | 21 | import java.io.IOException; 22 | 23 | public class MultiPartitionDeliveryTest extends UDPCommTest { 24 | public MultiPartitionDeliveryTest() throws IOException { 25 | super(2); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/java/org/apache/s4/comm/udp/SimpleDeliveryTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.comm.udp; 20 | 21 | import java.io.IOException; 22 | 23 | public class SimpleDeliveryTest extends UDPCommTest { 24 | public SimpleDeliveryTest() throws IOException { 25 | super(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /subprojects/s4-comm/src/test/resources/udp.s4.comm.properties: -------------------------------------------------------------------------------- 1 | s4.comm.emitter.class = org.apache.s4.comm.udp.UDPEmitter 2 | s4.comm.emitter.remote.class = org.apache.s4.comm.udp.UDPRemoteEmitter 3 | s4.comm.listener.class = org.apache.s4.comm.udp.UDPListener 4 | s4.cluster.name = cluster1 5 | s4.cluster.zk_address = localhost:2181 6 | s4.cluster.zk_session_timeout = 10000 7 | s4.cluster.zk_connection_timeout = 10000 8 | -------------------------------------------------------------------------------- /subprojects/s4-core/s4-core.gradle: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | description = 'The S4 core platform.' 21 | 22 | dependencies { 23 | compile project(":s4-base") 24 | compile project(":s4-comm") 25 | compile libraries.commons_codec 26 | compile libraries.jcommander 27 | compile libraries.asm 28 | compile libraries.netty 29 | compile libraries.zkclient 30 | testCompile project(path: ':s4-comm', configuration: 'tests') 31 | testCompile libraries.gradle_base_services 32 | testCompile libraries.gradle_core 33 | testCompile libraries.gradle_tooling_api 34 | testCompile libraries.gradle_wrapper 35 | testCompile libraries.mockito_core 36 | } 37 | 38 | test { 39 | forkEvery=1; 40 | } 41 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/Key.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core; 20 | 21 | import java.util.List; 22 | 23 | import org.apache.commons.lang.StringUtils; 24 | import org.apache.s4.base.Event; 25 | import org.apache.s4.base.KeyFinder; 26 | 27 | /** 28 | * The Key class is used to get the value of the key on a specific type of event. This is done to abstract all the 29 | * complexity required to get the value. The method for getting the value is implemented in a method of an object of 30 | * type KeyFinder. 31 | * 32 | * The application programmer provides the events and the corresponding finders. The framework will use it to key on 33 | * events. 34 | */ 35 | public class Key { 36 | 37 | final private KeyFinder finder; 38 | final private String separator; 39 | 40 | public Key(KeyFinder finder, String separator) { 41 | this.finder = finder; 42 | this.separator = separator; 43 | } 44 | 45 | public List getList(T event) { 46 | return finder.get(event); 47 | } 48 | 49 | public String get(T event) { 50 | List keys = getList(event); 51 | 52 | return StringUtils.join(keys, separator); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/RemoteSender.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core; 20 | 21 | import org.apache.s4.base.Emitter; 22 | import org.apache.s4.base.EventMessage; 23 | import org.apache.s4.base.Hasher; 24 | 25 | /** 26 | * Sends events to a remote cluster. 27 | * 28 | */ 29 | public class RemoteSender { 30 | 31 | final private Emitter emitter; 32 | final private Hasher hasher; 33 | int targetPartition = 0; 34 | 35 | public RemoteSender(Emitter emitter, Hasher hasher) { 36 | super(); 37 | this.emitter = emitter; 38 | this.hasher = hasher; 39 | } 40 | 41 | public void send(String hashKey, EventMessage eventMessage) { 42 | if (hashKey == null) { 43 | // round robin by default 44 | emitter.send(Math.abs(targetPartition++ % emitter.getPartitionCount()), eventMessage); 45 | } else { 46 | int partition = (int) (hasher.hash(hashKey) % emitter.getPartitionCount()); 47 | emitter.send(partition, eventMessage); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/Streamable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | /** 24 | * We use this interface to put events into objects. 25 | * 26 | * @param 27 | */ 28 | public interface Streamable { 29 | 30 | /** 31 | * Starting the stream starts the associated dequeuing thread. 32 | */ 33 | void start(); 34 | 35 | /** 36 | * Put an event into the streams. 37 | * 38 | * @param event 39 | */ 40 | public void put(Event event); 41 | 42 | /** 43 | * Stop and close all the streams. 44 | */ 45 | public void close(); 46 | 47 | /** 48 | * @return the name of this streamable object. 49 | */ 50 | public String getName(); 51 | } 52 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/adapter/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * This package contains utilities for creating and configuring adapter apps. 21 | */ 22 | package org.apache.s4.core.adapter; 23 | 24 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/CheckpointingTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import java.util.Map; 22 | import java.util.TimerTask; 23 | 24 | import org.apache.s4.core.ProcessingElement; 25 | 26 | /** 27 | * When checkpointing at regular time intervals, this class is used to actually perform the checkpoints. It iterates 28 | * among all instances of the specified prototype, and checkpoints every eligible instance. 29 | * 30 | */ 31 | public class CheckpointingTask extends TimerTask { 32 | 33 | ProcessingElement prototype; 34 | 35 | public CheckpointingTask(ProcessingElement prototype) { 36 | super(); 37 | this.prototype = prototype; 38 | } 39 | 40 | @Override 41 | public void run() { 42 | Map peInstances = prototype.getPEInstances(); 43 | for (Map.Entry entry : peInstances.entrySet()) { 44 | synchronized (entry.getValue()) { 45 | if (entry.getValue().isDirty()) { 46 | entry.getValue().checkpoint(); 47 | } 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/FetchTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import java.util.concurrent.Callable; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * Encapsulates a checkpoint fetching operation. 28 | * 29 | */ 30 | public class FetchTask implements Callable { 31 | 32 | private static Logger logger = LoggerFactory.getLogger(FetchTask.class); 33 | 34 | StateStorage stateStorage; 35 | CheckpointId checkpointId; 36 | 37 | public FetchTask(StateStorage stateStorage, CheckpointId checkpointId) { 38 | super(); 39 | this.stateStorage = stateStorage; 40 | this.checkpointId = checkpointId; 41 | } 42 | 43 | @Override 44 | public byte[] call() throws Exception { 45 | try { 46 | byte[] result = stateStorage.fetchState(checkpointId); 47 | return result; 48 | } catch (Exception e) { 49 | logger.error("Cannot fetch checkpoint data for {}", checkpointId, e); 50 | throw e; 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/FileSystemBackendCheckpointingModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import com.google.inject.AbstractModule; 22 | 23 | /** 24 | * Checkpointing module that uses the {@link DefaultFileSystemStateStorage} as a checkpointing backend. 25 | * 26 | */ 27 | public class FileSystemBackendCheckpointingModule extends AbstractModule { 28 | @Override 29 | protected void configure() { 30 | bind(StateStorage.class).to(DefaultFileSystemStateStorage.class); 31 | bind(CheckpointingFramework.class).to(SafeKeeper.class); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/NoOpCheckpointingFramework.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import org.apache.s4.core.ProcessingElement; 22 | 23 | /** 24 | * Implementation of {@link CheckpointingFramework} that does NO checkpointing. 25 | * 26 | */ 27 | public final class NoOpCheckpointingFramework implements CheckpointingFramework { 28 | 29 | @Override 30 | public StorageCallback saveState(ProcessingElement pe) { 31 | return null; 32 | } 33 | 34 | @Override 35 | public byte[] fetchSerializedState(CheckpointId key) { 36 | return null; 37 | } 38 | 39 | @Override 40 | public boolean isCheckpointable(ProcessingElement pe) { 41 | return false; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/SerializeTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import java.util.concurrent.Callable; 22 | 23 | import org.apache.s4.core.ProcessingElement; 24 | 25 | /** 26 | * Encaspulate a PE serialization operation. This operation locks the PE instance in order to avoid any inconsistent 27 | * serialized state. If serialization is successful, the PE is marked as "not dirty". 28 | * 29 | */ 30 | public class SerializeTask implements Callable { 31 | 32 | ProcessingElement pe; 33 | 34 | public SerializeTask(ProcessingElement pe) { 35 | super(); 36 | this.pe = pe; 37 | } 38 | 39 | @Override 40 | public byte[] call() throws Exception { 41 | synchronized (pe) { 42 | byte[] state = pe.serializeState(); 43 | pe.clearDirty(); 44 | return state; 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/StorageCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import org.apache.s4.core.ft.CheckpointingFramework.StorageResultCode; 22 | 23 | /** 24 | * 25 | * Callback for reporting the result of an asynchronous storage operation 26 | * 27 | */ 28 | public interface StorageCallback { 29 | 30 | /** 31 | * Notifies the result of a storage operation 32 | * 33 | * @param resultCode 34 | * code for the result : {@link StorageResultCode} 35 | * @param message 36 | * whatever message object is suitable 37 | */ 38 | public void storageOperationResult(CheckpointingFramework.StorageResultCode resultCode, Object message); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/StorageCallbackFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | /** 22 | * A factory for creating storage callbacks. Storage callback implementations 23 | * that can take specific actions upon success or failure of asynchronous 24 | * storage operations. 25 | * 26 | */ 27 | public interface StorageCallbackFactory { 28 | 29 | /** 30 | * Factory method 31 | * 32 | * @return returns a StorageCallback instance 33 | */ 34 | public StorageCallback createStorageCallback(); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/ft/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | *

    This package contains classes for providing some fault tolerance 21 | * to S4 PEs.

    22 | *

    The current approach is based on checkpointing.

    23 | *

    Checkpoints are taken periodically (configurable by time or 24 | * frequency of application events), and when restarting an S4 node, 25 | * saved checkpoints are automatically and lazily restored.

    26 | *

    Lazy restoration is triggered by an application event to a PE 27 | * that has not yet been restored.

    28 | *

    Checkpoints are stored in storage backends. Storage backends may 29 | * implement eager techniques to prefetch checkpoint data to be 30 | * recovered. Storage backends can be implemented for various kinds of systems, 31 | * and must implement the {@link org.apache.s4.core.ft.StateStorage} interface. 32 | * They are pluggable throught the module system. 33 | *

    34 | * The application programmer must take care of marking as transient 35 | * the fields that do not have to be persisted (or cannot be persisted). 36 | *

    Storage backends are pluggable and we provide some default 37 | * implementations in this package

    38 | */ 39 | package org.apache.s4.core.ft; 40 | 41 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/gen/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Utility classes for dynamic code generation, in particular proxies to enable dispatching 21 | * of events to methods of processing elements based on the runtime type of the event. 22 | */ 23 | package org.apache.s4.core.gen; -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Key classes of the S4 platform, implementing concepts such as Stream, Processing Element. 21 | */ 22 | package org.apache.s4.core; 23 | 24 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParametersInjectionModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.util; 20 | 21 | import java.util.Map; 22 | 23 | import com.google.inject.AbstractModule; 24 | import com.google.inject.name.Names; 25 | 26 | /** 27 | * Injects String parameters from a map. Used for loading parameters outside of config files. 28 | * 29 | */ 30 | public class ParametersInjectionModule extends AbstractModule { 31 | 32 | Map params; 33 | 34 | public ParametersInjectionModule(Map params) { 35 | this.params = params; 36 | } 37 | 38 | @Override 39 | protected void configure() { 40 | for (Map.Entry param : params.entrySet()) { 41 | bind(String.class).annotatedWith(Names.named(param.getKey())).toInstance(param.getValue()); 42 | } 43 | 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/util/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Utility classes. 21 | */ 22 | package org.apache.s4.core.util; -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/window/DefaultAggregatingSlot.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.window; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import com.google.common.collect.ImmutableList; 25 | 26 | /** 27 | * Window slot that keeps all data elements as a list. 28 | * 29 | * @param 30 | * Type of slot elements 31 | */ 32 | public class DefaultAggregatingSlot implements Slot { 33 | 34 | List data = null; 35 | boolean open = true; 36 | 37 | @Override 38 | public void update(T datum) { 39 | if (open) { 40 | if (data == null) { 41 | data = new ArrayList(); 42 | } 43 | data.add(datum); 44 | } 45 | } 46 | 47 | @Override 48 | public void close() { 49 | open = false; 50 | if (data == null) { 51 | data = ImmutableList.of(); 52 | } else { 53 | data = ImmutableList.copyOf(data); 54 | } 55 | } 56 | 57 | public List getAggregatedData() { 58 | return data == null ? (List) ImmutableList.of() : data; 59 | } 60 | 61 | public static class DefaultAggregatingSlotFactory implements SlotFactory> { 62 | 63 | @Override 64 | public DefaultAggregatingSlot createSlot() { 65 | return new DefaultAggregatingSlot(); 66 | } 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/window/OHCLSlot.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.window; 20 | 21 | public class OHCLSlot implements Slot { 22 | 23 | double open = -1; 24 | double high = -1; 25 | double low = -1; 26 | double close = -1; 27 | long ticks = 0; 28 | boolean isOpen; 29 | 30 | @Override 31 | public void update(Double data) { 32 | if (isOpen) { 33 | if (open == -1) { 34 | open = low = high = close = data; 35 | } else if (data > high) { 36 | high = data; 37 | } else if (data < low) { 38 | low = data; 39 | } 40 | close = data; 41 | ticks++; 42 | } 43 | } 44 | 45 | @Override 46 | public void close() { 47 | isOpen = false; 48 | } 49 | 50 | double getOpen() { 51 | return open; 52 | } 53 | 54 | double getClose() { 55 | return close; 56 | } 57 | 58 | double getHigh() { 59 | return high; 60 | } 61 | 62 | double getLow() { 63 | return low; 64 | } 65 | 66 | long getTicksCount() { 67 | return ticks; 68 | } 69 | 70 | public static class OHCLSlotFactory implements SlotFactory { 71 | 72 | @Override 73 | public OHCLSlot createSlot() { 74 | return new OHCLSlot(); 75 | } 76 | 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/window/Slot.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.window; 20 | 21 | /** 22 | * A convenience window slot, that aggregates elements of type . 23 | * 24 | * Users must add suitable getter methods to retrieve aggregated data. 25 | * 26 | * @param 27 | * elements to aggregate 28 | */ 29 | public interface Slot { 30 | 31 | /** 32 | * Add a single data element 33 | */ 34 | void update(T data); 35 | 36 | /** 37 | * Compute aggregated data on available gathered slot data, place slot and slot data in immutable state. 38 | */ 39 | void close(); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/window/SlotFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.window; 20 | 21 | /** 22 | * Defines factory for window slots 23 | * 24 | * @param 25 | * slot class or interface that is produced 26 | */ 27 | public interface SlotFactory { 28 | 29 | T createSlot(); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/core/window/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Provides facilities for processing events as samples, through time or count-based windows. 21 | * Currently we only provide a facility for sliding windows. 22 | * 23 | * NOTE: we are still working on improving the API here. 24 | */ 25 | package org.apache.s4.core.window; -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/deploy/DeploymentFailedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | /** 22 | * 23 | *

    24 | * Indicates that an application failed to be deployed to an S4 node. 25 | *

    26 | *

    27 | * Thrown during detection, fetching, loading or startup of applications deployed from a repository. 28 | *

    29 | */ 30 | public class DeploymentFailedException extends Exception { 31 | 32 | public DeploymentFailedException(String message, Throwable e) { 33 | super(message, e); 34 | } 35 | 36 | public DeploymentFailedException(String message) { 37 | super(message); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/deploy/DeploymentManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | /** 22 | * Marker interface for deployment managers. Allows to supply a no-op deployment manager through dependency injection. 23 | * (TODO that hack should be improved!) 24 | * 25 | */ 26 | public interface DeploymentManager { 27 | 28 | void start(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/deploy/FileSystemS4RFetcher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | import java.io.File; 22 | import java.io.FileInputStream; 23 | import java.io.FileNotFoundException; 24 | import java.io.InputStream; 25 | import java.net.URI; 26 | 27 | /** 28 | * Fetches S4R files from a file system, possibly distributed. 29 | * 30 | */ 31 | public class FileSystemS4RFetcher implements S4RFetcher { 32 | 33 | @Override 34 | public InputStream fetch(URI uri) throws DeploymentFailedException { 35 | try { 36 | return new FileInputStream(new File(uri)); 37 | } catch (FileNotFoundException e) { 38 | throw new DeploymentFailedException("Cannot retrieve file from uri [" + uri.toString() + "]"); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/deploy/NoOpDeploymentManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | /** 22 | * Does not handle any deployment (hence does not require any cluster configuration settings) 23 | * 24 | */ 25 | public class NoOpDeploymentManager implements DeploymentManager { 26 | 27 | @Override 28 | public void start() { 29 | // does nothing 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/deploy/S4RFetcher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | import java.io.InputStream; 22 | import java.net.URI; 23 | 24 | /** 25 | * This interface defines methods to fetch S4R archive files from a URI. Various protocols can be supported in the 26 | * implementation classes (e.g. file system, HTTP etc...) 27 | * 28 | */ 29 | public interface S4RFetcher { 30 | 31 | /** 32 | * Returns a stream to an S4R archive file 33 | * 34 | * @param uri 35 | * S4R archive identifier 36 | * @return an input stream for accessing the content of the S4R file 37 | * @throws DeploymentFailedException 38 | * when fetching fails 39 | */ 40 | InputStream fetch(URI uri) throws DeploymentFailedException; 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/java/org/apache/s4/deploy/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | /** 20 | * Dynamic application deployment framework, that allows identifying new available applications, 21 | * downloading them through various protocols, loading them and starting them in the current S4 node. 22 | */ 23 | package org.apache.s4.deploy; -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/resources/default.s4.core.properties: -------------------------------------------------------------------------------- 1 | s4.logger_level = DEBUG 2 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/base/SimpleEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.base; 20 | 21 | public class SimpleEvent extends Event { 22 | 23 | private String name; 24 | private long numGrapes; 25 | 26 | public SimpleEvent(String name, long numGrapes) { 27 | super(); 28 | this.name = name; 29 | this.numGrapes = numGrapes; 30 | } 31 | 32 | /** 33 | * @return the name 34 | */ 35 | public String getName() { 36 | return name; 37 | } 38 | 39 | public long getNumGrapes() { 40 | return numGrapes; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/ft/CheckpointingModuleWithUnrespondingFetchingStorageBackend.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import org.apache.s4.core.ft.FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.DummyZKStorageCallbackFactory; 22 | import org.apache.s4.fixtures.CommTestUtils; 23 | 24 | import com.google.inject.AbstractModule; 25 | import com.google.inject.name.Names; 26 | 27 | public class CheckpointingModuleWithUnrespondingFetchingStorageBackend extends AbstractModule { 28 | 29 | @Override 30 | protected void configure() { 31 | bind(String.class).annotatedWith(Names.named("s4.checkpointing.filesystem.storageRootPath")).toInstance( 32 | CommTestUtils.DEFAULT_STORAGE_DIR.getAbsolutePath()); 33 | bind(StateStorage.class).to(StorageWithUnrespondingFetching.class); 34 | bind(CheckpointingFramework.class).to(SafeKeeper.class); 35 | bind(StorageCallbackFactory.class).to(DummyZKStorageCallbackFactory.class); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountApp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import org.apache.s4.core.ft.CheckpointingConfig.CheckpointingMode; 22 | import org.apache.s4.wordcount.WordCountApp; 23 | 24 | public class FTWordCountApp extends WordCountApp { 25 | 26 | @Override 27 | protected void onInit() { 28 | super.onInit(); 29 | getPE("classifierPE").setCheckpointingConfig( 30 | new CheckpointingConfig.Builder(CheckpointingMode.EVENT_COUNT).frequency(1).build()); 31 | getPE("counterPE").setCheckpointingConfig( 32 | new CheckpointingConfig.Builder(CheckpointingMode.EVENT_COUNT).frequency(1).build()); 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithCountBasedCheckpointing.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import org.apache.s4.core.App; 22 | import org.apache.s4.core.ft.CheckpointingConfig.CheckpointingMode; 23 | 24 | public class S4AppWithCountBasedCheckpointing extends App { 25 | 26 | @Override 27 | protected void onStart() { 28 | } 29 | 30 | @Override 31 | protected void onInit() { 32 | StatefulTestPE pe = createPE(StatefulTestPE.class); 33 | pe.setSingleton(true); 34 | pe.setCheckpointingConfig(new CheckpointingConfig.Builder(CheckpointingMode.EVENT_COUNT).frequency(1).build()); 35 | createInputStream("inputStream", pe); 36 | } 37 | 38 | @Override 39 | protected void onClose() { 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithManualCheckpointing.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import org.apache.s4.core.App; 22 | 23 | /** 24 | * 25 | * 26 | */ 27 | public class S4AppWithManualCheckpointing extends App { 28 | 29 | @Override 30 | protected void onStart() { 31 | 32 | } 33 | 34 | @Override 35 | protected void onInit() { 36 | StatefulTestPE pe = createPE(StatefulTestPE.class); 37 | pe.setSingleton(true); 38 | createInputStream("inputStream", pe); 39 | } 40 | 41 | @Override 42 | protected void onClose() { 43 | // TODO Auto-generated method stub 44 | 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithTimeBasedCheckpointing.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.ft; 20 | 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import org.apache.s4.core.App; 24 | import org.apache.s4.core.ft.CheckpointingConfig.CheckpointingMode; 25 | 26 | public class S4AppWithTimeBasedCheckpointing extends App { 27 | 28 | @Override 29 | protected void onStart() { 30 | } 31 | 32 | @Override 33 | protected void onInit() { 34 | StatefulTestPE pe = createPE(StatefulTestPE.class); 35 | pe.setSingleton(true); 36 | // checkpoints (if applicable) every 1 ms! 37 | pe.setCheckpointingConfig(new CheckpointingConfig.Builder(CheckpointingMode.TIME).frequency(1) 38 | .timeUnit(TimeUnit.MILLISECONDS).build()); 39 | createInputStream("inputStream", pe); 40 | } 41 | 42 | @Override 43 | protected void onClose() { 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/overloadgen/B.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.overloadgen; 20 | 21 | import org.apache.s4.core.ProcessingElement; 22 | 23 | public class B extends ProcessingElement { 24 | 25 | 26 | 27 | @Override 28 | protected void onCreate() { 29 | // TODO Auto-generated method stub 30 | 31 | } 32 | 33 | @Override 34 | protected void onRemove() { 35 | // TODO Auto-generated method stub 36 | 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/overloadgen/C.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.overloadgen; 20 | 21 | import org.apache.s4.core.ProcessingElement; 22 | 23 | public class C extends ProcessingElement { 24 | 25 | public boolean processedEvent1Class = false; 26 | 27 | public void onEvent(Event1 event) { 28 | processedEvent1Class = true; 29 | } 30 | 31 | @Override 32 | protected void onCreate() { 33 | // TODO Auto-generated method stub 34 | 35 | } 36 | 37 | @Override 38 | protected void onRemove() { 39 | // TODO Auto-generated method stub 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/overloadgen/D.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.overloadgen; 20 | 21 | import org.apache.s4.base.Event; 22 | import org.apache.s4.core.ProcessingElement; 23 | 24 | 25 | public class D extends ProcessingElement { 26 | 27 | public boolean processedGenericEvent = false; 28 | public boolean processedEvent1 = false; 29 | 30 | public void onEvent(Event event) { 31 | processedGenericEvent =true; 32 | } 33 | 34 | public void onEvent(Event1 event) { 35 | processedEvent1 = true; 36 | } 37 | 38 | @Override 39 | protected void onCreate() { 40 | // TODO Auto-generated method stub 41 | 42 | } 43 | 44 | @Override 45 | protected void onRemove() { 46 | // TODO Auto-generated method stub 47 | 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/overloadgen/Event1.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.overloadgen; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class Event1 extends Event { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/overloadgen/Event1a.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.overloadgen; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class Event1a extends Event { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/overloadgen/Event2.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.overloadgen; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class Event2 extends Event { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/CountTriggerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.triggers; 20 | 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import org.apache.s4.core.TriggerTest; 24 | import org.junit.Assert; 25 | import org.junit.Test; 26 | 27 | public class CountTriggerTest extends TriggerTest { 28 | 29 | @Test 30 | public void testEventCountBasedTrigger() throws Exception { 31 | triggerType = TriggerType.COUNT_BASED; 32 | Assert.assertTrue(createTriggerAppAndSendEvent().await(5, TimeUnit.SECONDS)); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/NoTriggerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.triggers; 20 | 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import junit.framework.Assert; 24 | 25 | import org.apache.s4.core.TriggerTest; 26 | import org.junit.Test; 27 | 28 | public class NoTriggerTest extends TriggerTest { 29 | 30 | @Test 31 | public void testNoTrigger() throws Exception { 32 | triggerType = TriggerType.NONE; 33 | Assert.assertFalse(createTriggerAppAndSendEvent().await(5, TimeUnit.SECONDS)); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TimeTriggerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.triggers; 20 | 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import org.apache.s4.core.TriggerTest; 24 | import org.junit.Assert; 25 | import org.junit.Test; 26 | 27 | 28 | public class TimeTriggerTest extends TriggerTest { 29 | 30 | @Test 31 | public void testTimeBasedTrigger() throws Exception { 32 | triggerType = TriggerType.TIME_BASED; 33 | Assert.assertTrue(createTriggerAppAndSendEvent().await(5, TimeUnit.SECONDS)); 34 | 35 | } 36 | 37 | 38 | } 39 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggeredApp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.core.triggers; 20 | 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import org.apache.s4.base.Event; 24 | import org.apache.s4.core.App; 25 | import org.apache.s4.core.Stream; 26 | import org.apache.s4.core.TriggerTest; 27 | import org.apache.s4.wordcount.SentenceKeyFinder; 28 | 29 | import com.google.inject.Inject; 30 | 31 | public class TriggeredApp extends App { 32 | 33 | public Stream stream; 34 | 35 | @Inject 36 | public TriggeredApp() { 37 | super(); 38 | } 39 | 40 | @Override 41 | protected void onStart() { 42 | // TODO Auto-generated method stub 43 | 44 | } 45 | 46 | @Override 47 | protected void onInit() { 48 | 49 | TriggerablePE prototype = createPE(TriggerablePE.class); 50 | stream = createStream("stream", new SentenceKeyFinder(), prototype); 51 | switch (TriggerTest.triggerType) { 52 | case COUNT_BASED: 53 | prototype.setTrigger(Event.class, 1, 0, TimeUnit.SECONDS); 54 | break; 55 | case TIME_BASED: 56 | prototype.setTrigger(Event.class, 1, 1, TimeUnit.MILLISECONDS); 57 | default: 58 | break; 59 | } 60 | 61 | } 62 | 63 | @Override 64 | protected void onClose() { 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/deploy/AppConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | public class AppConstants { 22 | public static final String STARTED_ZNODE_1 = "/s4-test/test-app-1-started"; 23 | public static final String INITIALIZED_ZNODE_1 = "/s4-test/test-app-1-initialized"; 24 | 25 | public static final String STARTED_ZNODE_2 = "/s4-test/test-app-2-started"; 26 | public static final String INITIALIZED_ZNODE_2 = "/s4-test/test-app-2-initialized"; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/fixtures/AssignmentFromZKNoFailFast.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.fixtures; 20 | 21 | import org.apache.s4.comm.topology.AssignmentFromZK; 22 | import org.apache.zookeeper.Watcher.Event.KeeperState; 23 | 24 | import com.google.inject.Inject; 25 | import com.google.inject.name.Named; 26 | 27 | public class AssignmentFromZKNoFailFast extends AssignmentFromZK { 28 | 29 | @Inject 30 | public AssignmentFromZKNoFailFast(@Named("s4.cluster.name") String clusterName, 31 | @Named("s4.cluster.zk_address") String zookeeperAddress, 32 | @Named("s4.cluster.zk_session_timeout") int sessionTimeout, 33 | @Named("s4.cluster.zk_connection_timeout") int connectionTimeout) throws Exception { 34 | super(clusterName, zookeeperAddress, sessionTimeout, connectionTimeout); 35 | } 36 | 37 | @Override 38 | public void handleStateChanged(KeeperState state) throws Exception { 39 | // no fail fast 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/fixtures/ClusterFromZKNoFailFast.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.fixtures; 20 | 21 | import org.apache.s4.comm.topology.ClusterFromZK; 22 | import org.apache.zookeeper.Watcher.Event.KeeperState; 23 | 24 | import com.google.inject.Inject; 25 | import com.google.inject.name.Named; 26 | 27 | public class ClusterFromZKNoFailFast extends ClusterFromZK { 28 | 29 | @Inject 30 | public ClusterFromZKNoFailFast(@Named("s4.cluster.name") String clusterName, 31 | @Named("s4.cluster.zk_address") String zookeeperAddress, 32 | @Named("s4.cluster.zk_session_timeout") int sessionTimeout, 33 | @Named("s4.cluster.zk_connection_timeout") int connectionTimeout) throws Exception { 34 | super(clusterName, zookeeperAddress, sessionTimeout, connectionTimeout); 35 | } 36 | 37 | @Override 38 | public void handleStateChanged(KeeperState state) throws Exception { 39 | // no fail fast 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/fixtures/ClustersFromZKNoFailFast.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.fixtures; 20 | 21 | import org.apache.s4.comm.topology.ClustersFromZK; 22 | import org.apache.zookeeper.Watcher.Event.KeeperState; 23 | 24 | import com.google.inject.Inject; 25 | import com.google.inject.name.Named; 26 | 27 | public class ClustersFromZKNoFailFast extends ClustersFromZK { 28 | 29 | @Inject 30 | public ClustersFromZKNoFailFast(@Named("s4.cluster.name") String clusterName, 31 | @Named("s4.cluster.zk_address") String zookeeperAddress, 32 | @Named("s4.cluster.zk_session_timeout") int sessionTimeout, 33 | @Named("s4.cluster.zk_connection_timeout") int connectionTimeout) throws Exception { 34 | super(clusterName, zookeeperAddress, sessionTimeout, connectionTimeout); 35 | } 36 | 37 | @Override 38 | public void handleStateChanged(KeeperState state) throws Exception { 39 | // no fail fast 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/fixtures/MockCoreModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.fixtures; 20 | 21 | import org.apache.s4.base.Emitter; 22 | import org.apache.s4.base.Listener; 23 | import org.apache.s4.core.Receiver; 24 | import org.apache.s4.deploy.DeploymentManager; 25 | import org.apache.s4.deploy.NoOpDeploymentManager; 26 | import org.mockito.Mockito; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import com.google.inject.AbstractModule; 31 | 32 | /** 33 | * Core module mocking basic platform functionalities. 34 | * 35 | */ 36 | public class MockCoreModule extends AbstractModule { 37 | 38 | @SuppressWarnings("unused") 39 | private static Logger logger = LoggerFactory.getLogger(MockCoreModule.class); 40 | 41 | public MockCoreModule() { 42 | } 43 | 44 | @Override 45 | protected void configure() { 46 | bind(DeploymentManager.class).to(NoOpDeploymentManager.class); 47 | bind(Emitter.class).toInstance(Mockito.mock(Emitter.class)); 48 | bind(Listener.class).toInstance(Mockito.mock(Listener.class)); 49 | bind(Receiver.class).toInstance(Mockito.mock(Receiver.class)); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/fixtures/NonFailFastZookeeperClientsModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.fixtures; 20 | 21 | import org.apache.s4.comm.topology.Assignment; 22 | import org.apache.s4.comm.topology.Cluster; 23 | import org.apache.s4.comm.topology.Clusters; 24 | 25 | import com.google.inject.AbstractModule; 26 | 27 | /** 28 | * 29 | * Used for injecting non-fail-fast zookeeper client classes. 30 | * 31 | * Here is why: 32 | * 33 | *
      34 | *
    • tests contained in a single junit class are not forked: forking is on a class basis
      • 35 | *
    • zookeeper client classes are injected during the tests
      • 36 | *
    • zookeeper server is restarted between test methods.
      • 37 | *
    • zookeeper client classes from previous tests methods get a "expired" exception upon reconnection to the new 38 | * zookeeper instance. With a fail-fast implementation, this would kill the current test.
      • 39 | *
    40 | * 41 | * 42 | */ 43 | public class NonFailFastZookeeperClientsModule extends AbstractModule { 44 | 45 | public NonFailFastZookeeperClientsModule() { 46 | } 47 | 48 | @Override 49 | protected void configure() { 50 | bind(Assignment.class).to(AssignmentFromZKNoFailFast.class); 51 | bind(Cluster.class).to(ClusterFromZKNoFailFast.class); 52 | 53 | bind(Clusters.class).to(ClustersFromZKNoFailFast.class); 54 | 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/KeyValueEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import org.apache.s4.wordcount.StringEvent; 22 | 23 | public class KeyValueEvent extends StringEvent { 24 | 25 | String key; 26 | String value; 27 | 28 | public KeyValueEvent(String keyValue) { 29 | key = keyValue.split(";")[0]; 30 | value = keyValue.split(";")[1]; 31 | } 32 | 33 | public String getKey() { 34 | return key; 35 | } 36 | 37 | public String getValue() { 38 | return value; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/KeyValueKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | 22 | import java.util.Arrays; 23 | import java.util.List; 24 | 25 | import org.apache.s4.base.KeyFinder; 26 | 27 | public class KeyValueKeyFinder implements KeyFinder { 28 | 29 | public static final String UNIQUE_KEY = "KEY"; 30 | 31 | @Override 32 | public List get(final KeyValueEvent event) { 33 | return Arrays.asList(new String[] {UNIQUE_KEY}); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/SentenceKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import java.util.List; 22 | 23 | import org.apache.s4.base.Event; 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | import com.google.common.collect.ImmutableList; 27 | 28 | public class SentenceKeyFinder implements KeyFinder { 29 | 30 | private static final String SENTENCE_KEY = "sentence"; 31 | 32 | @Override 33 | public List get(Event event) { 34 | return ImmutableList.of(SENTENCE_KEY); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/StringEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class StringEvent extends Event { 24 | 25 | String string; 26 | 27 | public StringEvent() {} 28 | 29 | public StringEvent(String string) { 30 | super(); 31 | this.string = string; 32 | } 33 | 34 | public void setString(String string) { 35 | this.string = string; 36 | } 37 | 38 | public String getString() { 39 | return string; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/Word.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | 22 | public class Word { 23 | 24 | private String word; 25 | 26 | public Word() { 27 | } 28 | 29 | public Word(String word) { 30 | this.word = word; 31 | } 32 | 33 | public void setWord(String word) { 34 | this.word = word; 35 | } 36 | 37 | public String getWord() { 38 | return word; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class WordCountEvent extends Event { 24 | 25 | private String word; 26 | private int count; 27 | 28 | protected WordCountEvent() {} 29 | 30 | public WordCountEvent(String word, int count) { 31 | super(); 32 | this.word = word; 33 | this.count = count; 34 | } 35 | 36 | public String getWord() { 37 | return word; 38 | } 39 | 40 | public int getCount() { 41 | return count; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | import org.apache.s4.base.KeyFinder; 26 | 27 | public class WordCountKeyFinder implements KeyFinder { 28 | 29 | @SuppressWarnings("serial") 30 | @Override 31 | public List get(WordCountEvent event) { 32 | return new ArrayList(){{add("classifier");}}; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import com.google.inject.AbstractModule; 22 | 23 | public class WordCountModule extends AbstractModule { 24 | 25 | @Override 26 | protected void configure() { 27 | bind(WordCountApp.class); 28 | 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCounterPE.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import org.apache.s4.core.App; 22 | import org.apache.s4.core.ProcessingElement; 23 | import org.apache.s4.core.Stream; 24 | 25 | public class WordCounterPE extends ProcessingElement { 26 | 27 | int wordCounter; 28 | transient Stream wordClassifierStream; 29 | 30 | private WordCounterPE() {} 31 | 32 | public WordCounterPE(App app) { 33 | super(app); 34 | } 35 | 36 | public void setWordClassifierStream(Stream stream) { 37 | this.wordClassifierStream = stream; 38 | } 39 | 40 | public void onEvent(WordSeenEvent event) { 41 | wordCounter++; 42 | System.out.println("seen word " + event.getWord()); 43 | // NOTE: it seems the id is the key for now... 44 | wordClassifierStream.put(new WordCountEvent(getId(), wordCounter)); 45 | } 46 | 47 | @Override 48 | protected void onCreate() { 49 | // TODO Auto-generated method stub 50 | 51 | } 52 | 53 | @Override 54 | protected void onRemove() { 55 | // TODO Auto-generated method stub 56 | 57 | } 58 | 59 | 60 | 61 | } 62 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSeenEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class WordSeenEvent extends Event { 24 | 25 | private String word; 26 | 27 | protected WordSeenEvent() {} 28 | 29 | public WordSeenEvent(String word) { 30 | super(); 31 | this.word = word; 32 | } 33 | 34 | public String getWord() { 35 | return word; 36 | } 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSeenKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | import org.apache.s4.base.KeyFinder; 26 | 27 | public class WordSeenKeyFinder implements KeyFinder { 28 | 29 | @Override 30 | public List get(WordSeenEvent event) { 31 | List key = new ArrayList(); 32 | key.add(event.getWord()); 33 | return key; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSplitterPE.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.wordcount; 20 | 21 | import org.apache.s4.base.Event; 22 | import org.apache.s4.core.App; 23 | import org.apache.s4.core.ProcessingElement; 24 | import org.apache.s4.core.Stream; 25 | 26 | public class WordSplitterPE extends ProcessingElement { 27 | 28 | private Stream wordSeenStream; 29 | 30 | public WordSplitterPE(App app) { 31 | super(app); 32 | } 33 | 34 | public void onEvent(Event event) { 35 | String[] split = event.get("sentence").split(" "); 36 | for (String word : split) { 37 | wordSeenStream.put(new WordSeenEvent(word)); 38 | } 39 | } 40 | 41 | public void setWordSeenStream(Stream stream) { 42 | this.wordSeenStream = stream; 43 | } 44 | 45 | @Override 46 | protected void onCreate() { 47 | // TODO Auto-generated method stub 48 | 49 | } 50 | 51 | @Override 52 | protected void onRemove() { 53 | // TODO Auto-generated method stub 54 | 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/DurationKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class DurationKeyFinder implements KeyFinder { 27 | 28 | public List get(EventA event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the gender and add it to the list. */ 33 | results.add(Long.toString(event.getDuration())); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/EventA.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class EventA extends Event { 24 | 25 | private long duration; 26 | private int height; 27 | 28 | /** 29 | * @return the duration 30 | */ 31 | public long getDuration() { 32 | return duration; 33 | } 34 | 35 | /** 36 | * @param duration 37 | * the duration to set 38 | */ 39 | public void setDuration(long duration) { 40 | this.duration = duration; 41 | } 42 | 43 | /** 44 | * @return the height 45 | */ 46 | public int getHeight() { 47 | return height; 48 | } 49 | 50 | /** 51 | * @param height 52 | * the height to set 53 | */ 54 | public void setHeight(int height) { 55 | this.height = height; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/EventB.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class EventB extends Event { 24 | 25 | private String query; 26 | 27 | /** 28 | * @return the query 29 | */ 30 | public String getQuery() { 31 | return query; 32 | } 33 | 34 | /** 35 | * @param query 36 | * the query to set 37 | */ 38 | public void setQuery(String query) { 39 | this.query = query; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/HeightKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class HeightKeyFinder implements KeyFinder { 27 | 28 | public List get(EventA event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the gender and add it to the list. */ 33 | results.add(Integer.toString(event.getHeight())); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/MyApp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import java.util.concurrent.TimeUnit; 22 | 23 | public class MyApp extends BuilderS4DSL { 24 | 25 | @Override 26 | public void onInit() { 27 | 28 | pe("PEZ").type(PEZ.class).fireOn(EventA.class).afterInterval(5, TimeUnit.SECONDS).cache().size(1000) 29 | .expires(3, TimeUnit.HOURS).emit(EventB.class).to("PEX"). 30 | 31 | pe("PEY").type(PEY.class).prop("duration", "4").prop("height", "99").timer() 32 | .withPeriod(2, TimeUnit.MINUTES).emit(EventA.class).onField("stream3") 33 | .withKeyFinder(DurationKeyFinder.class).to("PEZ").emit(EventA.class).onField("heightpez") 34 | .withKeyFinder(HeightKeyFinder.class).to("PEZ"). 35 | 36 | pe("PEX").type(PEX.class).prop("query", "money").cache().size(100).expires(1, TimeUnit.MINUTES) 37 | .asSingleton().emit(EventB.class).withKeyFinder(QueryKeyFinder.class).to("PEY", "PEZ"). 38 | 39 | build(); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/PEX.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import org.apache.s4.core.App; 22 | import org.apache.s4.core.ProcessingElement; 23 | import org.apache.s4.core.Stream; 24 | 25 | public class PEX extends ProcessingElement { 26 | 27 | private String query; 28 | private Stream[] someStream; 29 | @SuppressWarnings("unused") 30 | private Stream[] streams; 31 | 32 | public PEX(App app) { 33 | super(app); 34 | } 35 | 36 | @Override 37 | public void onCreate() { 38 | 39 | } 40 | 41 | @Override 42 | public void onRemove() { 43 | 44 | } 45 | 46 | /** 47 | * @return the keyword 48 | */ 49 | String getKeyword() { 50 | return query; 51 | } 52 | 53 | /** 54 | * @param query 55 | * the keyword to set 56 | */ 57 | void setKeyword(String query) { 58 | this.query = query; 59 | } 60 | 61 | /** 62 | * @return the someStream 63 | */ 64 | public Stream[] getSomeStream() { 65 | return someStream; 66 | } 67 | 68 | /** 69 | * @param someStream 70 | * the someStream to set 71 | */ 72 | public void setSomeStream(Stream[] someStream) { 73 | this.someStream = someStream; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/PEZ.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import org.apache.s4.core.App; 22 | import org.apache.s4.core.ProcessingElement; 23 | import org.apache.s4.core.Stream; 24 | 25 | public class PEZ extends ProcessingElement { 26 | 27 | private Stream[] stream1; 28 | private Stream[] stream2; 29 | 30 | public PEZ(App app) { 31 | super(app); 32 | } 33 | 34 | /** 35 | * @return the stream1 36 | */ 37 | Stream[] getStream1() { 38 | return stream1; 39 | } 40 | 41 | /** 42 | * @param stream1 43 | * the stream1 to set 44 | */ 45 | void setStream1(Stream[] stream1) { 46 | this.stream1 = stream1; 47 | } 48 | 49 | /** 50 | * @return the stream2 51 | */ 52 | Stream[] getStream2() { 53 | return stream2; 54 | } 55 | 56 | /** 57 | * @param stream2 58 | * the stream2 to set 59 | */ 60 | void setStream2(Stream[] stream2) { 61 | this.stream2 = stream2; 62 | } 63 | 64 | @Override 65 | public void onCreate() { 66 | // TODO Auto-generated method stub 67 | 68 | } 69 | 70 | @Override 71 | public void onRemove() { 72 | // TODO Auto-generated method stub 73 | 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /subprojects/s4-edsl/src/test/java/org/apache/s4/edsl/QueryKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.edsl; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class QueryKeyFinder implements KeyFinder { 27 | 28 | public List get(EventB event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the gender and add it to the list. */ 33 | results.add(event.getQuery()); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-example/s4-example.gradle: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | 21 | description = 'Examples of S4 applications.' 22 | 23 | 24 | 25 | dependencies { 26 | 27 | compile project( ":s4-base" ) 28 | compile project( ":s4-core" ) 29 | compile project( ":s4-comm" ) 30 | compile project( ":s4-edsl" ) 31 | compile libraries.junit 32 | compile libraries.jcip 33 | 34 | } 35 | 36 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/AgeKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class AgeKeyFinder implements KeyFinder { 27 | 28 | public List get(UserEvent event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the age and add it to the list. */ 33 | results.add(Integer.toString(event.getAge())); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/CountEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class CountEvent extends Event { 24 | 25 | private String key; 26 | private long count; 27 | 28 | public CountEvent() { 29 | 30 | } 31 | 32 | CountEvent(String key, long count) { 33 | this.key = key; 34 | this.count = count; 35 | } 36 | 37 | CountEvent(String key, long count, long time) { 38 | super(time); 39 | this.key = key; 40 | this.count = count; 41 | } 42 | 43 | 44 | /** 45 | * @return the key 46 | */ 47 | public String getKey() { 48 | return key; 49 | } 50 | 51 | /** 52 | * @return the count 53 | */ 54 | public long getCount() { 55 | return count; 56 | } 57 | 58 | public String toString() { 59 | return String.format("Key: " + key + ", Count: %08d", count); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/CountKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class CountKeyFinder implements KeyFinder { 27 | 28 | public List get(CountEvent event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the user ID and add it to the list. */ 33 | results.add(event.getKey()); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/CounterPE.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | import org.apache.s4.base.Event; 22 | import org.apache.s4.core.App; 23 | import org.apache.s4.core.ProcessingElement; 24 | import org.apache.s4.core.Stream; 25 | 26 | public class CounterPE extends ProcessingElement { 27 | 28 | private Stream[] countStream; 29 | 30 | public CounterPE(App app) { 31 | super(app); 32 | } 33 | 34 | public Stream[] getCountStream() { 35 | return countStream; 36 | } 37 | 38 | public void setCountStream(Stream... countStream) { 39 | this.countStream = countStream; 40 | } 41 | 42 | private long counter = 0; 43 | 44 | public void onEvent(Event event) { 45 | 46 | counter += 1; 47 | } 48 | 49 | public void onTrigger(Event event) { 50 | 51 | CountEvent countEvent = new CountEvent(getId(), counter); 52 | emit(countEvent, countStream); 53 | } 54 | 55 | @Override 56 | protected void onCreate() { 57 | 58 | } 59 | 60 | @Override 61 | protected void onRemove() { 62 | 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/GenderKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class GenderKeyFinder implements KeyFinder { 27 | 28 | public List get(UserEvent event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the gender and add it to the list. */ 33 | results.add(Character.toString(event.getGender())); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/PrintPE.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | import net.jcip.annotations.ThreadSafe; 22 | 23 | import org.apache.s4.base.Event; 24 | import org.apache.s4.core.App; 25 | import org.apache.s4.core.ProcessingElement; 26 | 27 | @ThreadSafe 28 | public class PrintPE extends ProcessingElement { 29 | 30 | public PrintPE(App app) { 31 | super(app); 32 | } 33 | 34 | public void onEvent(Event event) { 35 | 36 | System.out.println(event.toString()); 37 | } 38 | 39 | @Override 40 | protected void onCreate() { 41 | // TODO Auto-generated method stub 42 | 43 | } 44 | 45 | @Override 46 | protected void onRemove() { 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/README.md: -------------------------------------------------------------------------------- 1 | S4 Counter Example (NOT updated for 0.5.0) 2 | ================== 3 | 4 | In this example we do the following: 5 | 6 | - Generate dummy events (UserEvent). 7 | - Key events by user, gender, age. 8 | - Count by user, gender, age. 9 | - Print partial counts. 10 | 11 | The following diagram shows the application graph: 12 | 13 | ![S4 Counter](https://github.com/leoneu/s4-piper/raw/master/etc/s4-counter-example.png) 14 | 15 | In in following diagram I show how Classes, PE Prototypes, PE instances, and Streams are related. 16 | 17 | ![S4 Objects](https://github.com/leoneu/s4-piper/raw/master/etc/s4-objects-example.png) 18 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/UserEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.Event; 25 | 26 | public class UserEvent extends Event { 27 | 28 | private String userID; 29 | private int age; 30 | private char gender; 31 | private List favorites; 32 | 33 | public UserEvent() { 34 | 35 | } 36 | 37 | UserEvent(String userID, int age, List favorites, char gender) { 38 | this.userID = userID; 39 | this.age = age; 40 | this.favorites = favorites; 41 | this.gender = gender; 42 | } 43 | 44 | /** 45 | * @return the userID 46 | */ 47 | public String getUserID() { 48 | return userID; 49 | } 50 | 51 | /** 52 | * @return the age 53 | */ 54 | public int getAge() { 55 | return age; 56 | } 57 | 58 | /** 59 | * @return the favorites 60 | */ 61 | public List getFavorites() { 62 | return favorites; 63 | } 64 | 65 | /** 66 | * @return the gender 67 | */ 68 | public char getGender() { 69 | return gender; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/counter/UserIDKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | public class UserIDKeyFinder implements KeyFinder { 26 | 27 | public List get(UserEvent event) { 28 | 29 | List results = new ArrayList(); 30 | 31 | /* Retrieve the user ID and add it to the list. */ 32 | results.add(event.getUserID()); 33 | 34 | return results; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/AgeKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.edsl.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class AgeKeyFinder implements KeyFinder { 27 | 28 | public List get(UserEvent event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the age and add it to the list. */ 33 | results.add(Integer.toString(event.getAge())); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/CountEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.edsl.counter; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | public class CountEvent extends Event { 24 | 25 | private String key; 26 | private long count; 27 | 28 | public CountEvent() { 29 | 30 | } 31 | 32 | CountEvent(String key, long count) { 33 | this.key = key; 34 | this.count = count; 35 | } 36 | 37 | CountEvent(String key, long count, long time) { 38 | super(time); 39 | this.key = key; 40 | this.count = count; 41 | } 42 | 43 | 44 | /** 45 | * @return the key 46 | */ 47 | public String getKey() { 48 | return key; 49 | } 50 | 51 | /** 52 | * @return the count 53 | */ 54 | public long getCount() { 55 | return count; 56 | } 57 | 58 | public String toString() { 59 | return String.format("Key: " + key + ", Count: %08d", count); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/CountKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.edsl.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class CountKeyFinder implements KeyFinder { 27 | 28 | public List get(CountEvent event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the user ID and add it to the list. */ 33 | results.add(event.getKey()); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/GenderKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.edsl.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class GenderKeyFinder implements KeyFinder { 27 | 28 | public List get(UserEvent event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the gender and add it to the list. */ 33 | results.add(Character.toString(event.getGender())); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/PrintPE.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.edsl.counter; 20 | 21 | import org.apache.s4.base.Event; 22 | import org.apache.s4.core.App; 23 | import org.apache.s4.core.ProcessingElement; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | public class PrintPE extends ProcessingElement { 28 | 29 | private static final Logger logger = LoggerFactory.getLogger(PrintPE.class); 30 | 31 | public PrintPE(App app) { 32 | super(app); 33 | } 34 | 35 | public void onEvent(Event event) { 36 | 37 | logger.info(">>> [{}].", event.toString()); 38 | } 39 | 40 | @Override 41 | protected void onCreate() { 42 | } 43 | 44 | @Override 45 | protected void onRemove() { 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/UserEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.edsl.counter; 20 | 21 | 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.Event; 25 | 26 | public class UserEvent extends Event { 27 | 28 | private String userID; 29 | private int age; 30 | private char gender; 31 | private List favorites; 32 | 33 | public UserEvent() { 34 | 35 | } 36 | 37 | UserEvent(String userID, int age, List favorites, char gender) { 38 | this.userID = userID; 39 | this.age = age; 40 | this.favorites = favorites; 41 | this.gender = gender; 42 | } 43 | 44 | /** 45 | * @return the userID 46 | */ 47 | public String getUserID() { 48 | return userID; 49 | } 50 | 51 | /** 52 | * @return the age 53 | */ 54 | public int getAge() { 55 | return age; 56 | } 57 | 58 | /** 59 | * @return the favorites 60 | */ 61 | public List getFavorites() { 62 | return favorites; 63 | } 64 | 65 | /** 66 | * @return the gender 67 | */ 68 | public char getGender() { 69 | return gender; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /subprojects/s4-example/src/main/java/org/apache/s4/example/edsl/counter/UserIDKeyFinder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.edsl.counter; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.KeyFinder; 25 | 26 | public class UserIDKeyFinder implements KeyFinder { 27 | 28 | public List get(UserEvent event) { 29 | 30 | List results = new ArrayList(); 31 | 32 | /* Retrieve the user ID and add it to the list. */ 33 | results.add(event.getUserID()); 34 | 35 | return results; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/java/org/apache/s4/tools/FileExistsValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.tools; 20 | 21 | import java.io.File; 22 | 23 | import com.beust.jcommander.IParameterValidator; 24 | import com.beust.jcommander.ParameterException; 25 | 26 | public class FileExistsValidator implements IParameterValidator { 27 | 28 | @Override 29 | public void validate(String name, String value) throws ParameterException { 30 | if (!new File(value).exists()) { 31 | throw new ParameterException("File with path [" + value + "] specified in [" + name + "] does not exist"); 32 | } 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/java/org/apache/s4/tools/S4ArgsBase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.tools; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import com.beust.jcommander.IStringConverter; 25 | import com.beust.jcommander.Parameter; 26 | 27 | public abstract class S4ArgsBase { 28 | 29 | @Parameter(names = "-help", description = "usage") 30 | boolean help = false; 31 | 32 | @Parameter(names = "-s4ScriptPath", description = "path of the S4 script", hidden = true, required = false) 33 | String s4ScriptPath; 34 | 35 | @Parameter(names = "-gradleOpts", variableArity = true, description = "gradle system properties (as in GRADLE_OPTS environment properties) passed to gradle scripts", required = false, converter = GradleOptsConverter.class) 36 | List gradleOpts = new ArrayList(); 37 | 38 | // This removes automatically the -D of each gradle opt jvm parameter, if present 39 | public class GradleOptsConverter implements IStringConverter { 40 | 41 | @Override 42 | public String convert(String value) { 43 | if (value.startsWith("-D")) { 44 | return value.substring("-D".length()); 45 | } else { 46 | return value; 47 | } 48 | } 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/java/org/apache/s4/tools/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Implementations of S4-related commands. Includes parsing, validation and interpretation of the various commands and options available. 3 | */ 4 | package org.apache.s4.tools; -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/resources/templates/HelloApp.java.txt: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package hello; 20 | 21 | import java.util.Arrays; 22 | import java.util.List; 23 | 24 | import org.apache.s4.base.Event; 25 | import org.apache.s4.base.KeyFinder; 26 | import org.apache.s4.core.App; 27 | 28 | public class HelloApp extends App { 29 | 30 | @Override 31 | protected void onStart() { 32 | } 33 | 34 | @Override 35 | protected void onInit() { 36 | // create a prototype 37 | HelloPE helloPE = createPE(HelloPE.class); 38 | // Create a stream that listens to the "names" stream and passes events to the helloPE instance. 39 | createInputStream("names", new KeyFinder() { 40 | 41 | @Override 42 | public List get(Event event) { 43 | return Arrays.asList(new String[] { event.get("name") }); 44 | } 45 | }, helloPE); 46 | } 47 | 48 | @Override 49 | protected void onClose() { 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/resources/templates/HelloPE.java.txt: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package hello; 20 | 21 | import org.apache.s4.base.Event; 22 | import org.apache.s4.core.ProcessingElement; 23 | 24 | public class HelloPE extends ProcessingElement { 25 | 26 | // you should define downstream streams here and inject them in the app definition 27 | 28 | boolean seen = false; 29 | 30 | /** 31 | * This method is called upon a new Event on an incoming stream 32 | */ 33 | public void onEvent(Event event) { 34 | // in this example, we use the default generic Event type, by you can also define your own type 35 | System.out.println("Hello " + (seen ? "again " : "") + event.get("name") + "!"); 36 | seen = true; 37 | } 38 | 39 | @Override 40 | protected void onCreate() { 41 | } 42 | 43 | @Override 44 | protected void onRemove() { 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/resources/templates/gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | /gradlew $@ -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/resources/templates/newApp.README: -------------------------------------------------------------------------------- 1 | _____ _ _ 2 | / ____|| || | 3 | | (___ | || |_ 4 | \___ \ |__ _| 5 | ____) | | | 6 | |_____/ |_| 7 | 8 | You just created a new S4 project in ! 9 | 10 | It follows a maven-like structure: 11 | - the build file is at the root of the project 12 | - sources are in src/main/java 13 | 14 | We use gradle for building this project. 15 | 16 | To build the project: 17 | - "./gradlew" (from the root of the project, this calls the gradle script from the s4 installation) 18 | 19 | An "s4" script has been created at the root of the project's directory. It calls the s4 script from your S4 installation. 20 | 21 | 22 | To execute the application in a new S4 cluster (see the S4 wiki https://cwiki.apache.org/confluence/display/S4/ for more information): 23 | 1. start a ZooKeeper instance "./s4 zkServer" (./s4 zkServer -help provides a list of options) 24 | 2. define a logical cluster for your application "./s4 newCluster -cluster= -nbTasks= -flp=" 25 | 3. start a node and attach it to the cluster "./s4 node -cluster=" 26 | 4. package the application "./s4 s4r -a= -b=`pwd`/build.gradle 27 | 5. deploy the application "./s4 deploy -s4r=`pwd`/build/libs/.s4r -c= -appName=" 28 | 6. check the status of the S4 cluster "s4 status" 29 | 30 | If you want to inject events from application 2 into application 1 on cluster 1: 31 | - application 1 must define an input stream with a name (say: stream1) 32 | - application 2 must define an output stream with the same name (stream1) 33 | 34 | If you want to use a simple adapter process, listening to an external source, converting incoming data into S4 events, and sending that to S4 apps, you can define 35 | your own app that extends the AdapterApp class. Then to start it (for instance): 36 | - ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter -p=s4.adapter.output.stream=stream1 37 | 38 | -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/resources/templates/s4: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | case "$1" in 4 | "adapter") 5 | # we need something different in order to pass the classpath of the current project 6 | # you must specify: appClassName (extends AdapterApp class) , cluster name, output stream name 7 | # current syntax is not yet straightforward: 8 | # example : ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter -p=s4.adapter.output.stream:s1 9 | shift 1 10 | /gradlew cp 11 | java -cp `cat classpath.txt` org.apache.s4.core.Main $@ 12 | ;; 13 | *) 14 | echo "calling referenced s4 script : " 15 | (cd && $@) 16 | ;; 17 | esac -------------------------------------------------------------------------------- /subprojects/s4-tools/src/main/resources/templates/settings.gradle: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | rootProject.name= 20 | -------------------------------------------------------------------------------- /test-apps/consumer-app/src/main/java/s4app/ConsumerApp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package s4app; 20 | 21 | import org.apache.s4.core.App; 22 | 23 | public class ConsumerApp extends App { 24 | 25 | private ConsumerPE consumerPE; 26 | 27 | @Override 28 | protected void onStart() { 29 | System.out.println("Starting ShowTimeApp..."); 30 | } 31 | 32 | @Override 33 | protected void onInit() { 34 | System.out.println("Initing ShowTimeApp..."); 35 | 36 | ConsumerPE consumerPE = createPE(ConsumerPE.class, "consumer"); 37 | consumerPE.setSingleton(true); 38 | 39 | /* This stream will receive events from another app. */ 40 | createInputStream("tickStream", consumerPE); 41 | } 42 | 43 | @Override 44 | protected void onClose() { 45 | System.out.println("Closing ShowTimeApp..."); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /test-apps/producer-app/src/main/java/s4app/ProducerApp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package s4app; 20 | 21 | import org.apache.s4.core.App; 22 | 23 | public class ProducerApp extends App { 24 | 25 | private ProducerPE producerPE; 26 | 27 | @Override 28 | protected void onStart() { 29 | System.out.println("Starting CounterApp..."); 30 | ((ProducerPE) producerPE.getInstanceForKey("single")).sendMessages(); 31 | } 32 | 33 | // generic array due to varargs generates a warning. 34 | @Override 35 | protected void onInit() { 36 | System.out.println("Initing CounterApp..."); 37 | 38 | producerPE = createPE(ProducerPE.class, "producer"); 39 | producerPE.setStreams(createOutputStream("tickStream")); 40 | 41 | } 42 | 43 | @Override 44 | protected void onClose() { 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/A.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | import org.I0Itec.zkclient.ZkClient; 22 | import org.apache.s4.deploy.AppConstants; 23 | 24 | public class A { 25 | 26 | public A(ZkClient zkClient) { 27 | try { 28 | zkClient.createEphemeral(AppConstants.STARTED_ZNODE_1, null); 29 | } catch (Exception e) { 30 | System.exit(-1); 31 | } 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/AppConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.deploy; 20 | 21 | public class AppConstants { 22 | public static final String STARTED_ZNODE_1 = "/s4-test/test-app-1-started"; 23 | public static final String INITIALIZED_ZNODE_1 = "/s4-test/test-app-1-initialized"; 24 | 25 | public static final String STARTED_ZNODE_2 = "/s4-test/test-app-2-started"; 26 | public static final String INITIALIZED_ZNODE_2 = "/s4-test/test-app-2-initialized"; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /test-apps/simple-deployable-app-1/src/main/resources/resource.txt: -------------------------------------------------------------------------------- 1 | Salut! -------------------------------------------------------------------------------- /test-apps/twitter-adapter/README.txt: -------------------------------------------------------------------------------- 1 | Please refer to README.txt in twitter-counter application -------------------------------------------------------------------------------- /test-apps/twitter-counter/README.txt: -------------------------------------------------------------------------------- 1 | An application that displays the current top 10 topics, as gathered from the twitter sample stream. 2 | It was ported and adapted from S4 0.3 3 | 4 | Architecture: 5 | - twitter-adapter app in adapter node connects to the twitter stream, extracts the tweeted text and passes that to the application cluster 6 | - twitter-counter app in the application cluster receives the text of the tweets, extracts the topics, counts topic occurences and periodically displays the top 10 topics on the console 7 | 8 | How to configure: 9 | - you need a twitter4j.properties file in your home dir, with the following properties filled: 10 | debug=true|false 11 | user= 12 | password= 13 | 14 | How to run: 15 | 16 | Please follow the instructions in the S4 piper walkthrough at the following place: 17 | https://cwiki.apache.org/confluence/display/S4/S4+piper+walkthrough -------------------------------------------------------------------------------- /test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopicEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.s4.example.twitter; 20 | 21 | import org.apache.s4.base.Event; 22 | 23 | /** 24 | * Transports the topic name and count information 25 | * 26 | */ 27 | public class TopicEvent extends Event { 28 | 29 | String topic; 30 | int count; 31 | 32 | public TopicEvent() { 33 | } 34 | 35 | public TopicEvent(String topic, int count) { 36 | this.topic = topic; 37 | this.count = count; 38 | } 39 | 40 | public String getTopic() { 41 | return topic; 42 | } 43 | 44 | public int getCount() { 45 | return count; 46 | } 47 | 48 | } 49 | --------------------------------------------------------------------------------