├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── architecture-overview.jpg ├── build.sh ├── monitoring-setup-overview.jpg ├── multiple-processor-pipeline.jpg ├── pom.xml ├── publisher-subscriber.jpg ├── spqr-base ├── .gitignore ├── pom.xml ├── spqr-base.iml └── src │ ├── main │ └── java │ │ ├── com │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ ├── exception │ │ │ ├── ComponentInitializationFailedException.java │ │ │ ├── NonUniqueIdentifierException.java │ │ │ ├── PipelineInstantiationFailedException.java │ │ │ ├── QueueInitializationFailedException.java │ │ │ ├── RemoteClientConnectionFailedException.java │ │ │ └── RequiredInputMissingException.java │ │ │ ├── node │ │ │ ├── message │ │ │ │ ├── NodeDeRegistration.java │ │ │ │ └── NodeRegistration.java │ │ │ └── resource │ │ │ │ └── pipeline │ │ │ │ ├── ListRegisteredMicroPipelinesResponse.java │ │ │ │ ├── MicroPipelineInstantiationResponse.java │ │ │ │ └── MicroPipelineShutdownResponse.java │ │ │ └── pipeline │ │ │ ├── MicroPipelineConfiguration.java │ │ │ ├── MicroPipelineValidationResult.java │ │ │ ├── MicroPipelineValidator.java │ │ │ ├── component │ │ │ ├── MicroPipelineComponent.java │ │ │ ├── MicroPipelineComponentConfiguration.java │ │ │ ├── MicroPipelineComponentType.java │ │ │ ├── annotation │ │ │ │ └── SPQRComponent.java │ │ │ ├── emitter │ │ │ │ └── Emitter.java │ │ │ ├── operator │ │ │ │ ├── DelayedResponseCollector.java │ │ │ │ ├── DelayedResponseOperator.java │ │ │ │ ├── DelayedResponseOperatorWaitStrategy.java │ │ │ │ ├── DirectResponseOperator.java │ │ │ │ ├── Operator.java │ │ │ │ └── OperatorConfiguration.java │ │ │ └── source │ │ │ │ ├── IncomingMessageCallback.java │ │ │ │ └── Source.java │ │ │ ├── message │ │ │ └── StreamingDataMessage.java │ │ │ ├── metrics │ │ │ ├── MetricReporterType.java │ │ │ └── MicroPipelineMetricsReporterConfiguration.java │ │ │ ├── queue │ │ │ └── StreamingMessageQueueConfiguration.java │ │ │ └── statistics │ │ │ └── MicroPipelineStatistics.java │ │ └── uk │ │ └── co │ │ └── real_logic │ │ └── queues │ │ ├── BlockingWaitStrategy.java │ │ ├── MessageWaitStrategy.java │ │ ├── OneToOneConcurrentArrayQueue3.java │ │ └── PaddedAtomicLong.java │ └── test │ └── java │ └── com │ └── ottogroup │ └── bi │ └── spqr │ └── pipeline │ └── MicroPipelineValidatorTest.java ├── spqr-metrics ├── .gitignore ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── ottogroup │ └── bi │ └── spqr │ └── metrics │ ├── MetricsHandler.java │ ├── MetricsReporterFactory.java │ ├── ResetOnReadCounter.java │ └── kafka │ └── KafkaReporter.java ├── spqr-micro-pipeline ├── .gitignore ├── README.MD ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── pipeline │ │ ├── MicroPipeline.java │ │ ├── MicroPipelineFactory.java │ │ ├── MicroPipelineManager.java │ │ ├── component │ │ ├── emitter │ │ │ └── EmitterRuntimeEnvironment.java │ │ ├── operator │ │ │ ├── DelayedResponseOperatorRuntimeEnvironment.java │ │ │ ├── DirectResponseOperatorRuntimeEnvironment.java │ │ │ ├── MessageCountResponseWaitStrategy.java │ │ │ ├── OperatorTriggeredWaitStrategy.java │ │ │ └── TimerBasedResponseWaitStrategy.java │ │ └── source │ │ │ └── SourceRuntimeEnvironment.java │ │ ├── exception │ │ └── UnknownWaitStrategyException.java │ │ └── queue │ │ ├── StreamingMessageQueue.java │ │ ├── StreamingMessageQueueConsumer.java │ │ ├── StreamingMessageQueueProducer.java │ │ ├── chronicle │ │ ├── DefaultStreamingMessageQueue.java │ │ ├── DefaultStreamingMessageQueueConsumer.java │ │ └── DefaultStreamingMessageQueueProducer.java │ │ ├── memory │ │ ├── InMemoryStreamingMessageQueue.java │ │ ├── InMemoryStreamingMessageQueueConsumer.java │ │ └── InMemoryStreamingMessageQueueProducer.java │ │ └── strategy │ │ ├── StreamingMessageQueueBlockingWaitStrategy.java │ │ ├── StreamingMessageQueueDirectPassStrategy.java │ │ ├── StreamingMessageQueueSleepingWaitStrategy.java │ │ └── StreamingMessageQueueWaitStrategy.java │ └── test │ ├── java │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── pipeline │ │ ├── MicroPipelineFactoryTest.java │ │ ├── MicroPipelineManagerTest.java │ │ ├── MicroPipelineTest.java │ │ └── component │ │ ├── emitter │ │ ├── CountDownLatchTestEmitter.java │ │ └── EmitterRuntimeEnvironmentTest.java │ │ ├── operator │ │ ├── DelayedResponseOperatorRuntimeEnvironmentTest.java │ │ └── DirectResponseOperatorRuntimeEnvironmentTest.java │ │ ├── queue │ │ └── chronicle │ │ │ └── DefaultStreamingMessageQueueTest.java │ │ └── source │ │ ├── RandomNumberTestSource.java │ │ └── SourceRuntimeEnvironmentTest.java │ └── resources │ └── log4j.properties ├── spqr-node ├── .gitignore ├── pom.xml └── src │ ├── main │ ├── config │ │ ├── log4j.properties │ │ └── spqr-node.cfg │ ├── java │ │ └── com │ │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ └── node │ │ │ ├── resman │ │ │ └── SPQRResourceManagerClient.java │ │ │ ├── resource │ │ │ └── pipeline │ │ │ │ └── MicroPipelineResource.java │ │ │ └── server │ │ │ ├── SPQRNodeServer.java │ │ │ ├── SPQRNodeShutdownHandler.java │ │ │ └── cfg │ │ │ ├── SPQRNodeMetricsConfiguration.java │ │ │ ├── SPQRNodeServerConfiguration.java │ │ │ ├── SPQRProcessingNodeConfiguration.java │ │ │ └── SPQRResourceManagerConfiguration.java │ └── scripts │ │ └── spqr-node.sh │ └── test │ ├── java │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── node │ │ ├── resman │ │ └── SPQRResourceManagerClientTest.java │ │ ├── resource │ │ └── pipeline │ │ │ └── MicroPipelineResourceTest.java │ │ └── server │ │ └── SPQRNodeServerTest.java │ └── resources │ ├── log4j.properties │ ├── twitter-filter-to-kafka.json │ ├── twitter-to-kafka.json │ ├── webtrends-filter-to-kafka.json │ └── webtrends-to-kafka.json ├── spqr-operators ├── spqr-esper │ ├── .gitignore │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ottogroup │ │ │ │ └── bi │ │ │ │ └── spqr │ │ │ │ └── operator │ │ │ │ └── esper │ │ │ │ └── EsperOperator.java │ │ └── resources │ │ │ └── log4j.properties │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ └── operator │ │ │ └── esper │ │ │ └── EsperOperatorTest.java │ │ └── resources │ │ └── log4j.properties ├── spqr-json │ ├── .gitignore │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ └── operator │ │ │ └── json │ │ │ ├── JsonContentType.java │ │ │ ├── aggregator │ │ │ ├── JsonContentAggregator.java │ │ │ ├── JsonContentAggregatorFieldSetting.java │ │ │ └── JsonContentAggregatorResult.java │ │ │ ├── filter │ │ │ ├── JsonContentFilter.java │ │ │ └── JsonContentFilterFieldSetting.java │ │ │ └── flatten │ │ │ └── JsonContentFlattening.java │ │ └── test │ │ └── java │ │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── operator │ │ └── json │ │ └── filter │ │ └── JsonContentFilterTest.java ├── spqr-kafka │ ├── .gitignore │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ └── operator │ │ │ └── kafka │ │ │ ├── emitter │ │ │ └── KafkaTopicEmitter.java │ │ │ └── source │ │ │ ├── KafkaTopicSource.java │ │ │ └── KafkaTopicStreamConsumer.java │ │ └── test │ │ └── java │ │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── operator │ │ └── kafka │ │ └── emitter │ │ └── KafkaTopicEmitterTest.java ├── spqr-twitter │ ├── .gitignore │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ └── operator │ │ │ └── twitter │ │ │ └── source │ │ │ └── TwitterStreamSource.java │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ └── operator │ │ │ └── twitter │ │ │ └── source │ │ │ └── TwitterStreamSourceTest.java │ │ └── resources │ │ └── log4j.properties └── spqr-webtrends │ ├── .gitignore │ ├── pom.xml │ └── src │ ├── main │ └── java │ │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── operator │ │ └── webtrends │ │ └── source │ │ ├── WebtrendStreamSource.java │ │ └── WebtrendsTokenRequest.java │ └── test │ └── resources │ └── log4j.properties ├── spqr-repository ├── .gitignore ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── repository │ │ ├── CachedComponentClassLoader.java │ │ ├── ComponentDescriptor.java │ │ ├── ComponentRepository.java │ │ └── exception │ │ ├── ComponentAlreadySubmittedException.java │ │ ├── ComponentInstantiationFailedException.java │ │ └── UnknownComponentException.java │ └── test │ ├── java │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── repository │ │ ├── CachedComponentClassLoaderTest.java │ │ └── ComponentRepositoryTest.java │ └── resources │ └── log4j.properties ├── spqr-resman ├── .gitignore ├── pom.xml └── src │ ├── main │ ├── config │ │ ├── log4j.properties │ │ └── spqr-resman.cfg │ ├── java │ │ └── com │ │ │ └── ottogroup │ │ │ └── bi │ │ │ └── spqr │ │ │ └── resman │ │ │ ├── node │ │ │ ├── SPQRNodeClient.java │ │ │ ├── SPQRNodeManager.java │ │ │ ├── SPQRNodeSupervisor.java │ │ │ └── SPQRNodeSupervisorResult.java │ │ │ ├── resource │ │ │ ├── node │ │ │ │ └── SPQRNodeManagementResource.java │ │ │ └── pipeline │ │ │ │ └── SPQRPipelineManagementResource.java │ │ │ └── server │ │ │ ├── SPQRResourceManagerConfiguration.java │ │ │ ├── SPQRResourceManagerServer.java │ │ │ └── SPQRResourceManagerShutdownHandler.java │ └── scripts │ │ └── spqr-resman.sh │ └── test │ └── java │ └── com │ └── ottogroup │ └── bi │ └── spqr │ └── resman │ ├── node │ ├── SPQRNodeClientTest.java │ └── SPQRNodeManagerTest.java │ └── resource │ └── node │ └── SPQRNodeManagementResourceTest.java └── spqr-websocket-server ├── .gitignore ├── pom.xml └── src ├── main ├── config │ ├── log4j.properties │ └── spqr-websocket-server.cfg ├── java │ └── com │ │ └── ottogroup │ │ └── bi │ │ └── spqr │ │ └── websocket │ │ ├── kafka │ │ ├── KafkaTopicConsumer.java │ │ ├── KafkaTopicRequest.java │ │ ├── KafkaTopicStreamConsumer.java │ │ └── KafkaTopicWebSocketEmitter.java │ │ └── server │ │ ├── SPQRWebSocketServer.java │ │ ├── SPQRWebSocketServerHandler.java │ │ ├── SPQRWebSocketServerInitializer.java │ │ ├── TestClient.java │ │ └── cfg │ │ ├── SPQRKafkaConsumerConfiguration.java │ │ └── SPQRWebSocketServerConfiguration.java └── scripts │ └── spqr-websocket-server.sh └── test ├── java └── com │ └── ottogroup │ └── bi │ └── spqr │ └── websocket │ └── server │ └── SPQRWebSocketServerTest.java └── resources └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .settings/ 3 | .project 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | -------------------------------------------------------------------------------- /architecture-overview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ottogroup/SPQR/c0edd8eaa3378b658b482f98ccf693278e5e94b2/architecture-overview.jpg -------------------------------------------------------------------------------- /monitoring-setup-overview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ottogroup/SPQR/c0edd8eaa3378b658b482f98ccf693278e5e94b2/monitoring-setup-overview.jpg -------------------------------------------------------------------------------- /multiple-processor-pipeline.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ottogroup/SPQR/c0edd8eaa3378b658b482f98ccf693278e5e94b2/multiple-processor-pipeline.jpg -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 3.0.1 6 | 7 | 8 | 9 | com.ottogroup.bi.spqr 10 | spqr-parent 11 | 0.6.0-SNAPSHOT 12 | pom 13 | 14 | 15 | spqr-build 16 | SPQR build project 17 | https://github.com/ottogroup/spqr.git 18 | 2014 19 | 20 | 21 | spqr-base 22 | spqr-repository 23 | spqr-metrics 24 | spqr-micro-pipeline 25 | spqr-node 26 | spqr-resman 27 | spqr-websocket-server 28 | spqr-operators/spqr-kafka 29 | spqr-operators/spqr-twitter 30 | spqr-operators/spqr-webtrends 31 | spqr-operators/spqr-json 32 | spqr-operators/spqr-esper 33 | 34 | 35 | 36 | 37 | Otto (GmbH & Co KG) 38 | http://www.ottogroup.com 39 | 40 | 41 | 42 | 43 | 44 | mnxfst 45 | christian.kreutzfeldt@ottogroup.com 46 | Christian Kreutzfeldt 47 | Otto (GmbH & Co KG) 48 | http://www.ottogroup.com 49 | 50 | Project Lead 51 | Architect 52 | Developer 53 | 54 | http://twitter.com/mnxfst 55 | Europe/Berlin 56 | 57 | 58 | gwesterm 59 | gerdutz.westermann@ottogroup.com 60 | Otto (GmbH & Co KG) 61 | http://www.ottogroup.com 62 | 63 | Developer 64 | Advisor 65 | 66 | Europe/Berlin 67 | 68 | 69 | 70 | 71 | 72 | Apache License, Version 2.0 73 | http://www.apache.org/licenses/LICENSE-2.0.txt 74 | repo 75 | 76 | 77 | 78 | 79 | scm:git:git://github.com/ottogroup/SPQR.git 80 | scm:git:git@github.com:ottogroup/SPQR.git 81 | https://github.com/ottogroup/SPQR 82 | HEAD 83 | 84 | 85 | 86 | GitHub 87 | https://github.com/ottogroup/SPQR/issues 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /publisher-subscriber.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ottogroup/SPQR/c0edd8eaa3378b658b482f98ccf693278e5e94b2/publisher-subscriber.jpg -------------------------------------------------------------------------------- /spqr-base/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.settings 3 | /bin 4 | /.classpath 5 | /.project 6 | /target/ 7 | -------------------------------------------------------------------------------- /spqr-base/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | 5 | com.ottogroup.bi.spqr 6 | spqr-parent 7 | 0.6.0-SNAPSHOT 8 | 9 | 10 | 11 | spqr-base 12 | jar 13 | 14 | 15 | spqr-base 16 | SPQR base provides commonly used components 17 | https://github.com/ottogroup/SPQR.git 18 | 2015 19 | 20 | 21 | 22 | Apache License, Version 2.0 23 | http://www.apache.org/licenses/LICENSE-2.0.txt 24 | repo 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | junit 33 | junit 34 | 4.11 35 | test 36 | 37 | 38 | 39 | org.mockito 40 | mockito-all 41 | 1.10.8 42 | test 43 | 44 | 45 | 46 | 47 | log4j 48 | log4j 49 | 1.2.17 50 | compile 51 | 52 | 53 | 54 | 55 | org.apache.commons 56 | commons-lang3 57 | 3.3.2 58 | compile 59 | 60 | 61 | 62 | 63 | com.fasterxml.jackson.core 64 | jackson-databind 65 | 2.5.1 66 | compile 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-compiler-plugin 79 | 3.2 80 | 81 | 1.7 82 | 1.7 83 | UTF-8 84 | 85 | 86 | 87 | 88 | 89 | maven-dependency-plugin 90 | 91 | 92 | package 93 | 94 | copy-dependencies 95 | 96 | 97 | provided 98 | junit,org.slf4j,org.mockito,log4j 99 | ${project.build.directory}/lib 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /spqr-base/spqr-base.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/exception/ComponentInitializationFailedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.exception; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent; 19 | 20 | /** 21 | * Thrown in case the initialization of a {@link MicroPipelineComponent} failed 22 | * @author mnxfst 23 | * @since Mar 6, 2015 24 | */ 25 | public class ComponentInitializationFailedException extends Exception { 26 | 27 | private static final long serialVersionUID = -7893237496464557534L; 28 | 29 | public ComponentInitializationFailedException() { 30 | } 31 | 32 | public ComponentInitializationFailedException(String message) { 33 | super(message); 34 | } 35 | 36 | public ComponentInitializationFailedException(Throwable cause) { 37 | super(cause); 38 | } 39 | 40 | public ComponentInitializationFailedException(String message, Throwable cause) { 41 | super(message, cause); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/exception/NonUniqueIdentifierException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.exception; 17 | 18 | /** 19 | * @author mnxfst 20 | * @since Mar 13, 2015 21 | */ 22 | public class NonUniqueIdentifierException extends Exception { 23 | 24 | private static final long serialVersionUID = 1683810160081066568L; 25 | 26 | public NonUniqueIdentifierException() { 27 | } 28 | 29 | public NonUniqueIdentifierException(String message) { 30 | super(message); 31 | } 32 | 33 | public NonUniqueIdentifierException(Throwable cause) { 34 | super(cause); 35 | } 36 | 37 | public NonUniqueIdentifierException(String message, Throwable cause) { 38 | super(message, cause); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/exception/PipelineInstantiationFailedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.exception; 17 | 18 | 19 | /** 20 | * Thrown in case the instantiation of a {@link MicroPipeline} failed for any reason 21 | * @author mnxfst 22 | * @since Mar 13, 2015 23 | */ 24 | public class PipelineInstantiationFailedException extends Exception { 25 | 26 | private static final long serialVersionUID = -2785833660966006782L; 27 | 28 | public PipelineInstantiationFailedException() { 29 | } 30 | 31 | public PipelineInstantiationFailedException(String message) { 32 | super(message); 33 | } 34 | 35 | public PipelineInstantiationFailedException(Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public PipelineInstantiationFailedException(String message, Throwable cause) { 40 | super(message, cause); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/exception/QueueInitializationFailedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.exception; 17 | 18 | 19 | /** 20 | * Thrown in case the initialization of a {@link StreamingMessageQueue} failed 21 | * @author mnxfst 22 | * @since Mar 6, 2015 23 | */ 24 | public class QueueInitializationFailedException extends Exception { 25 | 26 | private static final long serialVersionUID = 7131295681971026459L; 27 | 28 | public QueueInitializationFailedException() { 29 | } 30 | 31 | public QueueInitializationFailedException(String message) { 32 | super(message); 33 | } 34 | 35 | public QueueInitializationFailedException(Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public QueueInitializationFailedException(String message, Throwable cause) { 40 | super(message, cause); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/exception/RemoteClientConnectionFailedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.exception; 17 | 18 | 19 | /** 20 | * Exception thrown by client on any failure during remote api access 21 | * @author mnxfst 22 | * @since Apr 13, 2015 23 | */ 24 | public class RemoteClientConnectionFailedException extends Exception { 25 | 26 | private static final long serialVersionUID = 2259833389466539291L; 27 | 28 | public RemoteClientConnectionFailedException() { 29 | } 30 | 31 | public RemoteClientConnectionFailedException(String message) { 32 | super(message); 33 | } 34 | 35 | public RemoteClientConnectionFailedException(Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public RemoteClientConnectionFailedException(String message, Throwable cause) { 40 | super(message, cause); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/exception/RequiredInputMissingException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.spqr.exception; 18 | 19 | /** 20 | * Thrown in case a method misses a required input 21 | * @author mnxfst 22 | * @since 12.06.2014 23 | * 24 | */ 25 | public class RequiredInputMissingException extends Exception { 26 | 27 | private static final long serialVersionUID = 4327217901687575387L; 28 | 29 | public RequiredInputMissingException() { 30 | } 31 | 32 | public RequiredInputMissingException(String message) { 33 | super(message); 34 | } 35 | 36 | public RequiredInputMissingException(Throwable cause) { 37 | super(cause); 38 | } 39 | 40 | public RequiredInputMissingException(String message, Throwable cause) { 41 | super(message, cause); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/node/message/NodeDeRegistration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.node.message; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.fasterxml.jackson.annotation.JsonRootName; 22 | 23 | /** 24 | * All necessary information required for processing node de-registration 25 | * @author mnxfst 26 | * @since Apr 14, 2015 27 | */ 28 | public class NodeDeRegistration implements Serializable { 29 | 30 | private static final long serialVersionUID = 2883144319190044706L; 31 | 32 | public enum NodeDeRegistrationState implements Serializable { 33 | OK, MISSING_NODE_ID, NO_SUCH_NODE_ID, TECHNICAL_ERROR 34 | } 35 | 36 | /** 37 | * Response to de-registration request 38 | * @author mnxfst 39 | * @since Apr 14, 2015 40 | */ 41 | @JsonRootName(value="nodeDeRegistrationRequest") 42 | public static class NodeDeRegistrationResponse implements Serializable { 43 | 44 | private static final long serialVersionUID = -1108469543678244332L; 45 | 46 | @JsonProperty(value="id", required=true) 47 | private String id = null; 48 | @JsonProperty(value="state", required=true) 49 | private NodeDeRegistrationState state = NodeDeRegistrationState.OK; 50 | @JsonProperty(value="message", required=true) 51 | private String message = null; 52 | 53 | public NodeDeRegistrationResponse() { 54 | } 55 | 56 | public NodeDeRegistrationResponse(final String id, final NodeDeRegistrationState state, final String message) { 57 | this.id = id; 58 | this.state = state; 59 | this.message = message; 60 | } 61 | 62 | public String getId() { 63 | return id; 64 | } 65 | 66 | public void setId(String id) { 67 | this.id = id; 68 | } 69 | 70 | public NodeDeRegistrationState getState() { 71 | return state; 72 | } 73 | 74 | public void setState(NodeDeRegistrationState state) { 75 | this.state = state; 76 | } 77 | 78 | public String getMessage() { 79 | return message; 80 | } 81 | 82 | public void setMessage(String message) { 83 | this.message = message; 84 | } 85 | 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/node/resource/pipeline/ListRegisteredMicroPipelinesResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.node.resource.pipeline; 17 | 18 | import java.io.Serializable; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | import java.util.Set; 22 | 23 | import com.fasterxml.jackson.annotation.JsonProperty; 24 | import com.fasterxml.jackson.annotation.JsonRootName; 25 | import com.ottogroup.bi.spqr.pipeline.MicroPipelineConfiguration; 26 | 27 | /** 28 | * Lists the identifiers of all registered micro pipelines on the given processing node 29 | * @author mnxfst 30 | * @since May 27, 2015 31 | */ 32 | @JsonRootName(value="registeredMicroPipelinesResponse") 33 | public class ListRegisteredMicroPipelinesResponse implements Serializable { 34 | 35 | private static final long serialVersionUID = -8492445888382782367L; 36 | 37 | @JsonProperty(value="node", required=true) 38 | private String node = null; 39 | @JsonProperty(value="pids", required=true) 40 | private Set pipelineIds = null; 41 | @JsonProperty(value="cfgs", required=true) 42 | private Map configurations = new HashMap<>(); 43 | 44 | public ListRegisteredMicroPipelinesResponse() { 45 | } 46 | 47 | public ListRegisteredMicroPipelinesResponse(final String node, final Set pipelineIds) { 48 | this.pipelineIds = pipelineIds; 49 | this.node = node; 50 | } 51 | 52 | public ListRegisteredMicroPipelinesResponse(final String node, final Set pipelineIds, final Map configurations) { 53 | this.pipelineIds = pipelineIds; 54 | this.node = node; 55 | this.configurations = configurations; 56 | } 57 | 58 | public String getNode() { 59 | return node; 60 | } 61 | 62 | public void setNode(String node) { 63 | this.node = node; 64 | } 65 | 66 | public Set getPipelineIds() { 67 | return pipelineIds; 68 | } 69 | 70 | public void setPipelineIds(Set pipelineIds) { 71 | this.pipelineIds = pipelineIds; 72 | } 73 | 74 | public Map getConfigurations() { 75 | return configurations; 76 | } 77 | 78 | public void setConfigurations( 79 | Map configurations) { 80 | this.configurations = configurations; 81 | } 82 | 83 | 84 | } 85 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/node/resource/pipeline/MicroPipelineInstantiationResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.node.resource.pipeline; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.ottogroup.bi.spqr.pipeline.MicroPipelineValidationResult; 22 | 23 | /** 24 | * Message sent in response to pipeline instantiation request issued towards a spqr node 25 | * @author mnxfst 26 | * @since Mar 13, 2015 27 | */ 28 | public class MicroPipelineInstantiationResponse implements Serializable { 29 | 30 | private static final long serialVersionUID = -4416878440036933504L; 31 | 32 | @JsonProperty(value="state", required=true) 33 | private MicroPipelineValidationResult state = MicroPipelineValidationResult.OK; 34 | @JsonProperty(value="msg", required=true) 35 | private String message = null; 36 | @JsonProperty(value="pid", required=true) 37 | private String pipelineId = null; 38 | 39 | public MicroPipelineInstantiationResponse() { 40 | } 41 | 42 | public MicroPipelineInstantiationResponse(final String pipelineId, final MicroPipelineValidationResult state, final String message) { 43 | this.pipelineId = pipelineId; 44 | this.state = state; 45 | this.message = message; 46 | } 47 | 48 | public MicroPipelineValidationResult getState() { 49 | return state; 50 | } 51 | 52 | public void setState(MicroPipelineValidationResult state) { 53 | this.state = state; 54 | } 55 | 56 | public String getMessage() { 57 | return message; 58 | } 59 | 60 | public void setMessage(String message) { 61 | this.message = message; 62 | } 63 | 64 | public String getPipelineId() { 65 | return pipelineId; 66 | } 67 | 68 | public void setPipelineId(String pipelineId) { 69 | this.pipelineId = pipelineId; 70 | } 71 | 72 | 73 | } 74 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/node/resource/pipeline/MicroPipelineShutdownResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.node.resource.pipeline; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | /** 23 | * Message sent in response to pipeline shutdown request issued towards a spqr node 24 | * @author mnxfst 25 | * @since Mar 16, 2015 26 | */ 27 | public class MicroPipelineShutdownResponse implements Serializable { 28 | 29 | private static final long serialVersionUID = 212062638728588678L; 30 | 31 | public enum MicroPipelineShutdownState implements Serializable { 32 | OK, PIPELINE_ID_MISSING, TECHNICAL_ERROR 33 | } 34 | 35 | @JsonProperty(value="state", required=true) 36 | private MicroPipelineShutdownState state = MicroPipelineShutdownState.OK; 37 | @JsonProperty(value="msg", required=true) 38 | private String message = null; 39 | @JsonProperty(value="pid", required=true) 40 | private String pipelineId = null; 41 | 42 | public MicroPipelineShutdownResponse() { 43 | } 44 | 45 | /** 46 | * Initializes the response using the provided input 47 | * @param pipelineId 48 | * @param state 49 | * @param msg 50 | */ 51 | public MicroPipelineShutdownResponse(final String pipelineId, final MicroPipelineShutdownState state, final String msg) { 52 | this.pipelineId = pipelineId; 53 | this.state = state; 54 | this.message = msg; 55 | } 56 | 57 | public MicroPipelineShutdownState getState() { 58 | return state; 59 | } 60 | 61 | public void setState(MicroPipelineShutdownState state) { 62 | this.state = state; 63 | } 64 | 65 | public String getMessage() { 66 | return message; 67 | } 68 | 69 | public void setMessage(String message) { 70 | this.message = message; 71 | } 72 | 73 | public String getPipelineId() { 74 | return pipelineId; 75 | } 76 | 77 | public void setPipelineId(String pipelineId) { 78 | this.pipelineId = pipelineId; 79 | } 80 | 81 | 82 | } 83 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/MicroPipelineConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline; 17 | 18 | import java.io.Serializable; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import com.fasterxml.jackson.annotation.JsonProperty; 23 | import com.fasterxml.jackson.annotation.JsonRootName; 24 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponentConfiguration; 25 | import com.ottogroup.bi.spqr.pipeline.metrics.MicroPipelineMetricsReporterConfiguration; 26 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConfiguration; 27 | 28 | /** 29 | * Configuration required for setting up a {@link MicroPipeline} instance 30 | * @author mnxfst 31 | * @since Mar 6, 2015 32 | */ 33 | @JsonRootName(value="microPipelineConfiguration") 34 | public class MicroPipelineConfiguration implements Serializable { 35 | 36 | private static final long serialVersionUID = 3063209038310841880L; 37 | 38 | /** pipeline identifier which must be unique within the cluster */ 39 | @JsonProperty(value="id", required=true) 40 | private String id = null; 41 | /** configuration of queues to be used by components */ 42 | @JsonProperty(value="queues", required=true) 43 | private List queues = new ArrayList<>(); 44 | /** component configurations */ 45 | @JsonProperty(value="components", required=true) 46 | private List components = new ArrayList<>(); 47 | /** metrics reporter */ 48 | @JsonProperty(value="metricsReporter", required=false) 49 | private List metricsReporter = new ArrayList(); 50 | 51 | public String getId() { 52 | return id; 53 | } 54 | public void setId(String id) { 55 | this.id = id; 56 | } 57 | public List getQueues() { 58 | return queues; 59 | } 60 | public void setQueues(List queues) { 61 | this.queues = queues; 62 | } 63 | public List getComponents() { 64 | return components; 65 | } 66 | public void setComponents(List components) { 67 | this.components = components; 68 | } 69 | public List getMetricsReporter() { 70 | return metricsReporter; 71 | } 72 | public void setMetricsReporter( 73 | List metricsReporter) { 74 | this.metricsReporter = metricsReporter; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/MicroPipelineValidationResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * Result states returned by {@link MicroPipelineValidator#validate(MicroPipelineConfiguration)} on config validation 22 | * @author mnxfst 23 | * @since Apr 13, 2015 24 | */ 25 | public enum MicroPipelineValidationResult implements Serializable { 26 | 27 | // general results 28 | OK, // configuration is valid (no errors found that could be inferred from configuration 29 | MISSING_CONFIGURATION, // configuration object is missing at all 30 | MISSING_COMPONENTS, // configuration misses component configuration which renders the queue useless 31 | MISSING_QUEUES, // configuration misses queue configuration and thus communication between components is impossible 32 | PIPELINE_INITIALIZATION_FAILED, // initialization of the overall pipeline failed for any reason (see optional text message in response if provided) 33 | QUEUE_INITIALIZATION_FAILED, // initialization of a configured queue failed for any reason (see optional text message in response if provided) 34 | COMPONENT_INITIALIZATION_FAILED, // initialization of a configured component failed for any reason (see optional text message in response if provided) 35 | TECHNICAL_ERROR, // general error ... no categorization possible 36 | NON_UNIQUE_PIPELINE_ID, // 37 | MISSING_PIPELINE_ID, // pipeline identifier missing - mostly generated ... thus: generation error 38 | 39 | // component specific results 40 | MISSING_COMPONENT_CONFIGURATION, // provided component configuration is empty 41 | MISSING_COMPONENT_ID, // any of the provided component configurations misses the required identifier 42 | NON_UNIQUE_COMPONENT_ID, // any of the provided component configurations shows a non-unique (value is pipeline bounded) identifier 43 | MISSING_COMPONENT_TYPE, // any of the provided component configurations misses the required type 44 | MISSING_COMPONENT_NAME, // any of the provided component configurations misses the required name 45 | MISSING_COMPONENT_VERSION, // any of the provided component configurations misses the required version 46 | UNKNOWN_COMPONENT_NAME, // any of the provided component configurations references an unknown component name 47 | UNKNOWN_COMPONENT_VERSION, // any of the provided component configurations references an unknown component version 48 | MISSING_SOURCE_QUEUE, // any of the operator or emitter components shows no source queue 49 | UNKNOWN_SOURCE_QUEUE, // any of the operator or emitter components reference an unknown source queue 50 | NOT_PERMITTED_SOURCE_QUEUE_REF, // any of the source components reference a source queue which is not permitted for that type 51 | MISSING_DESTINATION_QUEUE, // any of the source or operator components shows no destination queue 52 | UNKNOWN_DESTINATION_QUEUE, // any of the source or operator components reference an unknown destination queue 53 | NOT_PERMITTED_DESTINATION_QUEUE_REF, // any of the emitter components reference a destination queue which is not permitted for that type 54 | 55 | // queue specific results 56 | MISSING_QUEUE_CONFIGURATION, // provided queue configuration is empty 57 | MISSING_QUEUE_ID, // any of the provided queue configurations misses the required identifier 58 | NON_UNIQUE_QUEUE_ID // any of the provided queue configurations shows a non-unique (value is pipeline bounded) identifier 59 | } 60 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/MicroPipelineComponent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component; 17 | 18 | import java.util.Properties; 19 | 20 | import com.ottogroup.bi.spqr.exception.ComponentInitializationFailedException; 21 | import com.ottogroup.bi.spqr.exception.RequiredInputMissingException; 22 | import com.ottogroup.bi.spqr.pipeline.component.emitter.Emitter; 23 | import com.ottogroup.bi.spqr.pipeline.component.operator.Operator; 24 | import com.ottogroup.bi.spqr.pipeline.component.source.Source; 25 | 26 | /** 27 | * Interface to the three major building blocks of a {@link MicroPipeline}: {@link Source}, {@link Operator} and {@link Emitter} 28 | * @author mnxfst 29 | * @since Dec 15, 2014 30 | */ 31 | public interface MicroPipelineComponent { 32 | 33 | /** 34 | * Sets the component identifier 35 | * @param id 36 | */ 37 | public void setId(final String id); 38 | 39 | /** 40 | * Returns the component identifier which is unique 41 | * within the boundaries of a {@link MicroPipeline} 42 | * @return 43 | */ 44 | public String getId(); 45 | 46 | /** 47 | * Initializes the component before it gets executed 48 | * @param properties 49 | * @throws RequiredInputMissingException input missing required to initialize the component 50 | * @throws ComponentInitializationFailedException failed to initialize the component 51 | */ 52 | public void initialize(final Properties properties) throws RequiredInputMissingException, ComponentInitializationFailedException; 53 | 54 | /** 55 | * Tells the component to shut itself down 56 | */ 57 | public boolean shutdown(); 58 | 59 | /** 60 | * Returns the {@link MicroPipelineComponentType} 61 | * @return 62 | */ 63 | public MicroPipelineComponentType getType(); 64 | } 65 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/MicroPipelineComponentConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component; 17 | 18 | import java.io.Serializable; 19 | import java.util.Properties; 20 | 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.fasterxml.jackson.annotation.JsonRootName; 23 | 24 | /** 25 | * Configuration required for setting up a {@link MicroPipelineComponent} instance 26 | * @author mnxfst 27 | * @since Mar 6, 2015 28 | */ 29 | @JsonRootName(value="componentConfiguration") 30 | public class MicroPipelineComponentConfiguration implements Serializable { 31 | 32 | private static final long serialVersionUID = 3896521083804105532L; 33 | 34 | /** component identifier */ 35 | @JsonProperty(value="id", required=true) 36 | private String id = null; 37 | /** component type: SOURCE, OPERATOR, ... */ 38 | @JsonProperty(value="type", required=true) 39 | private MicroPipelineComponentType type = null; 40 | /** component class */ 41 | @JsonProperty(value="name", required=true) 42 | private String name = null; 43 | /** component version */ 44 | @JsonProperty(value="version", required=true) 45 | private String version = null; 46 | /** settings required for component configuration */ 47 | @JsonProperty(value="settings", required=true) 48 | private Properties settings = new Properties(); 49 | /** identifier of queues to consume content from */ 50 | @JsonProperty(value="fromQueue", required=true) 51 | private String fromQueue = null; 52 | /** identifier of queues to produce content to */ 53 | @JsonProperty(value="toQueue", required=true) 54 | private String toQueue = null; 55 | /** attach message counter */ 56 | @JsonProperty(value="attachMessageCounter", required=false) 57 | private boolean attachMessageCounter = false; 58 | /** attach processing timer */ 59 | @JsonProperty(value="attachProcessingTimer", required=false) 60 | private boolean attachProcessingTimer = false; 61 | 62 | 63 | public String getId() { 64 | return id; 65 | } 66 | public void setId(String id) { 67 | this.id = id; 68 | } 69 | public MicroPipelineComponentType getType() { 70 | return type; 71 | } 72 | public void setType(MicroPipelineComponentType type) { 73 | this.type = type; 74 | } 75 | public Properties getSettings() { 76 | return settings; 77 | } 78 | public void setSettings(Properties settings) { 79 | this.settings = settings; 80 | } 81 | 82 | public String getName() { 83 | return name; 84 | } 85 | public void setName(String name) { 86 | this.name = name; 87 | } 88 | 89 | public String getVersion() { 90 | return version; 91 | } 92 | public void setVersion(String version) { 93 | this.version = version; 94 | } 95 | public String getFromQueue() { 96 | return fromQueue; 97 | } 98 | public void setFromQueue(String fromQueue) { 99 | this.fromQueue = fromQueue; 100 | } 101 | public String getToQueue() { 102 | return toQueue; 103 | } 104 | public void setToQueue(String toQueue) { 105 | this.toQueue = toQueue; 106 | } 107 | public boolean isAttachMessageCounter() { 108 | return attachMessageCounter; 109 | } 110 | public void setAttachMessageCounter(boolean attachMessageCounter) { 111 | this.attachMessageCounter = attachMessageCounter; 112 | } 113 | public boolean isAttachProcessingTimer() { 114 | return attachProcessingTimer; 115 | } 116 | public void setAttachProcessingTimer(boolean attachProcessingTimer) { 117 | this.attachProcessingTimer = attachProcessingTimer; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/MicroPipelineComponentType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * Names all known types of {@link MicroPipelineComponent} 22 | * @author mnxfst 23 | * @since Mar 6, 2015 24 | */ 25 | public enum MicroPipelineComponentType implements Serializable { 26 | SOURCE, DIRECT_RESPONSE_OPERATOR, DELAYED_RESPONSE_OPERATOR, EMITTER 27 | } 28 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/annotation/SPQRComponent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.spqr.pipeline.component.annotation; 18 | 19 | import static java.lang.annotation.ElementType.TYPE; 20 | 21 | import java.lang.annotation.Documented; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.RetentionPolicy; 24 | import java.lang.annotation.Target; 25 | 26 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponentType; 27 | import com.ottogroup.bi.spqr.pipeline.component.emitter.Emitter; 28 | import com.ottogroup.bi.spqr.pipeline.component.operator.Operator; 29 | import com.ottogroup.bi.spqr.pipeline.component.source.Source; 30 | 31 | 32 | /** 33 | * Used to mark an implementation to be an SPQR {@link MicroPipelineComponentType component} ({@link Source}, {@link Operator} or {@link Emitter}) 34 | * @author mnxfst 35 | * @since Oct 29, 2014 36 | * 37 | */ 38 | @Documented 39 | @Target ( { TYPE } ) 40 | @Retention ( value = RetentionPolicy.RUNTIME ) 41 | public @interface SPQRComponent { 42 | 43 | MicroPipelineComponentType type(); 44 | String name(); 45 | String version(); 46 | String description(); 47 | } 48 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/emitter/Emitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.emitter; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent; 19 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 20 | 21 | /** 22 | * To be implemented by all {@link MicroPipelineComponent micro pipeline components} that export data 23 | * to any destination, eg. Kafka, ElasticSearch or file system 24 | * @author mnxfst 25 | * @since Dec 15, 2014 26 | */ 27 | public interface Emitter extends MicroPipelineComponent { 28 | 29 | /** 30 | * Provides a new message to the operator 31 | * @param message 32 | */ 33 | public boolean onMessage(final StreamingDataMessage message); 34 | 35 | /** 36 | * Returns the total number of messages processed by this component 37 | * @return 38 | */ 39 | public long getTotalNumOfMessages(); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/DelayedResponseCollector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 19 | 20 | /** 21 | * Interface to be implemented by all classes that may be triggered by the {@link DelayedResponseOperatorWaitStrategy} 22 | * to retrieve {@link StreamingDataMessage} instances from a {@link DelayedResponseOperator} 23 | * @author mnxfst 24 | * @since Mar 11, 2015 25 | */ 26 | public interface DelayedResponseCollector { 27 | 28 | /** 29 | * Signal to retrieve messages from {@link DelayedResponseOperator} 30 | */ 31 | public void retrieveMessages(); 32 | 33 | } 34 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/DelayedResponseOperator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 19 | 20 | 21 | /** 22 | * Receives an input {@link StreamingDataMessage message} but defers its results to be fetched later, eg. aggregate or group operator. The 23 | * configuration must reference a {@link DelayedResponseOperatorWaitStrategy} to be applied when executing this operator. The strategy must 24 | * be referenced inside the operator settings using the key named by {@link DelayedResponseOperator#CFG_WAIT_STRATEGY_NAME}. Additional 25 | * settings required for strategy initialization must be provided using the content of {@link DelayedResponseOperator#CFG_WAIT_STRATEGY_SETTINGS_PREFIX} 26 | * as prefix followed by the options name and its value, eg. waitStrategy.cfg.testKey=testValue. 27 | * @author mnxfst 28 | * @since Dec 14, 2014 29 | * TODO exporting delayed result may be triggered by internal timer/counter which is provided from the outside?! 30 | */ 31 | public interface DelayedResponseOperator extends Operator { 32 | 33 | public static final String CFG_WAIT_STRATEGY_NAME = "waitStrategy.name"; 34 | public static final String CFG_WAIT_STRATEGY_SETTINGS_PREFIX = "waitStrategy.cfg."; 35 | 36 | /** 37 | * Provides a new message to the operator 38 | * @param message 39 | */ 40 | public void onMessage(final StreamingDataMessage message); 41 | 42 | /** 43 | * Accesses the result generated by the operator 44 | * @return 45 | */ 46 | public StreamingDataMessage[] getResult(); 47 | 48 | /** 49 | * Returns the number of messages processed since the last call 50 | * of {@link DelayedResponseOperator#getResult()} 51 | * @return 52 | */ 53 | public long getNumberOfMessagesSinceLastResult(); 54 | 55 | /** 56 | * Assigns the {@link DelayedResponseOperatorWaitStrategy} used for triggering result retrieval 57 | * @param waitStrategy 58 | */ 59 | public void setWaitStrategy(final DelayedResponseOperatorWaitStrategy waitStrategy); 60 | } 61 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/DelayedResponseOperatorWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import java.util.Properties; 19 | 20 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 21 | 22 | /** 23 | * Interface to implement when providing a new strategy to be applied on {@link DelayedResponseOperator} 24 | * while waiting before sending response messages 25 | * @author mnxfst 26 | * @since Mar 11, 2015 27 | */ 28 | public interface DelayedResponseOperatorWaitStrategy extends Runnable { 29 | 30 | /** 31 | * Assigns the {@link DelayedResponseCollector message collector} for retrieving delayed 32 | * content from {@link DelayedResponseOperator} 33 | * @param delayedResponseCollector 34 | */ 35 | public void setDelayedResponseCollector(final DelayedResponseCollector delayedResponseCollector); 36 | 37 | /** 38 | * Signals towards the wait strategy that message retrieval must be forced no matter 39 | * if the condition evaluated by the strategy holds or not. The instance must be 40 | * re-set to mark a new collection period afterwards. 41 | */ 42 | public void release(); 43 | 44 | /** 45 | * Notifies the wait strategy about each new {@link StreamingDataMessage} passed to the attached 46 | * {@link DelayedResponseOperator}. The strategy may ignore it as it is timer based or evaluate its 47 | * body as it is content based or simply count the number of incoming messages as it is volume based. 48 | * @param message 49 | */ 50 | public void onMessage(final StreamingDataMessage message); 51 | 52 | /** 53 | * Signals the strategy to shut itself down 54 | */ 55 | public void shutdown(); 56 | 57 | /** 58 | * Initializes the wait strategy using the provided {@link Properties} 59 | * @param properties 60 | */ 61 | public void initialize(final Properties properties); 62 | 63 | } 64 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/DirectResponseOperator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 19 | 20 | 21 | /** 22 | * Receives an input {@link StreamingDataMessage message} and generates a response immediately, eg. filter, map or merge operator 23 | * @author mnxfst 24 | * @since Dec 14, 2014 25 | */ 26 | public interface DirectResponseOperator extends Operator { 27 | 28 | /** 29 | * Receives a single message, processes its contents and responds with zero 30 | * or multiple {@link StreamingDataMessage} instances 31 | * @param message 32 | * @return 33 | */ 34 | public StreamingDataMessage[] onMessage(final StreamingDataMessage message); 35 | } 36 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/Operator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent; 19 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 20 | 21 | 22 | /** 23 | * Basic interface to be implemented by all {@link StreamingDataMessage} processing tasks 24 | * @author mnxfst 25 | * @since Dec 14, 2014 26 | */ 27 | public interface Operator extends MicroPipelineComponent { 28 | 29 | 30 | /** 31 | * Returns the total number of messages processed by this component 32 | * @return 33 | */ 34 | public long getTotalNumOfMessages(); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/OperatorConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import java.io.Serializable; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | import java.util.Properties; 22 | 23 | import com.fasterxml.jackson.annotation.JsonProperty; 24 | import com.fasterxml.jackson.annotation.JsonRootName; 25 | 26 | /** 27 | * Describes the operator 28 | * @author mnxfst 29 | * @since Mar 5, 2015 30 | */ 31 | @JsonRootName(value="operatorConfiguration") 32 | public class OperatorConfiguration implements Serializable { 33 | 34 | private static final long serialVersionUID = -2761386244558266748L; 35 | 36 | /** operator identifier which must be unique within the boundaries of the {@link MicroPipeline} the operator belongs to */ 37 | @JsonProperty(value="id", required=true) 38 | private String id = null; 39 | /** inbox identifier which must be unique within the boundaries of the {@link MicroPipeline} the operator belongs to */ 40 | @JsonProperty(value="inbox", required=true) 41 | private String inboxId = null; 42 | /** optional inbox settings - typically provided by {@link MicroPipeline} in case a different implementation is selected in favor of the default */ 43 | @JsonProperty(value="inboxSettings", required=false) 44 | private Properties inboxSettings = new Properties(); 45 | /** outboxes the operator distributes its output to - only the identifiers are provided as additional configuration from the inbox configuration */ 46 | @JsonProperty(value="outboxes", required=true) 47 | private Map outboxIds = new HashMap<>(); 48 | 49 | 50 | } 51 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/source/IncomingMessageCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.source; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 19 | 20 | 21 | /** 22 | * Callback invoked for each {@link StreamingDataMessage} received by a {@link Source}. An instance 23 | * of a class implementing this interface is passed on to the source by invoking 24 | * {@link Source#setIncomingMessageCallbale(IncomingMessageCallback)}. The source implementation must 25 | * take care of it and ensure that incoming messages are handed over in order to get them into to 26 | * {@link MicroPipeline} for further processing. 27 | * @author mnxfst 28 | * @since Mar 5, 2015 29 | */ 30 | public interface IncomingMessageCallback { 31 | 32 | /** 33 | * Executed for each incoming {@link StreamingDataMessage} received by a {@link Source}. 34 | * @param message 35 | */ 36 | public void onMessage(final StreamingDataMessage message); 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/component/source/Source.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.source; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent; 19 | 20 | /** 21 | * To be implemented by all {@link MicroPipelineComponent components} that receive data 22 | * from any source and provide it to the {@link Pipeline} 23 | * @author mnxfst 24 | * @since Dec 15, 2014 25 | */ 26 | public interface Source extends MicroPipelineComponent, Runnable { 27 | 28 | /** 29 | * Assigns the {@link IncomingMessageCallback} that must be called for each 30 | * incoming message 31 | * @param incomingMessageCallback 32 | */ 33 | public void setIncomingMessageCallback(final IncomingMessageCallback incomingMessageCallback); 34 | 35 | } -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/message/StreamingDataMessage.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.message; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.fasterxml.jackson.annotation.JsonRootName; 22 | 23 | /** 24 | * Data structure used to transport data through a {@link MicroPipeline} 25 | * @author mnxfst 26 | * @since Mar 5, 2015 27 | */ 28 | @JsonRootName ( value = "streamingDataSourceMessage" ) 29 | public class StreamingDataMessage implements Serializable { 30 | 31 | private static final long serialVersionUID = 1280809436656124315L; 32 | 33 | /** message body a.k.a its content */ 34 | @JsonProperty ( value = "body", required = true ) 35 | private byte[] body = null; 36 | 37 | /** time the message entered the system */ 38 | @JsonProperty ( value = "timestamp", required = true ) 39 | private long timestamp = 0; 40 | 41 | /** 42 | * Default constructor 43 | */ 44 | public StreamingDataMessage() { 45 | } 46 | 47 | /** 48 | * Initializes the message using the provided input 49 | * @param body 50 | * @param timestamp 51 | */ 52 | public StreamingDataMessage(final byte[] body, final long timestamp) { 53 | this.body = body; 54 | this.timestamp = timestamp; 55 | } 56 | 57 | public byte[] getBody() { 58 | return body; 59 | } 60 | 61 | public void setBody(byte[] body) { 62 | this.body = body; 63 | } 64 | 65 | public long getTimestamp() { 66 | return timestamp; 67 | } 68 | 69 | public void setTimestamp(long timestamp) { 70 | this.timestamp = timestamp; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/metrics/MetricReporterType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.metrics; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * Supporter reporter types 22 | * @author mnxfst 23 | * @since May 27, 2015 24 | */ 25 | public enum MetricReporterType implements Serializable { 26 | CONSOLE, KAFKA, CSV, GRAPHITE 27 | } 28 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/metrics/MicroPipelineMetricsReporterConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.metrics; 17 | 18 | import java.io.Serializable; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import com.fasterxml.jackson.annotation.JsonProperty; 23 | import com.fasterxml.jackson.annotation.JsonRootName; 24 | 25 | /** 26 | * Provides settings for activating metrics collection within micro pipelines 27 | * @author mnxfst 28 | * @since May 27, 2015 29 | */ 30 | @JsonRootName(value="microPipelineMetricsConfiguration") 31 | public class MicroPipelineMetricsReporterConfiguration implements Serializable { 32 | 33 | private static final long serialVersionUID = -3613911950147597674L; 34 | 35 | public static final String SETTING_CSV_OUTPUT_FILE = "outputFile"; 36 | 37 | public static final String SETTING_GRAPHITE_HOST = "host"; 38 | public static final String SETTING_GRAPHITE_PORT = "port"; 39 | 40 | public static final String SETTING_KAFKA_BROKER_LIST = "brokerList"; 41 | public static final String SETTING_KAFKA_ZOOKEEPER_CONNECT = "zookeeperConnect"; 42 | public static final String SETTING_KAFKA_CLIENT_ID = "clientId"; 43 | public static final String SETTING_KAFKA_TOPIC_ID = "topicId"; 44 | 45 | /** metrics reporter identifier, eg. kafkaReporter */ 46 | @JsonProperty(value="id", required=true) 47 | private String id = null; 48 | /** type, eg. CONSOLE, KAFKA, CSV, GRAPHITE */ 49 | @JsonProperty(value="type", required=true) 50 | private MetricReporterType type = MetricReporterType.CONSOLE; 51 | /** reporting period given in seconds */ 52 | @JsonProperty(value="period", required=true) 53 | private int period = 1; 54 | /** settings */ 55 | @JsonProperty(value="settings", required=true) 56 | private Map settings = new HashMap(); 57 | 58 | public void addSetting(final String key, final String value) { 59 | this.settings.put(key, value); 60 | } 61 | 62 | public String getId() { 63 | return id; 64 | } 65 | 66 | public void setId(String id) { 67 | this.id = id; 68 | } 69 | 70 | public MetricReporterType getType() { 71 | return type; 72 | } 73 | 74 | public void setType(MetricReporterType type) { 75 | this.type = type; 76 | } 77 | 78 | public Map getSettings() { 79 | return settings; 80 | } 81 | 82 | public void setSettings(Map settings) { 83 | this.settings = settings; 84 | } 85 | 86 | public int getPeriod() { 87 | return period; 88 | } 89 | 90 | public void setPeriod(int period) { 91 | this.period = period; 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/StreamingMessageQueueConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue; 17 | 18 | import java.io.Serializable; 19 | import java.util.Properties; 20 | 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.fasterxml.jackson.annotation.JsonRootName; 23 | 24 | /** 25 | * Configuration required for setting up a {@link StreamingMessageQueue} 26 | * @author mnxfst 27 | * @since Mar 6, 2015 28 | */ 29 | @JsonRootName(value="streamingMessageQueueConfiguration") 30 | public class StreamingMessageQueueConfiguration implements Serializable { 31 | 32 | private static final long serialVersionUID = -8500785433030074837L; 33 | 34 | /** component identifier */ 35 | @JsonProperty(value="id", required=true) 36 | private String id = null; 37 | /** add message insertion counter */ 38 | @JsonProperty(value="attachInsertionCounter", required=false) 39 | private boolean attachInsertionCounter = false; 40 | /** add message retrieval counter */ 41 | @JsonProperty(value="attachRetrievalCounter", required=false) 42 | private boolean attachRetrievalCounter = false; 43 | /** queue settings */ 44 | @JsonProperty(value="queueSettings", required=true) 45 | private Properties properties = null; 46 | 47 | public StreamingMessageQueueConfiguration() { 48 | } 49 | 50 | public StreamingMessageQueueConfiguration(final String id) { 51 | this.id = id; 52 | } 53 | 54 | public String getId() { 55 | return id; 56 | } 57 | 58 | public void setId(String id) { 59 | this.id = id; 60 | } 61 | 62 | public Properties getProperties() { 63 | return properties; 64 | } 65 | 66 | public void setProperties(Properties properties) { 67 | this.properties = properties; 68 | } 69 | 70 | public boolean isAttachInsertionCounter() { 71 | return attachInsertionCounter; 72 | } 73 | 74 | public void setAttachInsertionCounter(boolean attachInsertionCounter) { 75 | this.attachInsertionCounter = attachInsertionCounter; 76 | } 77 | 78 | public boolean isAttachRetrievalCounter() { 79 | return attachRetrievalCounter; 80 | } 81 | 82 | public void setAttachRetrievalCounter(boolean attachRetrievalCounter) { 83 | this.attachRetrievalCounter = attachRetrievalCounter; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/uk/co/real_logic/queues/BlockingWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package uk.co.real_logic.queues; 17 | 18 | import java.util.Queue; 19 | import java.util.concurrent.TimeUnit; 20 | import java.util.concurrent.locks.Condition; 21 | import java.util.concurrent.locks.Lock; 22 | import java.util.concurrent.locks.ReentrantLock; 23 | 24 | /** 25 | * Implements a wait strategy in the style of {@link https://github.com/jbrisbin/disruptor/blob/master/src/main/java/com/lmax/disruptor/BlockingWaitStrategy.java}. 26 | * It tries to read messages from an assigned {@link Queue}. If there is no content the strategy blocks until 27 | * either a timeout is reached or sleeping is interrupted from the outside. Apply this strategy when throughput and latency may 28 | * be spoiled in favor of CPU consumption. 29 | * @author mnxfst 30 | * @since Apr 20, 2015 31 | */ 32 | public class BlockingWaitStrategy implements MessageWaitStrategy { 33 | 34 | private final Lock lock = new ReentrantLock(); 35 | private final Condition condition = lock.newCondition(); 36 | 37 | /** 38 | * @see uk.co.real_logic.queues.MessageWaitStrategy#waitFor(java.util.Queue) 39 | */ 40 | public byte[] waitFor(Queue queue) throws InterruptedException { 41 | 42 | byte[] message = null; 43 | 44 | if((message = queue.poll()) == null) { 45 | 46 | // acquire lock 47 | lock.lock(); 48 | try { 49 | // try to fetch the next element from the queue. 50 | // if there is no entry available, wait for external notification (forceLockRelease required) 51 | while((message = queue.poll()) == null) { 52 | condition.await(); 53 | } 54 | 55 | } finally { 56 | // release lock 57 | lock.unlock(); 58 | } 59 | } 60 | 61 | return message; 62 | } 63 | 64 | /** 65 | * @see com.ottogroup.bi.spqr.websocket.strategy.MessageWaitStrategy#waitFor(Queue, long, TimeUnit)} 66 | */ 67 | public byte[] waitFor(Queue queue, long timeout, TimeUnit timeoutUnit) throws InterruptedException { 68 | byte[] message = null; 69 | 70 | if((message = queue.poll()) == null) { 71 | 72 | // acquire lock 73 | lock.lock(); 74 | try { 75 | // try to fetch the next element from the queue. 76 | // if there is no entry available, wait for time out and try again 77 | while((message = queue.poll()) == null) { 78 | if(!condition.await(timeout, timeoutUnit)) { 79 | break; 80 | } 81 | } 82 | 83 | } finally { 84 | // release lock 85 | lock.unlock(); 86 | } 87 | } 88 | 89 | return message; 90 | } 91 | 92 | /** 93 | * @see uk.co.real_logic.queues.MessageWaitStrategy#forceLockRelease() 94 | */ 95 | public void forceLockRelease() { 96 | 97 | // acquire lock 98 | lock.lock(); 99 | try { 100 | // free all 101 | condition.signalAll(); 102 | } finally { 103 | // release lock 104 | lock.unlock(); 105 | } 106 | 107 | 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/uk/co/real_logic/queues/MessageWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package uk.co.real_logic.queues; 17 | 18 | import java.util.Queue; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | /** 22 | * Interface to be implemented by a wait strategies applied where waiting for message is required 23 | * @author mnxfst 24 | * @since Apr 20, 2015 25 | */ 26 | public interface MessageWaitStrategy { 27 | 28 | /** 29 | * Wait for next element from referenced {@link Queue}. 30 | * @param queue 31 | * @return 32 | * @throws InterruptedException 33 | */ 34 | public byte[] waitFor(final Queue queue) throws InterruptedException; 35 | 36 | /** 37 | * Wait for next element from referenced {@link Queue}. If the timeout is 38 | * reached the result may contain null. 39 | * @param queue 40 | * @param timeout 41 | * @param timeoutUnit 42 | * @return 43 | * @throws InterruptedException 44 | */ 45 | public byte[] waitFor(final Queue queue, final long timeout, final TimeUnit timeoutUnit) throws InterruptedException; 46 | 47 | /** 48 | * Forces release of existing locks 49 | */ 50 | public void forceLockRelease(); 51 | } 52 | -------------------------------------------------------------------------------- /spqr-base/src/main/java/uk/co/real_logic/queues/PaddedAtomicLong.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Real Logic Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package uk.co.real_logic.queues; 17 | 18 | import java.util.concurrent.atomic.AtomicLong; 19 | 20 | public class PaddedAtomicLong extends AtomicLong 21 | { 22 | private static final long serialVersionUID = 3103909637944143851L; 23 | 24 | public PaddedAtomicLong() 25 | { 26 | } 27 | 28 | public PaddedAtomicLong(final long initialValue) 29 | { 30 | super(initialValue); 31 | } 32 | 33 | public volatile long p1, p2, p3, p4, p5, p6 = 7; 34 | } 35 | -------------------------------------------------------------------------------- /spqr-metrics/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.classpath 3 | -------------------------------------------------------------------------------- /spqr-metrics/src/main/java/com/ottogroup/bi/spqr/metrics/ResetOnReadCounter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.metrics; 17 | 18 | import com.codahale.metrics.Counter; 19 | 20 | /** 21 | * Extends the default {@link Counter} implementation such that the 22 | * value is reset to 0 when calling {@link #getCount()} 23 | * @author mnxfst 24 | * @since May 21, 2015 25 | */ 26 | public class ResetOnReadCounter extends Counter { 27 | 28 | /** 29 | * @see com.codahale.metrics.Counter#getCount() 30 | */ 31 | public long getCount() { 32 | long val = super.getCount(); 33 | super.dec(val); 34 | return val; 35 | } 36 | 37 | } 38 | 39 | 40 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.settings 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/README.MD: -------------------------------------------------------------------------------- 1 | spqr-micro-pipeline 2 | =================== 3 | 4 | Contains everything a pipeline needs -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/MessageCountResponseWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import java.util.Properties; 19 | 20 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 21 | 22 | /** 23 | * Counts the number of {@link StreamingDataMessage} instances received and signals {@link DelayedResponseCollector} to collect results 24 | * after having seen a configured number of items 25 | * @author mnxfst 26 | * @since Mar 12, 2015 27 | * 28 | */ 29 | public class MessageCountResponseWaitStrategy implements DelayedResponseOperatorWaitStrategy { 30 | 31 | public static final String WAIT_STRATEGY_NAME = "messageCount"; 32 | public static final int DEFAULT_MAX_MESSAGE_COUNT = 50; 33 | public static final String CFG_MAX_MESSAGE_COUNT_KEY = "maxMessages"; 34 | 35 | private DelayedResponseCollector delayedResponseCollector = null; 36 | /** holds the max number of message after which the strategy signals the collector to collect results. default is set to 50 */ 37 | private int maxMessageCount = DEFAULT_MAX_MESSAGE_COUNT; 38 | /** counts the number of messages seen after last value collection or since start - reset to 0 after each collection run */ 39 | private int messageCount = 0; 40 | 41 | /** 42 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#initialize(java.util.Properties) 43 | */ 44 | public void initialize(Properties properties) { 45 | try { 46 | this.maxMessageCount = Integer.parseInt(properties.getProperty(DelayedResponseOperator.CFG_WAIT_STRATEGY_SETTINGS_PREFIX + CFG_MAX_MESSAGE_COUNT_KEY)); 47 | if(this.maxMessageCount < 1) 48 | this.maxMessageCount = DEFAULT_MAX_MESSAGE_COUNT; 49 | } catch(Exception e) { 50 | this.maxMessageCount = DEFAULT_MAX_MESSAGE_COUNT; 51 | } 52 | this.messageCount = 0; 53 | } 54 | 55 | /** 56 | * @see java.lang.Runnable#run() 57 | */ 58 | public void run() { 59 | // no code as it is triggered directly on each message by runtime environment ... no need for async behavior here 60 | } 61 | 62 | /** 63 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#release() 64 | */ 65 | public void release() { 66 | this.delayedResponseCollector.retrieveMessages(); 67 | this.messageCount = 0; 68 | } 69 | 70 | /** 71 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#onMessage(com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage) 72 | */ 73 | public void onMessage(StreamingDataMessage message) { 74 | messageCount++; 75 | if(this.messageCount >= this.maxMessageCount) { 76 | release(); // ask collector to collect messages and reset counter to 0 77 | } 78 | } 79 | 80 | /** 81 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#shutdown() 82 | */ 83 | public void shutdown() { 84 | } 85 | 86 | /** 87 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#setDelayedResponseCollector(com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseCollector) 88 | */ 89 | public void setDelayedResponseCollector(DelayedResponseCollector delayedResponseCollector) { 90 | this.delayedResponseCollector = delayedResponseCollector; 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/component/operator/OperatorTriggeredWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.component.operator; 17 | 18 | import java.util.Properties; 19 | 20 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 21 | 22 | /** 23 | * Triggered directly by {@link DelayedResponseOperator} 24 | * @author mnxfst 25 | * @since Apr 23, 2015 26 | */ 27 | public class OperatorTriggeredWaitStrategy implements DelayedResponseOperatorWaitStrategy { 28 | 29 | public static final String WAIT_STRATEGY_NAME = "messageCount"; 30 | 31 | private DelayedResponseCollector delayedResponseCollector = null; 32 | 33 | /** 34 | * @see java.lang.Runnable#run() 35 | */ 36 | public void run() { 37 | // no code as it is triggered directly from operator ... no need for async behavior here 38 | } 39 | 40 | /** 41 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#setDelayedResponseCollector(com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseCollector) 42 | */ 43 | public void setDelayedResponseCollector(DelayedResponseCollector delayedResponseCollector) { 44 | this.delayedResponseCollector = delayedResponseCollector; 45 | } 46 | 47 | /** 48 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#release() 49 | */ 50 | public void release() { 51 | this.delayedResponseCollector.retrieveMessages(); 52 | } 53 | 54 | /** 55 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#onMessage(com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage) 56 | */ 57 | public void onMessage(StreamingDataMessage message) { 58 | // do nothing ... as the operator controls the result retrieval 59 | } 60 | 61 | /** 62 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#shutdown() 63 | */ 64 | public void shutdown() { 65 | // do nothing ... as there is nothing to shut down 66 | } 67 | 68 | /** 69 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy#initialize(java.util.Properties) 70 | */ 71 | public void initialize(Properties properties) { 72 | // do nothing ... as this serves only as a bridge between the operator and the runtime environment 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/exception/UnknownWaitStrategyException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.exception; 17 | 18 | import com.ottogroup.bi.spqr.pipeline.component.operator.DelayedResponseOperatorWaitStrategy; 19 | 20 | /** 21 | * Thrown in case a requested {@link DelayedResponseOperatorWaitStrategy} does not exist 22 | * @author mnxfst 23 | * @since Mar 12, 2015 24 | */ 25 | public class UnknownWaitStrategyException extends Exception { 26 | 27 | private static final long serialVersionUID = 53649492583889138L; 28 | 29 | public UnknownWaitStrategyException() { 30 | } 31 | 32 | public UnknownWaitStrategyException(String message) { 33 | super(message); 34 | } 35 | 36 | public UnknownWaitStrategyException(Throwable cause) { 37 | super(cause); 38 | } 39 | 40 | public UnknownWaitStrategyException(String message, Throwable cause) { 41 | super(message, cause); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/StreamingMessageQueue.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue; 17 | 18 | import java.util.Properties; 19 | 20 | import com.codahale.metrics.Counter; 21 | import com.ottogroup.bi.spqr.exception.RequiredInputMissingException; 22 | import com.ottogroup.bi.spqr.pipeline.MicroPipeline; 23 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent; 24 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 25 | 26 | /** 27 | * Provides the interface for a queue used internally inside the {@link MicroPipeline} to interconnect 28 | * {@link MicroPipelineComponent} instances. It transports {@link StreamingDataMessage} entities only 29 | * and must be accessed by only one producer but supports multiple consumers. 30 | * @author mnxfst 31 | * @since Mar 5, 2015 32 | * TODO provide statistics interface 33 | */ 34 | public interface StreamingMessageQueue { 35 | 36 | ///////////////////////////////////////////////////////////////////////// 37 | // available settings for queue instances 38 | public static final String CFG_QUEUE_TYPE = "type"; 39 | ///////////////////////////////////////////////////////////////////////// 40 | 41 | /** 42 | * Sets the component identifier 43 | * @param id 44 | */ 45 | public void setId(final String id); 46 | 47 | /** 48 | * Returns the component identifier which is unique 49 | * within the boundaries of a {@link MicroPipeline} 50 | * @return 51 | */ 52 | public String getId(); 53 | 54 | /** 55 | * Initializes the component before it gets executed 56 | * @param properties 57 | * @throws RequiredInputMissingException 58 | */ 59 | public void initialize(final Properties properties) throws RequiredInputMissingException; 60 | 61 | /** 62 | * Tells the component to shut itself down 63 | */ 64 | public boolean shutdown(); 65 | 66 | /** 67 | * Inserts the given {@link StreamingDataMessage} into the underlying queue 68 | * @param message 69 | * @return 70 | */ 71 | public boolean insert(final StreamingDataMessage message); 72 | 73 | /** 74 | * Retrieves the next {@link StreamingDataMessage} from the underlying queue 75 | * @return 76 | */ 77 | public StreamingDataMessage next(); 78 | 79 | /** 80 | * Returns a {@link StreamingMessageQueueProducer} instance with access to underlying queue 81 | * @return 82 | */ 83 | public StreamingMessageQueueProducer getProducer(); 84 | 85 | /** 86 | * Returns a {@link StreamingMessageQueueConsumer} instance with access to underlying queue 87 | * @return 88 | */ 89 | public StreamingMessageQueueConsumer getConsumer(); 90 | 91 | /** 92 | * Attaches an optional {@link Counter} instance for counting inserted messages 93 | * @param counter 94 | */ 95 | public void setMessageInsertionCounter(final Counter counter); 96 | 97 | /** 98 | * Attaches an optional {@link Counter} instance for counting retrieved messages 99 | * @param counter 100 | */ 101 | public void setMessageRetrievalCounter(final Counter counter); 102 | } 103 | 104 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/StreamingMessageQueueConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue; 17 | 18 | import com.codahale.metrics.Counter; 19 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 20 | import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy; 21 | 22 | /** 23 | * Provides read-only access to a {@link StreamingMessageQueue} 24 | * @author mnxfst 25 | * @since Mar 5, 2015 26 | */ 27 | public interface StreamingMessageQueueConsumer { 28 | 29 | /** 30 | * Returns the identifier of the queue this consumer is attached to 31 | * @return 32 | */ 33 | public String getQueueId(); 34 | 35 | /** 36 | * Retrieves the next {@link StreamingDataMessage} from the underlying queue 37 | * @return 38 | */ 39 | public StreamingDataMessage next(); 40 | 41 | /** 42 | * Returns the optional {@link StreamingMessageQueueWaitStrategy} assigned to the queue 43 | * @return 44 | */ 45 | public StreamingMessageQueueWaitStrategy getWaitStrategy(); 46 | 47 | /** 48 | * Attaches an optional {@link Counter} instance for counting retrieved messages 49 | * @param counter 50 | */ 51 | public void setMessageRetrievalCounter(final Counter counter); 52 | } 53 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/StreamingMessageQueueProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue; 17 | 18 | import com.codahale.metrics.Counter; 19 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 20 | import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy; 21 | 22 | /** 23 | * Provides write-only access to a {@link StreamingMessageQueue} 24 | * @author mnxfst 25 | * @since Mar 5, 2015 26 | */ 27 | public interface StreamingMessageQueueProducer { 28 | 29 | /** 30 | * Returns the identifier of the queue this producer is attached to 31 | * @return 32 | */ 33 | public String getQueueId(); 34 | 35 | /** 36 | * Inserts the given {@link StreamingDataMessage} into the underlying queue 37 | * @param message 38 | * @return 39 | */ 40 | public boolean insert(final StreamingDataMessage message); 41 | 42 | /** 43 | * Returns the optional {@link StreamingMessageQueueWaitStrategy} assigned to the underlying queue 44 | * @return 45 | */ 46 | public StreamingMessageQueueWaitStrategy getWaitStrategy(); 47 | 48 | /** 49 | * Attaches an optional {@link Counter} instance for counting inserted messages 50 | * @param counter 51 | */ 52 | public void setMessageInsertionCounter(final Counter counter); 53 | 54 | } 55 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/chronicle/DefaultStreamingMessageQueueConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.chronicle; 17 | 18 | import com.codahale.metrics.Counter; 19 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 20 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer; 21 | import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy; 22 | 23 | import net.openhft.chronicle.ExcerptTailer; 24 | 25 | /** 26 | * Default {@link StreamingMessageQueueConsumer} implementation accessing {@link DefaultStreamingMessageQueue} 27 | * @author mnxfst 28 | * @since Mar 5, 2015 29 | */ 30 | public class DefaultStreamingMessageQueueConsumer implements StreamingMessageQueueConsumer { 31 | 32 | private final String queueId; 33 | private final ExcerptTailer queueReader; 34 | private final StreamingMessageQueueWaitStrategy waitStrategy; 35 | private Counter messageRetrievalCounter = null; 36 | 37 | /** 38 | * Initializes the consumer using the provided input 39 | * @param queueId 40 | * @param queueReader 41 | * @param waitStrategy 42 | */ 43 | public DefaultStreamingMessageQueueConsumer(final String queueId, final ExcerptTailer queueReader, final StreamingMessageQueueWaitStrategy waitStrategy) { 44 | this.queueId = queueId; 45 | this.queueReader = queueReader; 46 | this.waitStrategy = waitStrategy; 47 | } 48 | 49 | /** 50 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#next() 51 | */ 52 | public StreamingDataMessage next() { 53 | 54 | // check if a new message is available and read it from chronicle if possible 55 | if(queueReader.nextIndex()) { 56 | long timestamp = queueReader.readLong(); 57 | int bytes = queueReader.readInt(); 58 | byte[] body = new byte[bytes]; 59 | queueReader.read(body); 60 | queueReader.finish(); 61 | 62 | if(this.messageRetrievalCounter != null) 63 | this.messageRetrievalCounter.inc(); 64 | 65 | return new StreamingDataMessage(body, timestamp); 66 | } 67 | 68 | // otherwise return null; 69 | return null; 70 | } 71 | 72 | /** 73 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#getWaitStrategy() 74 | */ 75 | public StreamingMessageQueueWaitStrategy getWaitStrategy() { 76 | return this.waitStrategy; 77 | } 78 | 79 | /** 80 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#getQueueId() 81 | */ 82 | public String getQueueId() { 83 | return queueId; 84 | } 85 | 86 | /** 87 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#setMessageRetrievalCounter(com.codahale.metrics.Counter) 88 | */ 89 | public void setMessageRetrievalCounter(Counter counter) { 90 | this.messageRetrievalCounter = counter; 91 | } 92 | 93 | 94 | 95 | } 96 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/chronicle/DefaultStreamingMessageQueueProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.chronicle; 17 | 18 | import com.codahale.metrics.Counter; 19 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 20 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer; 21 | import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy; 22 | 23 | import net.openhft.chronicle.ExcerptAppender; 24 | 25 | /** 26 | * Default {@link StreamingMessageQueueProducer} implementation accessing {@link DefaultStreamingMessageQueue} 27 | * @author mnxfst 28 | * @since Mar 5, 2015 29 | * TODO async implementation required 30 | */ 31 | public class DefaultStreamingMessageQueueProducer implements 32 | StreamingMessageQueueProducer { 33 | 34 | private final String queueId; 35 | private final ExcerptAppender queueProducer; 36 | private final StreamingMessageQueueWaitStrategy waitStrategy; 37 | private Counter messageInsertionCounter = null; 38 | 39 | /** 40 | * Initializes the producer using the provided input 41 | * @param queueId 42 | * @param queueProducer 43 | * @param waitStrategy 44 | */ 45 | public DefaultStreamingMessageQueueProducer(final String queueId, final ExcerptAppender queueProducer, final StreamingMessageQueueWaitStrategy waitStrategy) { 46 | this.queueId = queueId; 47 | this.queueProducer = queueProducer; 48 | this.waitStrategy = waitStrategy; 49 | } 50 | 51 | /** 52 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer#insert(com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage) 53 | */ 54 | public boolean insert(StreamingDataMessage message) { 55 | 56 | // TODO add concurrency handler to support multiple writers properly 57 | if(message != null) { 58 | synchronized (queueProducer) { 59 | queueProducer.startExcerpt(); 60 | queueProducer.writeLong(message.getTimestamp()); 61 | queueProducer.writeInt(message.getBody().length); 62 | queueProducer.write(message.getBody()); 63 | queueProducer.finish(); 64 | 65 | if(this.messageInsertionCounter != null) 66 | this.messageInsertionCounter.inc(); 67 | 68 | return true; 69 | } 70 | } 71 | 72 | return false; 73 | } 74 | 75 | /** 76 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer#getWaitStrategy() 77 | */ 78 | public StreamingMessageQueueWaitStrategy getWaitStrategy() { 79 | return this.waitStrategy; 80 | } 81 | 82 | public String getQueueId() { 83 | return queueId; 84 | } 85 | 86 | /** 87 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer#setMessageInsertionCounter(com.codahale.metrics.Counter) 88 | */ 89 | public void setMessageInsertionCounter(Counter counter) { 90 | this.messageInsertionCounter = counter; 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/memory/InMemoryStreamingMessageQueueConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.memory; 17 | 18 | import java.util.concurrent.ConcurrentLinkedQueue; 19 | 20 | import com.codahale.metrics.Counter; 21 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 22 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer; 23 | import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy; 24 | 25 | /** 26 | * Consumes messages from memory based {@link InMemoryStreamingMessageQueue queue} 27 | * @author mnxfst 28 | * @since Jul 3, 2015 29 | */ 30 | public class InMemoryStreamingMessageQueueConsumer implements StreamingMessageQueueConsumer { 31 | 32 | /** identifier of queue this consumer is attached to */ 33 | private final String queueId; 34 | /** queue the consumer reads from */ 35 | private final ConcurrentLinkedQueue queue; 36 | /** assigned wait strategy */ 37 | private final StreamingMessageQueueWaitStrategy waitStrategy; 38 | /** counter instance used for metric collection */ 39 | private Counter messageRetrievalCounter = null; 40 | 41 | /** 42 | * Initializes the consumer using the provided input 43 | * @param queueId 44 | * @param queue 45 | * @param waitStrategy 46 | */ 47 | public InMemoryStreamingMessageQueueConsumer(final String queueId, final ConcurrentLinkedQueue queue, final StreamingMessageQueueWaitStrategy waitStrategy) { 48 | this.queueId = queueId; 49 | this.queue = queue; 50 | this.waitStrategy = waitStrategy; 51 | } 52 | 53 | /** 54 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#getQueueId() 55 | */ 56 | public String getQueueId() { 57 | return this.queueId; 58 | } 59 | 60 | /** 61 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#next() 62 | */ 63 | public StreamingDataMessage next() { 64 | final StreamingDataMessage nextMessage = this.queue.poll(); 65 | if(this.messageRetrievalCounter != null && nextMessage != null) 66 | this.messageRetrievalCounter.inc(); 67 | return nextMessage; 68 | } 69 | 70 | /** 71 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#getWaitStrategy() 72 | */ 73 | public StreamingMessageQueueWaitStrategy getWaitStrategy() { 74 | return this.waitStrategy; 75 | } 76 | 77 | /** 78 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer#setMessageRetrievalCounter(com.codahale.metrics.Counter) 79 | */ 80 | public void setMessageRetrievalCounter(Counter counter) { 81 | this.messageRetrievalCounter = counter; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/memory/InMemoryStreamingMessageQueueProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.memory; 17 | 18 | import java.util.concurrent.ConcurrentLinkedQueue; 19 | 20 | import com.codahale.metrics.Counter; 21 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 22 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer; 23 | import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy; 24 | 25 | /** 26 | * Produces messages to attached {@link InMemoryStreamingMessageQueue} 27 | * @author mnxfst 28 | * @since Jul 3, 2015 29 | */ 30 | public class InMemoryStreamingMessageQueueProducer implements StreamingMessageQueueProducer { 31 | 32 | /** identifier of queue this consumer is attached to */ 33 | private final String queueId; 34 | /** queue the consumer reads from */ 35 | private final ConcurrentLinkedQueue queue; 36 | /** assigned wait strategy for fetching messages */ 37 | private final StreamingMessageQueueWaitStrategy waitStrategy; 38 | /** counts the number of message insertions */ 39 | private Counter messageInsertionCounter = null; 40 | 41 | /** 42 | * Initializes the producer using the provided input 43 | * @param queueId 44 | * @param queue 45 | * @param waitStrategy 46 | */ 47 | public InMemoryStreamingMessageQueueProducer(final String queueId, final ConcurrentLinkedQueue queue, final StreamingMessageQueueWaitStrategy waitStrategy) { 48 | this.queueId = queueId; 49 | this.queue = queue; 50 | this.waitStrategy = waitStrategy; 51 | } 52 | 53 | 54 | /** 55 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer#insert(com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage) 56 | */ 57 | public boolean insert(StreamingDataMessage message) { 58 | if(message != null) { 59 | this.queue.offer(message); 60 | if(this.messageInsertionCounter != null) 61 | this.messageInsertionCounter.inc(); 62 | } 63 | return false; 64 | } 65 | 66 | /** 67 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer#getWaitStrategy() 68 | */ 69 | public StreamingMessageQueueWaitStrategy getWaitStrategy() { 70 | return this.waitStrategy; 71 | } 72 | 73 | /** 74 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer#setMessageInsertionCounter(com.codahale.metrics.Counter) 75 | */ 76 | public void setMessageInsertionCounter(Counter counter) { 77 | this.messageInsertionCounter = counter; 78 | } 79 | 80 | /** 81 | * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer#getQueueId() 82 | */ 83 | public String getQueueId() { 84 | return this.queueId; 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/strategy/StreamingMessageQueueBlockingWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.strategy; 17 | 18 | import java.util.Queue; 19 | import java.util.concurrent.TimeUnit; 20 | import java.util.concurrent.locks.Condition; 21 | import java.util.concurrent.locks.Lock; 22 | import java.util.concurrent.locks.ReentrantLock; 23 | 24 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 25 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer; 26 | 27 | /** 28 | * Implements a wait strategy in the style of {@link https://github.com/jbrisbin/disruptor/blob/master/src/main/java/com/lmax/disruptor/BlockingWaitStrategy.java}. 29 | * It tries to read messages from an assigned {@link Queue}. If there is no content the strategy blocks until 30 | * either a timeout is reached or sleeping is interrupted from the outside. Apply this strategy when throughput and latency may 31 | * be spoiled in favor of CPU consumption. 32 | * @author mnxfst 33 | * @since Apr 20, 2015 34 | */ 35 | public class StreamingMessageQueueBlockingWaitStrategy implements StreamingMessageQueueWaitStrategy { 36 | 37 | public static final String STRATEGY_NAME = "blockingWait"; 38 | 39 | private final Lock lock = new ReentrantLock(); 40 | private final Condition condition = lock.newCondition(); 41 | 42 | /** 43 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#waitFor(StreamingMessageQueueConsumer))} 44 | */ 45 | public StreamingDataMessage waitFor(StreamingMessageQueueConsumer queue) throws InterruptedException { 46 | 47 | StreamingDataMessage message = null; 48 | if((message = queue.next()) == null) { 49 | 50 | // acquire lock 51 | lock.lock(); 52 | try { 53 | // try to fetch the next element from the queue. 54 | // if there is no entry available, wait for external notification (forceLockRelease required) 55 | while((message = queue.next()) == null) { 56 | condition.await(); 57 | } 58 | 59 | } finally { 60 | // release lock 61 | lock.unlock(); 62 | } 63 | } 64 | 65 | return message; 66 | 67 | } 68 | 69 | /** 70 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#waitFor(StreamingMessageQueueConsumer, long, TimeUnit)} 71 | */ 72 | public StreamingDataMessage waitFor(StreamingMessageQueueConsumer queue, long timeout, TimeUnit timeoutUnit) throws InterruptedException { 73 | 74 | StreamingDataMessage message = null; 75 | if((message = queue.next()) == null) { 76 | 77 | // acquire lock 78 | lock.lock(); 79 | try { 80 | // try to fetch the next element from the queue. 81 | // if there is no entry available, wait for external notification (forceLockRelease required) 82 | while((message = queue.next()) == null) { 83 | condition.await(timeout, timeoutUnit); 84 | } 85 | 86 | } finally { 87 | // release lock 88 | lock.unlock(); 89 | } 90 | } 91 | 92 | return message; 93 | } 94 | 95 | /** 96 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#forceLockRelease() 97 | */ 98 | public void forceLockRelease() { 99 | 100 | // acquire lock 101 | lock.lock(); 102 | try { 103 | // free all 104 | condition.signalAll(); 105 | } finally { 106 | // release lock 107 | lock.unlock(); 108 | } 109 | 110 | 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/strategy/StreamingMessageQueueDirectPassStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.strategy; 17 | 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 21 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer; 22 | 23 | /** 24 | * Default strategy if no other is provided. All timeouts are ignored and calls are directly forwarded 25 | * to the underlying {@link StreamingMessageQueueConsumer}. Even if the consumer returns null it 26 | * returns the value to the caller 27 | * @author mnxfst 28 | * @since Apr 21, 2015 29 | */ 30 | public class StreamingMessageQueueDirectPassStrategy implements StreamingMessageQueueWaitStrategy { 31 | 32 | public static final String STRATEGY_NAME = "directPass"; 33 | 34 | /** 35 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#waitFor(com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer) 36 | */ 37 | public StreamingDataMessage waitFor(StreamingMessageQueueConsumer queue) throws InterruptedException { 38 | return queue.next(); 39 | } 40 | 41 | /** 42 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#waitFor(com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer, long, java.util.concurrent.TimeUnit) 43 | */ 44 | public StreamingDataMessage waitFor(StreamingMessageQueueConsumer queue, long timeout, TimeUnit timeoutUnit) throws InterruptedException { 45 | return queue.next(); 46 | } 47 | 48 | /** 49 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#forceLockRelease() 50 | */ 51 | public void forceLockRelease() { 52 | // no-op 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/strategy/StreamingMessageQueueSleepingWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.strategy; 17 | 18 | import java.util.Queue; 19 | import java.util.concurrent.TimeUnit; 20 | import java.util.concurrent.locks.LockSupport; 21 | 22 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 23 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer; 24 | 25 | /** 26 | * Implements a wait strategy in the style of {@link https://github.com/LMAX-Exchange/disruptor/blob/master/src/main/java/com/lmax/disruptor/SleepingWaitStrategy.java}. 27 | * It tries to read messages from an assigned {@link Queue}. If there is no content the strategy blocks for a fixed time. 28 | * @author mnxfst 29 | * @since Jan 11, 2016 30 | */ 31 | public class StreamingMessageQueueSleepingWaitStrategy implements StreamingMessageQueueWaitStrategy { 32 | 33 | public static final String STRATEGY_NAME = "sleepingWait"; 34 | 35 | private final int retries = 200; 36 | 37 | /** 38 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#waitFor(com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer) 39 | */ 40 | public StreamingDataMessage waitFor(StreamingMessageQueueConsumer queue) throws InterruptedException { 41 | 42 | StreamingDataMessage message = null; 43 | int counter = retries; 44 | while((message = queue.next()) == null) { 45 | if(counter > 100) { 46 | --counter; 47 | } else if(counter > 0) { 48 | --counter; 49 | Thread.yield(); 50 | } else { 51 | LockSupport.parkNanos(1l); 52 | } 53 | } 54 | return message; 55 | } 56 | 57 | /** 58 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#waitFor(com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer, long, java.util.concurrent.TimeUnit) 59 | */ 60 | public StreamingDataMessage waitFor(StreamingMessageQueueConsumer queue, long timeout, TimeUnit timeoutUnit) throws InterruptedException { 61 | 62 | StreamingDataMessage message = null; 63 | int counter = retries; 64 | while((message = queue.next()) == null) { 65 | if(counter > 100) { 66 | --counter; 67 | } else if(counter > 0) { 68 | --counter; 69 | Thread.yield(); 70 | } else { 71 | LockSupport.parkNanos(1l); 72 | } 73 | } 74 | return message; 75 | } 76 | 77 | /** 78 | * @see com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy#forceLockRelease() 79 | */ 80 | public void forceLockRelease() { 81 | // nothing to do 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/main/java/com/ottogroup/bi/spqr/pipeline/queue/strategy/StreamingMessageQueueWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.pipeline.queue.strategy; 17 | 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 21 | import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer; 22 | 23 | /** 24 | * Interface to be implemented by all classes providing a wait strategy for 25 | * reading content from {@link StreamingMessageQueueConsumer} instances 26 | * @author mnxfst 27 | * @since Apr 20, 2015 28 | */ 29 | public interface StreamingMessageQueueWaitStrategy { 30 | 31 | /** 32 | * Wait for next element from referenced {@link StreamingMessageQueueConsumer}. 33 | * @param queue 34 | * @return 35 | * @throws InterruptedException 36 | */ 37 | public StreamingDataMessage waitFor(final StreamingMessageQueueConsumer queue) throws InterruptedException; 38 | 39 | /** 40 | * Wait for next element from referenced {@link StreamingMessageQueueConsumer}. If the timeout is 41 | * reached the result may contain null. 42 | * @param queue 43 | * @param timeout 44 | * @param timeoutUnit 45 | * @return 46 | * @throws InterruptedException 47 | */ 48 | public StreamingDataMessage waitFor(final StreamingMessageQueueConsumer queue, final long timeout, final TimeUnit timeoutUnit) throws InterruptedException; 49 | 50 | /** 51 | * Forces release of existing locks 52 | */ 53 | public void forceLockRelease(); 54 | 55 | } 56 | -------------------------------------------------------------------------------- /spqr-micro-pipeline/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | -------------------------------------------------------------------------------- /spqr-node/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-node/src/main/config/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, file 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | 10 | # Direct log messages to a log file 11 | log4j.appender.file=org.apache.log4j.RollingFileAppender 12 | log4j.appender.file.File=/opt/streaming/spqr/spqr-node/log/spqr-node-log4j.log 13 | log4j.appender.file.MaxFileSize=10MB 14 | log4j.appender.file.MaxBackupIndex=10 15 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 16 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /spqr-node/src/main/config/spqr-node.cfg: -------------------------------------------------------------------------------- 1 | spqrNode: 2 | log4jConfiguration: "/opt/streaming/spqr/spqr-node/etc/log4j.properties" 3 | numOfThreads: 0 4 | temporaryQueueFolder: "/opt/streaming/spqr/spqr-node/queues" 5 | componentRepositoryFolder: "/opt/streaming/spqr/spqr-node/repo/" 6 | 7 | host: "localhost" 8 | protocol: "http" 9 | servicePort: 7070 10 | adminPort: 7071 11 | 12 | #spqrMetrics: 13 | # attachMemoryUsageMetricCollector: true 14 | # attachFileDescriptorMetricCollector: true 15 | # attachClassLoadingMetricCollector: true 16 | # attachGCMetricCollector: true 17 | # attachThreadStateMetricCollector: true 18 | # metricsReporter: 19 | # - id: "kafka" 20 | # type: "KAFKA" 21 | # period: 5 22 | # settings: 23 | # zookeeperConnect: "localhost:2181" 24 | # brokerList: "localhost:9092" 25 | # topicId: "nodemetrics" 26 | # clientId: "sampleClient" 27 | 28 | resourceManager: 29 | mode: LOCAL 30 | protocol: "http" 31 | host: "localhost" 32 | port: 9090 33 | 34 | httpClient: 35 | timeout: 500ms 36 | connectionTimeout: 500ms 37 | timeToLive: 1h 38 | cookiesEnabled: false 39 | maxConnections: 1024 40 | maxConnectionsPerRoute: 1024 41 | keepAlive: 0ms 42 | retries: 0 43 | userAgent: spqr-node/0.2 44 | 45 | server: 46 | applicationConnectors: 47 | - type: http 48 | port: 7070 49 | adminConnectors: 50 | - type: http 51 | port: 7071 52 | 53 | logging: 54 | 55 | level: INFO 56 | 57 | loggers: 58 | 59 | # Sets the level for 'com.example.app' to DEBUG. 60 | com.ottogroup.bi.spqr: DEBUG 61 | 62 | appenders: 63 | - type: file 64 | threshold: DEBUG 65 | logFormat: "%-6level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %X{code} %msg %n" 66 | currentLogFilename: /opt/streaming/spqr/log/spqr-node/spqr-node.log 67 | archivedLogFilenamePattern: /opt/streaming/spqr/spqr-node/log/spqr-node-%d{yyyy-MM-dd}.log 68 | archivedFileCount: 7 69 | timeZone: UTC 70 | 71 | -------------------------------------------------------------------------------- /spqr-node/src/main/java/com/ottogroup/bi/spqr/node/server/SPQRNodeShutdownHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.node.server; 17 | 18 | import org.apache.log4j.Logger; 19 | 20 | import com.ottogroup.bi.spqr.node.resman.SPQRResourceManagerClient; 21 | import com.ottogroup.bi.spqr.pipeline.MicroPipeline; 22 | import com.ottogroup.bi.spqr.pipeline.MicroPipelineManager; 23 | 24 | /** 25 | * Invoked on application shutdown the handler ensures that all resources are properly released, 26 | * all {@link MicroPipeline} instances shut down and the node de-registered from the resource manager. 27 | * @author mnxfst 28 | * @since Mar 17, 2015 29 | */ 30 | public class SPQRNodeShutdownHandler extends Thread { 31 | 32 | /** our faithful logging service ... ;-) */ 33 | private static final Logger logger = Logger.getLogger(SPQRNodeShutdownHandler.class); 34 | 35 | private final MicroPipelineManager microPipelineManager; 36 | private final SPQRResourceManagerClient resourceManagerClient; 37 | private final String nodeId; 38 | 39 | /** 40 | * Initializes the shutdown handler using the provided input 41 | * @param microPipelineManager 42 | * @param resourceManagerClient 43 | * @param nodeId 44 | */ 45 | public SPQRNodeShutdownHandler(final MicroPipelineManager microPipelineManager, final SPQRResourceManagerClient resourceManagerClient, final String nodeId) { 46 | this.microPipelineManager = microPipelineManager; 47 | this.resourceManagerClient = resourceManagerClient; 48 | this.nodeId = nodeId; 49 | } 50 | 51 | /** 52 | * @see java.lang.Runnable#run() 53 | */ 54 | public void run() { 55 | 56 | logger.info("Preparing to shut down node"); 57 | 58 | try { 59 | this.microPipelineManager.shutdown(); 60 | logger.info("All running pipelines shut down..."); 61 | } catch(Exception e) { 62 | logger.info("Error while shutting down running pipelines: " + e.getMessage()); 63 | } 64 | 65 | if(this.resourceManagerClient != null) { 66 | try { 67 | this.resourceManagerClient.deregisterNode(nodeId); 68 | logger.info("Node de-registered from resource manager"); 69 | } catch(Exception e) { 70 | logger.error("Error while de-registering node from resource manager: " + e.getMessage()); 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /spqr-node/src/main/java/com/ottogroup/bi/spqr/node/server/cfg/SPQRNodeServerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.node.server.cfg; 17 | 18 | import io.dropwizard.Configuration; 19 | import io.dropwizard.client.JerseyClientConfiguration; 20 | 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.ottogroup.bi.spqr.node.server.SPQRNodeServer; 23 | 24 | /** 25 | * Configuration required for setting up {@link SPQRNodeServer} 26 | * @author mnxfst 27 | * @since Mar 13, 2015 28 | */ 29 | public class SPQRNodeServerConfiguration extends Configuration { 30 | 31 | /** processing node specific settings */ 32 | @JsonProperty(value="spqrNode", required=true) 33 | private SPQRProcessingNodeConfiguration spqrNode = null; 34 | /** resource manager configuration: local or remote */ 35 | @JsonProperty(value="resourceManager", required=true) 36 | private SPQRResourceManagerConfiguration resourceManagerConfiguration = null; 37 | /** http client configuration */ 38 | @JsonProperty(value="httpClient", required=true) 39 | private JerseyClientConfiguration httpClient; 40 | 41 | 42 | public SPQRProcessingNodeConfiguration getSpqrNode() { 43 | return spqrNode; 44 | } 45 | public void setSpqrNode(SPQRProcessingNodeConfiguration spqrNode) { 46 | this.spqrNode = spqrNode; 47 | } 48 | public SPQRResourceManagerConfiguration getResourceManagerConfiguration() { 49 | return resourceManagerConfiguration; 50 | } 51 | public void setResourceManagerConfiguration( 52 | SPQRResourceManagerConfiguration resourceManagerConfiguration) { 53 | this.resourceManagerConfiguration = resourceManagerConfiguration; 54 | } 55 | public JerseyClientConfiguration getHttpClient() { 56 | return httpClient; 57 | } 58 | public void setHttpClient(JerseyClientConfiguration httpClient) { 59 | this.httpClient = httpClient; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /spqr-node/src/main/java/com/ottogroup/bi/spqr/node/server/cfg/SPQRResourceManagerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.node.server.cfg; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.fasterxml.jackson.annotation.JsonRootName; 22 | 23 | /** 24 | * Provides all settings required for getting into contact with the remote resource manager 25 | * @author mnxfst 26 | * @since Apr 16, 2015 27 | */ 28 | @JsonRootName(value="resourceManagerConfiguration") 29 | public class SPQRResourceManagerConfiguration implements Serializable { 30 | 31 | private static final long serialVersionUID = -3346086687554212055L; 32 | 33 | public enum ResourceManagerMode implements Serializable { 34 | LOCAL, REMOTE 35 | } 36 | 37 | /** resource manager to use: local (standalone mode) or remote (cluster mode) */ 38 | @JsonProperty(value="mode", required=true) 39 | private ResourceManagerMode mode = ResourceManagerMode.LOCAL; 40 | /** protocol to use for remote communication, http or https */ 41 | @JsonProperty(value="protocol", required=false) 42 | private String protocol = "http"; 43 | /** host the resource manager lives on */ 44 | @JsonProperty(value="host", required=false) 45 | private String host = null; 46 | /** port the resource manager listens to */ 47 | @JsonProperty(value="port", required=false) 48 | private int port = 8080; 49 | 50 | public SPQRResourceManagerConfiguration() { 51 | } 52 | 53 | public ResourceManagerMode getMode() { 54 | return mode; 55 | } 56 | 57 | public void setMode(ResourceManagerMode mode) { 58 | this.mode = mode; 59 | } 60 | 61 | public String getProtocol() { 62 | return protocol; 63 | } 64 | 65 | public void setProtocol(String protocol) { 66 | this.protocol = protocol; 67 | } 68 | 69 | public String getHost() { 70 | return host; 71 | } 72 | 73 | public void setHost(String host) { 74 | this.host = host; 75 | } 76 | 77 | public int getPort() { 78 | return port; 79 | } 80 | 81 | public void setPort(int port) { 82 | this.port = port; 83 | } 84 | 85 | 86 | 87 | } 88 | -------------------------------------------------------------------------------- /spqr-node/src/main/scripts/spqr-node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BASEDIR=$(dirname $0) 4 | PIDFILE=$BASEDIR/pidfile 5 | 6 | if [ -z "$1" ]; then 7 | # write error message 8 | echo "usage: spqr-node.sh {start|stop} " 9 | exit 1 10 | fi 11 | 12 | if [ "$1" = "stop" ]; then 13 | 14 | # check for running service 15 | if [ ! -f "$PIDFILE" ]; then 16 | echo "spqr node server is not running!" 17 | exit 1 18 | fi 19 | 20 | kill -9 `cat $PIDFILE` 21 | rm $PIDFILE 22 | echo "spqr node server shut down!" 23 | exit 0 24 | fi 25 | 26 | if [ -z "$2" ]; then 27 | # write error message 28 | echo "usage: spqr-node.sh {start|stop} " 29 | exit 1 30 | fi 31 | 32 | if [ "$1" = "start" ]; then 33 | 34 | # check for running service 35 | if [ -f "$PIDFILE" ]; then 36 | echo "spqr node server already running! process id: `cat $PIDFILE`" 37 | exit 1 38 | fi 39 | 40 | CLASSPATH=.:$BASEDIR/../lib/* 41 | java -d64 -server -XX:MaxPermSize=512M -XX:MaxGCPauseMillis=500 -XX:+UseG1GC -Xms1G -cp $CLASSPATH com.ottogroup.bi.spqr.node.server.SPQRNodeServer server $2 & 42 | echo $! > $PIDFILE 43 | echo "spqr node server running. process id: `cat $PIDFILE`" 44 | exit 0 45 | fi 46 | 47 | if [ "$1" = "restart" ]; then 48 | # check for running service 49 | if [ -f "$PIDFILE" ]; then 50 | kill -9 `cat $PIDFILE` 51 | rm $PIDFILE 52 | echo "spqr node server shut down!" 53 | fi 54 | 55 | CLASSPATH=.:$BASEDIR/../lib/* 56 | java -d64 -server -XX:MaxPermSize=512M -XX:MaxGCPauseMillis=500 -XX:+UseG1GC -Xms1G -cp $CLASSPATH com.ottogroup.bi.spqr.node.server.SPQRNodeServer server $2 & 57 | echo $! > $PIDFILE 58 | echo "spqr node server running. process id: `cat $PIDFILE`" 59 | exit 0 60 | fi -------------------------------------------------------------------------------- /spqr-node/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /spqr-node/src/test/resources/twitter-filter-to-kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "twitter-filter-to-kafka", 3 | "queues" : [ 4 | { "id" : "twitter-content", "queueSettings" : null }, 5 | { "id" : "filtered-content", "queueSettings" : null } 6 | ], 7 | "components" : [ { 8 | "id" : "twitter-stream-reader", 9 | "type" : "SOURCE", 10 | "name" : "twitterSource", 11 | "version" : "0.0.1", 12 | "settings" : { 13 | "twitter.consumer.key" : "", 14 | "twitter.consumer.secret" : "", 15 | "twitter.token.key" : "", 16 | "twitter.token.secret" : "", 17 | "twitter.tweet.terms" : "fifa,uefa,soccer" 18 | }, 19 | "fromQueue" : "", 20 | "toQueue" : "twitter-content" 21 | }, 22 | { 23 | "id" : "twitter-content-filter", 24 | "type" : "DIRECT_RESPONSE_OPERATOR", 25 | "name" : "jsonContentFilter", 26 | "version" : "0.0.1", 27 | "settings" : { 28 | "field.1.path": "text", 29 | "field.1.expression": "(?i).*soccer.*", 30 | "field.1.type": "STRING" 31 | }, 32 | "fromQueue": "twitter-content", 33 | "toQueue": "filtered-content" 34 | }, 35 | { 36 | "id" : "kafka-topic-emitter", 37 | "type" : "EMITTER", 38 | "name" : "kafkaEmitter", 39 | "version" : "0.0.1", 40 | "settings" : { 41 | "clientId" : "twitterToKafka", 42 | "topic" : "twitter", 43 | "metadataBrokerList" : "localhost:9092", 44 | "zookeeperConnect" : "localhost:2181", 45 | "messageAcking" : "false", 46 | "charset" : "UTF-8" 47 | }, 48 | "fromQueue" : "filtered-content", 49 | "toQueue" : "" 50 | } ] 51 | } -------------------------------------------------------------------------------- /spqr-node/src/test/resources/twitter-to-kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "twitter-to-kafka", 3 | "queues" : [ { 4 | "id" : "twitter-content", 5 | "queueSettings" : null 6 | } ], 7 | "components" : [ { 8 | "id" : "twitter-stream-reader", 9 | "type" : "SOURCE", 10 | "name" : "twitterSource", 11 | "version" : "0.0.1", 12 | "settings" : { 13 | "twitter.consumer.key" : "", 14 | "twitter.consumer.secret" : "", 15 | "twitter.token.key" : "", 16 | "twitter.token.secret" : "", 17 | "twitter.tweet.terms" : "fifa,uefa,soccer" 18 | }, 19 | "fromQueue" : "", 20 | "toQueue" : "twitter-content" 21 | }, { 22 | "id" : "kafka-topic-emitter", 23 | "type" : "EMITTER", 24 | "name" : "kafkaEmitter", 25 | "version" : "0.0.1", 26 | "settings" : { 27 | "clientId" : "twitterToKafka", 28 | "topic" : "twitter", 29 | "metadataBrokerList" : "localhost:9092", 30 | "zookeeperConnect" : "localhost:2181", 31 | "messageAcking" : "false", 32 | "charset" : "UTF-8" 33 | }, 34 | "fromQueue" : "twitter-content", 35 | "toQueue" : "" 36 | } ] 37 | } -------------------------------------------------------------------------------- /spqr-node/src/test/resources/webtrends-filter-to-kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "webtrends-filter-to-kafka", 3 | "queues" : [ 4 | { "id" : "webtrends-content", "queueSettings" : null }, 5 | { "id" : "filtered-content", "queueSettings" : null } 6 | ], 7 | "components" : [ { 8 | "id" : "webtrends-stream-reader", 9 | "type" : "SOURCE", 10 | "name" : "webtrendsSource", 11 | "version" : "0.0.1", 12 | "settings" : { 13 | "webtrends.stream.version" : "2.1", 14 | "webtrends.auth.audience" : "auth.webtrends.com", 15 | "webtrends.stream.type" : "return_all", 16 | "webtrends.schema.version" : "2.1", 17 | "webtrends.auth.scope" : "sapi.webtrends.com", 18 | "webtrends.stream.url" : "ws://sapi.webtrends.com/streaming", 19 | "webtrends.auth.url" : "https://sauth.webtrends.com/v1/token", 20 | "webtrends.client.id" : "", 21 | "webtrends.client.secret" : "", 22 | "webtrends.stream.query" : "select data.*" 23 | }, 24 | "fromQueue" : "", 25 | "toQueue" : "webtrends-content" 26 | }, 27 | 28 | { 29 | "id" : "webtrends-content-filter", 30 | "type" : "DIRECT_RESPONSE_OPERATOR", 31 | "name" : "jsonContentFilter", 32 | "version" : "0.0.1", 33 | "settings" : { 34 | "field.1.path": "data.cs-host", 35 | "field.1.expression": "www.my-site.com", 36 | "field.1.type": "STRING" 37 | }, 38 | "fromQueue": "webtrends-content", 39 | "toQueue": "filtered-content" 40 | }, 41 | 42 | { 43 | "id" : "kafka-topic-emitter", 44 | "type" : "EMITTER", 45 | "name" : "kafkaEmitter", 46 | "version" : "0.0.1", 47 | "settings" : { 48 | "clientId" : "webtrendsToKafka", 49 | "topic" : "webtrends", 50 | "metadataBrokerList" : "localhost:9092", 51 | "zookeeperConnect" : "localhost:2181", 52 | "messageAcking" : "false", 53 | "charset" : "UTF-8" 54 | }, 55 | "fromQueue" : "filtered-content", 56 | "toQueue" : "" 57 | } ] 58 | } -------------------------------------------------------------------------------- /spqr-node/src/test/resources/webtrends-to-kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "webtrends-to-kafka", 3 | "queues" : [ { 4 | "id" : "webtrends-content", 5 | "queueSettings" : { 6 | "type" : "memory" 7 | } 8 | } ], 9 | "components" : [ { 10 | "id" : "webtrends-stream-reader", 11 | "type" : "SOURCE", 12 | "name" : "webtrendsSource", 13 | "version" : "0.0.1", 14 | "settings" : { 15 | "webtrends.stream.version" : "2.1", 16 | "webtrends.auth.audience" : "auth.webtrends.com", 17 | "webtrends.stream.type" : "return_all", 18 | "webtrends.schema.version" : "2.1", 19 | "webtrends.auth.scope" : "sapi.webtrends.com", 20 | "webtrends.stream.url" : "ws://sapi.webtrends.com/streaming", 21 | "webtrends.auth.url" : "https://sauth.webtrends.com/v1/token", 22 | "webtrends.client.id" : "", 23 | "webtrends.client.secret" : "", 24 | "webtrends.stream.query" : "select *" 25 | }, 26 | "fromQueue" : "", 27 | "toQueue" : "webtrends-content" 28 | }, { 29 | "id" : "kafka-topic-emitter", 30 | "type" : "EMITTER", 31 | "name" : "kafkaEmitter", 32 | "version" : "0.0.1", 33 | "settings" : { 34 | "clientId" : "webtrendsToKafka", 35 | "topic" : "webtrends", 36 | "metadataBrokerList" : "localhost:9092", 37 | "zookeeperConnect" : "localhost:2181", 38 | "messageAcking" : "false", 39 | "charset" : "UTF-8" 40 | }, 41 | "fromQueue" : "webtrends-content", 42 | "toQueue" : "" 43 | } ] 44 | } -------------------------------------------------------------------------------- /spqr-operators/spqr-esper/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-operators/spqr-esper/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 4.0.0 5 | 6 | 7 | com.ottogroup.bi.spqr 8 | spqr-parent 9 | 0.6.0-SNAPSHOT 10 | ../../pom.xml 11 | 12 | 13 | 14 | spqr-esper 15 | jar 16 | 17 | 18 | spqr-esper 19 | SPQR ESPER operator 20 | https://github.com/ottogroup/SPQR.git 21 | 2015 22 | 23 | 24 | 25 | Apache License, Version 2.0 26 | http://www.apache.org/licenses/LICENSE-2.0.txt 27 | repo 28 | 29 | 30 | 31 | 32 | 33 | 34 | junit 35 | junit 36 | 4.11 37 | test 38 | 39 | 40 | 41 | 42 | org.mockito 43 | mockito-all 44 | 1.10.8 45 | test 46 | 47 | 48 | 49 | 50 | com.ottogroup.bi.spqr 51 | spqr-base 52 | ${project.version} 53 | provided 54 | 55 | 56 | 57 | 58 | com.espertech 59 | esper 60 | 5.2.0 61 | 62 | 63 | 64 | org.apache.ant 65 | ant 66 | 1.9.4 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | org.apache.maven.plugins 77 | maven-jar-plugin 78 | 2.3.1 79 | 80 | ${project.build.directory}/lib 81 | 82 | 83 | 84 | 85 | 86 | maven-dependency-plugin 87 | 88 | 89 | package 90 | 91 | copy-dependencies 92 | 93 | 94 | provided 95 | junit,org.slf4j,org.mockito,log4j 96 | ${project.build.directory}/lib 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | org.apache.maven.plugins 105 | maven-compiler-plugin 106 | 2.3.2 107 | 108 | 1.7 109 | 1.7 110 | UTF-8 111 | 112 | 113 | 114 | 115 | -------------------------------------------------------------------------------- /spqr-operators/spqr-esper/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | -------------------------------------------------------------------------------- /spqr-operators/spqr-esper/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | -------------------------------------------------------------------------------- /spqr-operators/spqr-json/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | /.classpath 4 | /.project 5 | /bin/ 6 | -------------------------------------------------------------------------------- /spqr-operators/spqr-json/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | com.ottogroup.bi.spqr 7 | spqr-parent 8 | 0.6.0-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 12 | 13 | spqr-json 14 | jar 15 | 16 | 17 | spqr-json 18 | SPQR JSON operators 19 | https://github.com/ottogroup/SPQR.git 20 | 2015 21 | 22 | 23 | 24 | Apache License, Version 2.0 25 | http://www.apache.org/licenses/LICENSE-2.0.txt 26 | repo 27 | 28 | 29 | 30 | 31 | 32 | 33 | junit 34 | junit 35 | 4.11 36 | test 37 | 38 | 39 | 40 | 41 | org.mockito 42 | mockito-all 43 | 1.10.8 44 | test 45 | 46 | 47 | 48 | 49 | com.ottogroup.bi.spqr 50 | spqr-base 51 | ${project.version} 52 | provided 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-jar-plugin 66 | 2.3.1 67 | 68 | ${project.build.directory}/lib 69 | 70 | 71 | 72 | 73 | 74 | 75 | maven-dependency-plugin 76 | 77 | 78 | package 79 | 80 | copy-dependencies 81 | 82 | 83 | provided 84 | junit,org.slf4j,org.mockito,log4j 85 | ${project.build.directory}/lib 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | org.apache.maven.plugins 94 | maven-compiler-plugin 95 | 2.3.2 96 | 97 | 1.7 98 | 1.7 99 | UTF-8 100 | 101 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /spqr-operators/spqr-json/src/main/java/com/ottogroup/bi/spqr/operator/json/JsonContentType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.operator.json; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * Allowed value types: STRING, NUMERICAL, UNKNOWN 22 | * @author mnxfst 23 | * @since Apr 8, 2015 24 | */ 25 | public enum JsonContentType implements Serializable { 26 | STRING, NUMERICAL, UNKNOWN 27 | } 28 | -------------------------------------------------------------------------------- /spqr-operators/spqr-json/src/main/java/com/ottogroup/bi/spqr/operator/json/aggregator/JsonContentAggregatorFieldSetting.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.spqr.operator.json.aggregator; 18 | 19 | import java.io.Serializable; 20 | 21 | import com.ottogroup.bi.spqr.operator.json.JsonContentType; 22 | 23 | /** 24 | * Stores all relevant information required for aggregating json content field data 25 | * @author mnxfst 26 | * @since Nov 12, 2014 27 | */ 28 | public class JsonContentAggregatorFieldSetting implements Serializable { 29 | 30 | private static final long serialVersionUID = 6129451872974489460L; 31 | 32 | private String field = null; 33 | private String[] path = null; 34 | private JsonContentType valueType = JsonContentType.UNKNOWN; 35 | 36 | /** 37 | * Default constructor 38 | */ 39 | public JsonContentAggregatorFieldSetting() { 40 | } 41 | 42 | /** 43 | * Initializes the settings using the provided input 44 | * @param field 45 | * @param path 46 | * @param valueType 47 | */ 48 | public JsonContentAggregatorFieldSetting(final String field, final String[] path, final JsonContentType valueType) { 49 | this.field = field; 50 | this.path = path; 51 | this.valueType = valueType; 52 | } 53 | 54 | public String getField() { 55 | return field; 56 | } 57 | 58 | public void setField(String field) { 59 | this.field = field; 60 | } 61 | 62 | public String[] getPath() { 63 | return path; 64 | } 65 | 66 | public void setPath(String[] path) { 67 | this.path = path; 68 | } 69 | 70 | public JsonContentType getValueType() { 71 | return valueType; 72 | } 73 | 74 | public void setValueType(JsonContentType valueType) { 75 | this.valueType = valueType; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /spqr-operators/spqr-json/src/main/java/com/ottogroup/bi/spqr/operator/json/filter/JsonContentFilterFieldSetting.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.operator.json.filter; 17 | 18 | import java.util.regex.Pattern; 19 | 20 | import com.ottogroup.bi.spqr.operator.json.JsonContentType; 21 | 22 | 23 | /** 24 | * Holds the settings for each field configured for {@link JsonContentFilter} 25 | * @author mnxfst 26 | * @since Apr 8, 2015 27 | */ 28 | public class JsonContentFilterFieldSetting { 29 | 30 | private final String[] path; 31 | private final Pattern expression; 32 | private final JsonContentType valueType; 33 | 34 | /** 35 | * Initializes the settings using the provided input 36 | * @param path 37 | * @param expression 38 | * @param valueType 39 | */ 40 | public JsonContentFilterFieldSetting(final String[] path, final Pattern expression, final JsonContentType valueType) { 41 | this.expression = expression; 42 | this.path = path; 43 | this.valueType = valueType; 44 | } 45 | 46 | public String[] getPath() { 47 | return path; 48 | } 49 | 50 | public Pattern getExpression() { 51 | return expression; 52 | } 53 | 54 | public JsonContentType getValueType() { 55 | return valueType; 56 | } 57 | 58 | 59 | } 60 | -------------------------------------------------------------------------------- /spqr-operators/spqr-json/src/main/java/com/ottogroup/bi/spqr/operator/json/flatten/JsonContentFlattening.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.operator.json.flatten; 17 | 18 | import java.util.Properties; 19 | 20 | import com.ottogroup.bi.spqr.exception.ComponentInitializationFailedException; 21 | import com.ottogroup.bi.spqr.exception.RequiredInputMissingException; 22 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponentType; 23 | import com.ottogroup.bi.spqr.pipeline.component.annotation.SPQRComponent; 24 | import com.ottogroup.bi.spqr.pipeline.component.operator.DirectResponseOperator; 25 | import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage; 26 | 27 | /** 28 | * Flattens JSON content to key/value structure where the key is represented by the concatenated JSON nodes and the key is the value attached to 29 | * the final node belonging to the path 30 | * 31 | * @author mnxfst 32 | * @since May 27, 2015 33 | */ 34 | @SPQRComponent(type=MicroPipelineComponentType.DIRECT_RESPONSE_OPERATOR, name="jsonContentFlattening", version="0.0.1", description="Flattens JSON content to k/v structure") 35 | public class JsonContentFlattening implements DirectResponseOperator { 36 | 37 | private String id; 38 | private long totalNumOfMessages; 39 | 40 | /** 41 | * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#initialize(java.util.Properties) 42 | */ 43 | public void initialize(Properties properties) throws RequiredInputMissingException, ComponentInitializationFailedException { 44 | // nothing to do ... right now 45 | } 46 | 47 | /** 48 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.DirectResponseOperator#onMessage(com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage) 49 | */ 50 | public StreamingDataMessage[] onMessage(StreamingDataMessage message) { 51 | return new StreamingDataMessage[]{message}; 52 | } 53 | 54 | /** 55 | * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#shutdown() 56 | */ 57 | public boolean shutdown() { 58 | return true; 59 | } 60 | 61 | /** 62 | * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#getType() 63 | */ 64 | public MicroPipelineComponentType getType() { 65 | return MicroPipelineComponentType.DIRECT_RESPONSE_OPERATOR; 66 | } 67 | 68 | /** 69 | * @see com.ottogroup.bi.spqr.pipeline.component.operator.Operator#getTotalNumOfMessages() 70 | */ 71 | public long getTotalNumOfMessages() { 72 | return this.totalNumOfMessages; 73 | } 74 | 75 | /** 76 | * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#setId(java.lang.String) 77 | */ 78 | public void setId(String id) { 79 | this.id = id; 80 | } 81 | 82 | /** 83 | * @see com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponent#getId() 84 | */ 85 | public String getId() { 86 | return this.id; 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /spqr-operators/spqr-kafka/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-operators/spqr-twitter/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-operators/spqr-twitter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | com.ottogroup.bi.spqr 7 | spqr-parent 8 | 0.6.0-SNAPSHOT 9 | ../../pom.xml 10 | 11 | 12 | 13 | spqr-twitter 14 | jar 15 | 16 | 17 | spqr-twitter 18 | SPQR Twitter stream consumer 19 | https://github.com/ottogroup/SPQR.git 20 | 2014 21 | 22 | 23 | 24 | Apache License, Version 2.0 25 | http://www.apache.org/licenses/LICENSE-2.0.txt 26 | repo 27 | 28 | 29 | 30 | 31 | 32 | 33 | junit 34 | junit 35 | 4.11 36 | test 37 | 38 | 39 | 40 | 41 | org.mockito 42 | mockito-all 43 | 1.10.8 44 | test 45 | 46 | 47 | 48 | 49 | com.ottogroup.bi.spqr 50 | spqr-base 51 | ${project.version} 52 | provided 53 | 54 | 55 | 56 | com.twitter 57 | hbc-core 58 | 2.2.0 59 | 60 | 61 | 62 | com.twitter 63 | hbc-twitter4j 64 | 2.2.0 65 | 66 | 67 | 68 | javax.servlet 69 | javax.servlet-api 70 | 3.1.0 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | org.apache.maven.plugins 81 | maven-jar-plugin 82 | 2.3.1 83 | 84 | ${project.build.directory}/lib 85 | 86 | 87 | 88 | 89 | 90 | maven-dependency-plugin 91 | 92 | 93 | package 94 | 95 | copy-dependencies 96 | 97 | 98 | provided 99 | junit,org.slf4j,org.mockito,log4j 100 | ${project.build.directory}/lib 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | org.apache.maven.plugins 109 | maven-compiler-plugin 110 | 2.3.2 111 | 112 | 1.7 113 | 1.7 114 | UTF-8 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /spqr-operators/spqr-twitter/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /spqr-operators/spqr-webtrends/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-operators/spqr-webtrends/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /spqr-repository/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.settings 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-repository/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | 5 | com.ottogroup.bi.spqr 6 | spqr-parent 7 | 0.6.0-SNAPSHOT 8 | 9 | 10 | 11 | spqr-repository 12 | jar 13 | 14 | 15 | spqr-repositpry 16 | SPQR component repository implementation 17 | https://github.com/ottogroup/SPQR.git 18 | 2015 19 | 20 | 21 | 22 | Apache License, Version 2.0 23 | http://www.apache.org/licenses/LICENSE-2.0.txt 24 | repo 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | junit 33 | junit 34 | 4.11 35 | test 36 | 37 | 38 | 39 | org.mockito 40 | mockito-all 41 | 1.10.8 42 | test 43 | 44 | 45 | 46 | 47 | com.ottogroup.bi.spqr 48 | spqr-base 49 | ${project.version} 50 | provided 51 | 52 | 53 | 54 | 55 | log4j 56 | log4j 57 | 1.2.17 58 | provided 59 | 60 | 61 | 62 | 63 | org.apache.commons 64 | commons-lang3 65 | 3.3.2 66 | compile 67 | 68 | 69 | 70 | 71 | com.fasterxml.jackson.core 72 | jackson-databind 73 | 2.5.1 74 | compile 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-compiler-plugin 87 | 3.2 88 | 89 | 1.7 90 | 1.7 91 | UTF-8 92 | 93 | 94 | 95 | 96 | 97 | maven-dependency-plugin 98 | 99 | 100 | package 101 | 102 | copy-dependencies 103 | 104 | 105 | provided 106 | junit,org.slf4j,org.mockito,log4j 107 | ${project.build.directory}/lib 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /spqr-repository/src/main/java/com/ottogroup/bi/spqr/repository/ComponentDescriptor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.repository; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.ottogroup.bi.spqr.pipeline.component.MicroPipelineComponentType; 22 | 23 | /** 24 | * Asap component description as extracted from classes annotated as {@link AsapComponent} 25 | * @author mnxfst 26 | * @since Oct 29, 2014 27 | * 28 | */ 29 | public class ComponentDescriptor implements Serializable { 30 | 31 | private static final long serialVersionUID = -1649046223771601278L; 32 | 33 | @JsonProperty ( value = "componentClass", required = true ) 34 | private String componentClass = null; 35 | @JsonProperty ( value = "type", required = true ) 36 | private MicroPipelineComponentType type = null; 37 | @JsonProperty ( value = "name", required = true ) 38 | private String name = null; 39 | @JsonProperty ( value = "version", required = true ) 40 | private String version = null; 41 | @JsonProperty ( value = "description", required = true ) 42 | private String description = null; 43 | 44 | /** 45 | * Default constructor 46 | */ 47 | public ComponentDescriptor() { 48 | } 49 | 50 | /** 51 | * Initializes the descriptor using the provided input 52 | * @param componentClass 53 | * @param type 54 | * @param name 55 | * @param version 56 | * @param description 57 | */ 58 | public ComponentDescriptor(final String componentClass, final MicroPipelineComponentType type, final String name, final String version, final String description) { 59 | this.componentClass = componentClass; 60 | this.type = type; 61 | this.name = name; 62 | this.version = version; 63 | this.description = description; 64 | } 65 | 66 | 67 | public String getComponentClass() { 68 | return componentClass; 69 | } 70 | 71 | public void setComponentClass(String componentClass) { 72 | this.componentClass = componentClass; 73 | } 74 | 75 | public MicroPipelineComponentType getType() { 76 | return type; 77 | } 78 | 79 | public void setType(MicroPipelineComponentType type) { 80 | this.type = type; 81 | } 82 | 83 | public String getName() { 84 | return name; 85 | } 86 | 87 | public void setName(String name) { 88 | this.name = name; 89 | } 90 | 91 | public String getVersion() { 92 | return version; 93 | } 94 | 95 | public void setVersion(String version) { 96 | this.version = version; 97 | } 98 | 99 | public String getDescription() { 100 | return description; 101 | } 102 | 103 | public void setDescription(String description) { 104 | this.description = description; 105 | } 106 | 107 | 108 | } 109 | -------------------------------------------------------------------------------- /spqr-repository/src/main/java/com/ottogroup/bi/spqr/repository/exception/ComponentAlreadySubmittedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.repository.exception; 17 | 18 | /** 19 | * Thrown in case a component is added twice to a micro pipeline 20 | * @author mnxfst 21 | * @since Dec 16, 2014 22 | */ 23 | public class ComponentAlreadySubmittedException extends Exception { 24 | 25 | private static final long serialVersionUID = -7320527783751346547L; 26 | 27 | public ComponentAlreadySubmittedException() { 28 | } 29 | 30 | public ComponentAlreadySubmittedException(String message) { 31 | super(message); 32 | } 33 | 34 | public ComponentAlreadySubmittedException(Throwable cause) { 35 | super(cause); 36 | } 37 | 38 | public ComponentAlreadySubmittedException(String message, Throwable cause) { 39 | super(message, cause); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /spqr-repository/src/main/java/com/ottogroup/bi/spqr/repository/exception/ComponentInstantiationFailedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.spqr.repository.exception; 18 | 19 | /** 20 | * Thrown in case the instantiation of a pipeline component failed to any reason 21 | * @author mnxfst 22 | * @since Sep 4, 2014 23 | */ 24 | public class ComponentInstantiationFailedException extends Exception { 25 | 26 | private static final long serialVersionUID = -8586826517371913125L; 27 | 28 | public ComponentInstantiationFailedException() { 29 | } 30 | 31 | public ComponentInstantiationFailedException(String message) { 32 | super(message); 33 | } 34 | 35 | public ComponentInstantiationFailedException(Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public ComponentInstantiationFailedException(String message, Throwable cause) { 40 | super(message, cause); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spqr-repository/src/main/java/com/ottogroup/bi/spqr/repository/exception/UnknownComponentException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.spqr.repository.exception; 18 | 19 | /** 20 | * Thrown in case an unknown component is requested 21 | * @author mnxfst 22 | * @since 12.06.2014 23 | * 24 | */ 25 | public class UnknownComponentException extends Exception { 26 | 27 | private static final long serialVersionUID = -665926912298363340L; 28 | 29 | public UnknownComponentException() { 30 | } 31 | 32 | public UnknownComponentException(String message) { 33 | super(message); 34 | } 35 | 36 | public UnknownComponentException(Throwable cause) { 37 | super(cause); 38 | } 39 | 40 | public UnknownComponentException(String message, Throwable cause) { 41 | super(message, cause); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /spqr-repository/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /spqr-resman/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.settings/ 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-resman/src/main/config/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, file 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | 10 | # Direct log messages to a log file 11 | log4j.appender.file=org.apache.log4j.RollingFileAppender 12 | log4j.appender.file.File=/opt/transport/streaming/spqr/spqr-resman/log/spqr-resman-log4j.log 13 | log4j.appender.file.MaxFileSize=10MB 14 | log4j.appender.file.MaxBackupIndex=10 15 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 16 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /spqr-resman/src/main/config/spqr-resman.cfg: -------------------------------------------------------------------------------- 1 | log4jConfiguration: "/opt/transport/streaming/spqr/spqr-resman/etc/log4j.properties" 2 | componentRepositoryFolder: "/opt/transport/streaming/spqr/spqr-resman/repo/" 3 | 4 | httpClient: 5 | timeout: 500ms 6 | connectionTimeout: 500ms 7 | timeToLive: 1h 8 | cookiesEnabled: false 9 | maxConnections: 1024 10 | maxConnectionsPerRoute: 1024 11 | keepAlive: 0ms 12 | retries: 0 13 | userAgent: spqr-resman/0.2 14 | 15 | server: 16 | applicationConnectors: 17 | - type: http 18 | port: 9090 19 | adminConnectors: 20 | - type: http 21 | port: 9091 22 | 23 | logging: 24 | 25 | level: INFO 26 | 27 | loggers: 28 | 29 | # Sets the level for 'com.example.app' to DEBUG. 30 | com.ottogroup.bi.spqr: DEBUG 31 | 32 | appenders: 33 | - type: file 34 | threshold: DEBUG 35 | logFormat: "%-6level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %X{code} %msg %n" 36 | currentLogFilename: /opt/transport/streaming/spqr/spqr-resman/log/spqr-node.log 37 | archivedLogFilenamePattern: /opt/transport/streaming/spqr/spqr-resman/log/spqr-resman-%d{yyyy-MM-dd}.log 38 | archivedFileCount: 7 39 | timeZone: UTC 40 | 41 | -------------------------------------------------------------------------------- /spqr-resman/src/main/java/com/ottogroup/bi/spqr/resman/node/SPQRNodeSupervisor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.resman.node; 17 | 18 | import java.util.concurrent.Callable; 19 | 20 | /** 21 | * Implements a supervisor which 22 | * @author mnxfst 23 | * @since Apr 13, 2015 24 | */ 25 | public class SPQRNodeSupervisor implements Callable { 26 | 27 | @Override 28 | public SPQRNodeSupervisorResult call() throws Exception { 29 | // TODO Auto-generated method stub 30 | return null; 31 | } 32 | 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /spqr-resman/src/main/java/com/ottogroup/bi/spqr/resman/node/SPQRNodeSupervisorResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.resman.node; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | /** 23 | * Information collected by the {@link SPQRNodeSupervisor} when checking the current 24 | * state of a spqr processing node 25 | * @author mnxfst 26 | * @since Apr 13, 2015 27 | */ 28 | public class SPQRNodeSupervisorResult implements Serializable { 29 | 30 | private static final long serialVersionUID = 8402125222047766899L; 31 | 32 | /** unique node identifier previously assigned by node manager */ 33 | @JsonProperty(value="nodeId", required=true) 34 | private String nodeId = null; 35 | 36 | 37 | } 38 | -------------------------------------------------------------------------------- /spqr-resman/src/main/java/com/ottogroup/bi/spqr/resman/resource/node/SPQRNodeManagementResource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.resman.resource.node; 17 | 18 | import javax.ws.rs.DELETE; 19 | import javax.ws.rs.POST; 20 | import javax.ws.rs.Path; 21 | import javax.ws.rs.PathParam; 22 | import javax.ws.rs.Produces; 23 | 24 | import com.codahale.metrics.annotation.Timed; 25 | import com.ottogroup.bi.spqr.node.message.NodeDeRegistration.NodeDeRegistrationResponse; 26 | import com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationRequest; 27 | import com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationResponse; 28 | import com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationState; 29 | import com.ottogroup.bi.spqr.resman.node.SPQRNodeManager; 30 | 31 | /** 32 | * Provides a REST API which allows active processing node registration and shutdown. All nodes that wish to participate in 33 | * or leave the cluster formed around and managed by the resource manager must use this API to notify the manager about their existence and availability. 34 | * @author mnxfst 35 | * @since Apr 10, 2015 36 | */ 37 | @Path("/nodes") 38 | public class SPQRNodeManagementResource { 39 | 40 | private final SPQRNodeManager nodeManager; 41 | 42 | public SPQRNodeManagementResource(final SPQRNodeManager nodeManager) { 43 | this.nodeManager = nodeManager; 44 | } 45 | 46 | /** 47 | * Registers the processing node which lives at the location provided inside the request 48 | * @param request 49 | * @return 50 | */ 51 | @Produces(value = "application/json") 52 | @Timed(name = "node-registration") 53 | @POST 54 | public NodeRegistrationResponse registerNode(final NodeRegistrationRequest request) { 55 | 56 | // ensure that the incoming request carries a valid and accessible body 57 | if(request == null) 58 | return new NodeRegistrationResponse("", NodeRegistrationState.MISSING_REQUEST, "Missing request body carrying required node information"); 59 | 60 | return this.nodeManager.registerNode(request.getProtocol(), request.getHost(), request.getServicePort(), request.getAdminPort()); 61 | } 62 | 63 | /** 64 | * De-Registers the referenced processing node and thus removes it from the SQPR cluster 65 | * @param nodeId 66 | * @return 67 | */ 68 | @Produces(value = "application/json") 69 | @Timed(name = "node-deregistration") 70 | @Path("{nodeId}") 71 | @DELETE 72 | public NodeDeRegistrationResponse deregisterNode(@PathParam("nodeId") final String nodeId) { 73 | return this.nodeManager.deregisterNode(nodeId); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /spqr-resman/src/main/java/com/ottogroup/bi/spqr/resman/resource/pipeline/SPQRPipelineManagementResource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.resman.resource.pipeline; 17 | 18 | import javax.ws.rs.POST; 19 | import javax.ws.rs.Path; 20 | import javax.ws.rs.Produces; 21 | 22 | import com.codahale.metrics.annotation.Timed; 23 | import com.ottogroup.bi.spqr.node.resource.pipeline.MicroPipelineInstantiationResponse; 24 | import com.ottogroup.bi.spqr.pipeline.MicroPipelineConfiguration; 25 | 26 | 27 | /** 28 | * Provides a REST API for managing pipelines within the SPQR cluster. It provides methods for 29 | *
    30 | *
  • pipeline instantiation
    • 31 | *
  • pipeline shutdown
    • 32 | *
  • fetching pipeline statistics
    • 33 | *
34 | * @author mnxfst 35 | * @since Apr 13, 2015 36 | */ 37 | @Path("/pipelines") 38 | public class SPQRPipelineManagementResource { 39 | 40 | 41 | /** 42 | * Registers a new processing node with the resource manager. It accepts a {@link MicroPipelineConfiguration} which 43 | * is distributed among cluster nodes. 44 | */ 45 | @Produces(value = "application/json") 46 | @Timed(name = "pipeline-instantiation") 47 | @POST 48 | public MicroPipelineInstantiationResponse instantiatePipeline(final MicroPipelineConfiguration configuration) { 49 | 50 | ////////////////////////////////////////////////////////////////////////// 51 | // validate provided configuration 52 | // MicroPipelineValidationResult cfgValidationResult = this.pipelineConfigurationValidator.validate(configuration); 53 | // if(cfgValidationResult != MicroPipelineValidationResult.OK) 54 | // return new MicroPipelineInstantiationResponse("", cfgValidationResult, "Validation of provided configuration failed"); 55 | // 56 | ////////////////////////////////////////////////////////////////////////// 57 | 58 | return null; 59 | 60 | } 61 | 62 | 63 | } 64 | -------------------------------------------------------------------------------- /spqr-resman/src/main/java/com/ottogroup/bi/spqr/resman/server/SPQRResourceManagerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.resman.server; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import io.dropwizard.Configuration; 21 | import io.dropwizard.client.JerseyClientConfiguration; 22 | 23 | /** 24 | * Provides a mapping for all relevant configuration options required on resource 25 | * manager initialization 26 | * @author mnxfst 27 | * @since Apr 10, 2015 28 | */ 29 | public class SPQRResourceManagerConfiguration extends Configuration { 30 | 31 | /** log4j */ 32 | @JsonProperty(value="log4jConfiguration", required=true) 33 | private String log4jConfiguration = null; 34 | /** component repository folder */ 35 | @JsonProperty(value="componentRepositoryFolder", required=true) 36 | private String componentRepositoryFolder = null; 37 | /** http client configuration */ 38 | @JsonProperty(value="httpClient", required=true) 39 | private JerseyClientConfiguration httpClient; 40 | 41 | public String getLog4jConfiguration() { 42 | return log4jConfiguration; 43 | } 44 | public void setLog4jConfiguration(String log4jConfiguration) { 45 | this.log4jConfiguration = log4jConfiguration; 46 | } 47 | public String getComponentRepositoryFolder() { 48 | return componentRepositoryFolder; 49 | } 50 | public void setComponentRepositoryFolder(String componentRepositoryFolder) { 51 | this.componentRepositoryFolder = componentRepositoryFolder; 52 | } 53 | public JerseyClientConfiguration getHttpClient() { 54 | return httpClient; 55 | } 56 | public void setHttpClient(JerseyClientConfiguration httpClient) { 57 | this.httpClient = httpClient; 58 | } 59 | 60 | 61 | 62 | } 63 | -------------------------------------------------------------------------------- /spqr-resman/src/main/java/com/ottogroup/bi/spqr/resman/server/SPQRResourceManagerShutdownHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.resman.server; 17 | 18 | import org.apache.log4j.Logger; 19 | 20 | /** 21 | * Manages the resource manager shutdown 22 | * @author mnxfst 23 | * @since Apr 10, 2015 24 | */ 25 | public class SPQRResourceManagerShutdownHandler extends Thread { 26 | 27 | /** our faithful logging facility ... ;-) */ 28 | private static final Logger logger = Logger.getLogger(SPQRResourceManagerShutdownHandler.class); 29 | 30 | /** 31 | * @see java.lang.Thread#run() 32 | */ 33 | public void run() { 34 | logger.info("SPQR resource manager instance successfully shut down"); 35 | } 36 | 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /spqr-resman/src/main/scripts/spqr-resman.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BASEDIR=$(dirname $0) 4 | PIDFILE=$BASEDIR/pidfile 5 | 6 | if [ -z "$1" ]; then 7 | # write error message 8 | echo "usage: spqr-resman.sh {start|stop} " 9 | exit 1 10 | fi 11 | 12 | if [ "$1" = "stop" ]; then 13 | 14 | # check for running service 15 | if [ ! -f "$PIDFILE" ]; then 16 | echo "spqr resource manager is not running!" 17 | exit 1 18 | fi 19 | 20 | kill -9 `cat $PIDFILE` 21 | rm $PIDFILE 22 | echo "spqr resource manager shut down!" 23 | exit 0 24 | fi 25 | 26 | if [ -z "$2" ]; then 27 | # write error message 28 | echo "usage: spqr-resman.sh {start|stop} " 29 | exit 1 30 | fi 31 | 32 | if [ "$1" = "start" ]; then 33 | 34 | # check for running service 35 | if [ -f "$PIDFILE" ]; then 36 | echo "spqr resource manager already running! process id: `cat $PIDFILE`" 37 | exit 1 38 | fi 39 | 40 | CLASSPATH=.:$BASEDIR/../lib/* 41 | java -d64 -server -XX:MaxPermSize=512M -XX:MaxGCPauseMillis=500 -XX:+UseG1GC -Xms1G -cp $CLASSPATH com.ottogroup.bi.spqr.resman.server.SPQRResourceManagerServer server $2 & 42 | echo $! > $PIDFILE 43 | echo "spqr resource manager running. process id: `cat $PIDFILE`" 44 | exit 0 45 | fi 46 | 47 | if [ "$1" = "restart" ]; then 48 | # check for running service 49 | if [ -f "$PIDFILE" ]; then 50 | kill -9 `cat $PIDFILE` 51 | rm $PIDFILE 52 | echo "spqr resource manager shut down!" 53 | fi 54 | 55 | CLASSPATH=.:$BASEDIR/../lib/* 56 | java -d64 -server -XX:MaxPermSize=512M -XX:MaxGCPauseMillis=500 -XX:+UseG1GC -Xms1G -cp $CLASSPATH com.ottogroup.bi.spqr.resman.server.SPQRResourceManagerServer server $2 & 57 | echo $! > $PIDFILE 58 | echo "spqr resource manager running. process id: `cat $PIDFILE`" 59 | exit 0 60 | fi -------------------------------------------------------------------------------- /spqr-resman/src/test/java/com/ottogroup/bi/spqr/resman/resource/node/SPQRNodeManagementResourceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.resman.resource.node; 17 | 18 | import org.junit.Assert; 19 | import org.junit.Test; 20 | import org.mockito.Mockito; 21 | 22 | import com.ottogroup.bi.spqr.node.message.NodeDeRegistration.NodeDeRegistrationResponse; 23 | import com.ottogroup.bi.spqr.node.message.NodeDeRegistration.NodeDeRegistrationState; 24 | import com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationRequest; 25 | import com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationResponse; 26 | import com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationState; 27 | import com.ottogroup.bi.spqr.resman.node.SPQRNodeManager; 28 | 29 | /** 30 | * Test case for {@link SPQRNodeManagementResource} 31 | * @author mnxfst 32 | * @since Apr 14, 2015 33 | */ 34 | public class SPQRNodeManagementResourceTest { 35 | 36 | /** 37 | * Test case for {@link SPQRNodeManagementResource#registerNode(com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationRequest)} being provided 38 | * null as input 39 | */ 40 | @Test 41 | public void testRegisterNode_withNullInput() { 42 | Assert.assertEquals("Invalid input", NodeRegistrationState.MISSING_REQUEST, new SPQRNodeManagementResource(Mockito.mock(SPQRNodeManager.class)).registerNode(null).getState()); 43 | } 44 | 45 | /** 46 | * Test case for {@link SPQRNodeManagementResource#registerNode(com.ottogroup.bi.spqr.node.message.NodeRegistration.NodeRegistrationRequest)} being provided 47 | * valid input 48 | */ 49 | @Test 50 | public void testRegisterNode_withValidInput() { 51 | SPQRNodeManager mockedNodeManager = Mockito.mock(SPQRNodeManager.class); 52 | NodeRegistrationRequest request = new NodeRegistrationRequest("http", "localhost", 8080, 8081); 53 | Mockito.when(mockedNodeManager.registerNode(request.getProtocol(), request.getHost(), request.getServicePort(), request.getAdminPort())). 54 | thenReturn(new NodeRegistrationResponse("test", NodeRegistrationState.OK)); 55 | Assert.assertEquals("Valid input provided", NodeRegistrationState.OK, new SPQRNodeManagementResource(mockedNodeManager).registerNode(request).getState()); 56 | Mockito.verify(mockedNodeManager).registerNode(request.getProtocol(), request.getHost(), request.getServicePort(), request.getAdminPort()); 57 | } 58 | 59 | /** 60 | * Test case for {@link SPQRNodeManagementResource#deregisterNode(String)} being provided null 61 | */ 62 | @Test 63 | public void testDeRegisterNode_withNullInput() { 64 | SPQRNodeManager manager = Mockito.mock(SPQRNodeManager.class); 65 | Mockito.when(manager.deregisterNode(null)).thenReturn(new NodeDeRegistrationResponse("", NodeDeRegistrationState.MISSING_NODE_ID, "")); 66 | Assert.assertEquals("Invalid input", NodeDeRegistrationState.MISSING_NODE_ID, new SPQRNodeManagementResource(manager).deregisterNode(null).getState()); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /spqr-websocket-server/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /.settings 3 | /.classpath 4 | /.project 5 | -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/config/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, file 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | 10 | # Direct log messages to a log file 11 | log4j.appender.file=org.apache.log4j.RollingFileAppender 12 | log4j.appender.file.File=/opt/transport/streaming/spqr/spqr-websocket-server/log/spqr-websocket-server.log 13 | log4j.appender.file.MaxFileSize=10MB 14 | log4j.appender.file.MaxBackupIndex=10 15 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 16 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/config/spqr-websocket-server.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "log4jConfigurationFile" : "/opt/transport/streaming/spqr/spqr-websocket-server/etc/log4j.properties", 3 | "port" : 9090, 4 | "bossEventGroupThreads" : 1, 5 | "workerEventGroupThreads" : 1, 6 | "kafkaConsumer" : { 7 | "kafkaAutoOffsetReset" : "largest", 8 | "zookeeperConnect" : "localhost:2081", 9 | "zookeeperSessionTimeout" : 500, 10 | "zookeeperSyncInterval" : 200, 11 | "zookeeperAutoCommitEnabled" : false, 12 | "zookeeperAutoCommitInterval" : 200 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/java/com/ottogroup/bi/spqr/websocket/kafka/KafkaTopicRequest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.websocket.kafka; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.fasterxml.jackson.annotation.JsonRootName; 22 | 23 | /** 24 | * Describes the format expected when clients request a specific topic to 25 | * be exported into a websocket 26 | * @author mnxfst 27 | * @since Apr 20, 2015 28 | */ 29 | @JsonRootName(value="kafkaTopicRequest") 30 | public class KafkaTopicRequest implements Serializable { 31 | 32 | private static final long serialVersionUID = 4099372895981197335L; 33 | 34 | @JsonProperty(value="topicId", required=true) 35 | private String topicId = null; 36 | 37 | public String getTopicId() { 38 | return topicId; 39 | } 40 | 41 | public void setTopicId(String topicId) { 42 | this.topicId = topicId; 43 | } 44 | 45 | 46 | } 47 | -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/java/com/ottogroup/bi/spqr/websocket/kafka/KafkaTopicStreamConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.websocket.kafka; 17 | 18 | import java.util.concurrent.locks.LockSupport; 19 | 20 | import kafka.consumer.ConsumerIterator; 21 | import kafka.consumer.KafkaStream; 22 | import kafka.message.MessageAndMetadata; 23 | import uk.co.real_logic.queues.MessageWaitStrategy; 24 | import uk.co.real_logic.queues.OneToOneConcurrentArrayQueue3; 25 | 26 | /** 27 | * Reads content from an assigned {@link KafkaStream} and writes the data to a provided queue. 28 | * @author mnxfst 29 | * @since Apr 20, 2015 30 | */ 31 | public class KafkaTopicStreamConsumer implements Runnable { 32 | 33 | /** externally provided queue to use for exchanging messages with underlying kafka emitter */ 34 | private final OneToOneConcurrentArrayQueue3 messages; 35 | /** stream instance to read messages from */ 36 | private final KafkaStream kafkaTopicPartitionStream; 37 | /** indicates that the consumer is still running - may be used to shut down the consumer by simply setting to 'false' */ 38 | private boolean running = false; 39 | private final MessageWaitStrategy messageWaitStrategy; 40 | 41 | private static final int RETRIES = 200; 42 | 43 | /** 44 | * Initializes the partition consumer using the provided input 45 | * @param kafkaTopicStream 46 | * @param messages queue to write received content to 47 | * @param messageWaitStrategy optional wait strategy applied when consuming data from queue. if provided it may be used to signal new elements 48 | */ 49 | public KafkaTopicStreamConsumer(final KafkaStream kafkaTopicStream, 50 | final OneToOneConcurrentArrayQueue3 messages, final MessageWaitStrategy messageWaitStrategy) { 51 | this.kafkaTopicPartitionStream = kafkaTopicStream; 52 | this.messages = messages; 53 | this.messageWaitStrategy = messageWaitStrategy; 54 | } 55 | 56 | /** 57 | * @see java.lang.Runnable#run() 58 | */ 59 | public void run() { 60 | 61 | // fetch message iterator from stream and mark the consumer as 'running' instance 62 | ConsumerIterator topicPartitionStreamIterator = this.kafkaTopicPartitionStream.iterator(); 63 | this.running = true; 64 | 65 | // keep on running ... until told to stop 66 | int counter = 0; 67 | while(running) { 68 | 69 | counter = RETRIES; 70 | while(!topicPartitionStreamIterator.hasNext()) { 71 | counter = waitFor(counter); // TODO provide a configurable wait strategy 72 | } 73 | 74 | // receive the next message from stream 75 | MessageAndMetadata message = topicPartitionStreamIterator.next(); 76 | if(message != null && message.message() != null && message.message().length > 0) { 77 | // if the message is neither null nor empty, insert it into the queue and signal the wait strategy to 78 | // release any existing locks -- if there is a wait strategy provided at all 79 | this.messages.add(message.message()); 80 | if(messageWaitStrategy != null) 81 | messageWaitStrategy.forceLockRelease(); 82 | } 83 | } 84 | } 85 | 86 | public int waitFor(int counter) { 87 | if(counter > 100) 88 | --counter; 89 | else if(counter > 0) { 90 | --counter; 91 | Thread.yield(); 92 | } else { 93 | LockSupport.parkNanos(1l); 94 | } 95 | 96 | return counter; 97 | } 98 | 99 | /** 100 | * Shuts down the partition consumer 101 | */ 102 | public void shutdown() { 103 | this.running = false; 104 | } 105 | 106 | 107 | } -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/java/com/ottogroup/bi/spqr/websocket/server/SPQRWebSocketServerInitializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.websocket.server; 17 | 18 | import io.netty.channel.ChannelInitializer; 19 | import io.netty.channel.ChannelPipeline; 20 | import io.netty.channel.socket.SocketChannel; 21 | import io.netty.handler.codec.http.HttpObjectAggregator; 22 | import io.netty.handler.codec.http.HttpServerCodec; 23 | 24 | /** 25 | * Initializes the server's processing chain 26 | * @author mnxfst 27 | * @since Apr 17, 2015 28 | */ 29 | public class SPQRWebSocketServerInitializer extends ChannelInitializer { 30 | 31 | /** 32 | * @see io.netty.channel.ChannelInitializer#initChannel(io.netty.channel.Channel) 33 | */ 34 | protected void initChannel(SocketChannel ch) throws Exception { 35 | ChannelPipeline pipeline = ch.pipeline(); 36 | pipeline.addLast(new HttpServerCodec()); 37 | pipeline.addLast(new HttpObjectAggregator(65536)); 38 | pipeline.addLast(new SPQRWebSocketServerHandler()); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/java/com/ottogroup/bi/spqr/websocket/server/cfg/SPQRKafkaConsumerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.websocket.server.cfg; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.ottogroup.bi.spqr.websocket.kafka.KafkaTopicConsumer; 22 | 23 | /** 24 | * All required configuration options for setting up the {@link KafkaTopicConsumer} 25 | * @author mnxfst 26 | * @since Apr 17, 2015 27 | */ 28 | public class SPQRKafkaConsumerConfiguration implements Serializable { 29 | 30 | private static final long serialVersionUID = 5190373154294783314L; 31 | 32 | /** how to handle offset determination when no inital offset is available from zookeeper: "smallest", "largest" */ 33 | @JsonProperty(value="kafkaAutoOffsetReset", required=true) 34 | private String autoOffsetReset = "largest"; 35 | /** zookeeper connect string, eg: localhost:2181 */ 36 | @JsonProperty(value="zookeeperConnect", required=true) 37 | private String zookeeperConnect = null; 38 | /** zookeeper timeout given in milliseconds */ 39 | @JsonProperty(value="zookeeperSessionTimeout", required=true) 40 | private int zookeeperSessionTimeout = 0; 41 | /** zookeeper sync'ing interval given in milliseconds */ 42 | @JsonProperty(value="zookeeperSyncInterval", required=true) 43 | private int zookeeperSyncInterval = 0; 44 | /** enables auto committing offsets to zookeeper, values: "true", "false" */ 45 | @JsonProperty(value="zookeeperAutoCommitEnabled", required=true) 46 | private boolean zookeeperAutoCommitEnabled = false; 47 | /** interval given in milliseconds to execute offset auto commits to zookeeper */ 48 | @JsonProperty(value="zookeeperAutoCommitInterval", required=true) 49 | private int zookeeperAutoCommitInterval = 0; 50 | 51 | public String getAutoOffsetReset() { 52 | return autoOffsetReset; 53 | } 54 | public void setAutoOffsetReset(String autoOffsetReset) { 55 | this.autoOffsetReset = autoOffsetReset; 56 | } 57 | public String getZookeeperConnect() { 58 | return zookeeperConnect; 59 | } 60 | public void setZookeeperConnect(String zookeeperConnect) { 61 | this.zookeeperConnect = zookeeperConnect; 62 | } 63 | public int getZookeeperSessionTimeout() { 64 | return zookeeperSessionTimeout; 65 | } 66 | public void setZookeeperSessionTimeout(int zookeeperSessionTimeout) { 67 | this.zookeeperSessionTimeout = zookeeperSessionTimeout; 68 | } 69 | public int getZookeeperSyncInterval() { 70 | return zookeeperSyncInterval; 71 | } 72 | public void setZookeeperSyncInterval(int zookeeperSyncInterval) { 73 | this.zookeeperSyncInterval = zookeeperSyncInterval; 74 | } 75 | public boolean isZookeeperAutoCommitEnabled() { 76 | return zookeeperAutoCommitEnabled; 77 | } 78 | public void setZookeeperAutoCommitEnabled(boolean zookeeperAutoCommitEnabled) { 79 | this.zookeeperAutoCommitEnabled = zookeeperAutoCommitEnabled; 80 | } 81 | public int getZookeeperAutoCommitInterval() { 82 | return zookeeperAutoCommitInterval; 83 | } 84 | public void setZookeeperAutoCommitInterval(int zookeeperAutoCommitInterval) { 85 | this.zookeeperAutoCommitInterval = zookeeperAutoCommitInterval; 86 | } 87 | public static long getSerialversionuid() { 88 | return serialVersionUID; 89 | } 90 | 91 | 92 | } 93 | -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/java/com/ottogroup/bi/spqr/websocket/server/cfg/SPQRWebSocketServerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.websocket.server.cfg; 17 | 18 | import java.io.Serializable; 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.fasterxml.jackson.annotation.JsonRootName; 22 | import com.ottogroup.bi.spqr.websocket.server.SPQRWebSocketServer; 23 | 24 | /** 25 | * Provides all configuration options required for setting up the {@link SPQRWebSocketServer} 26 | * @author mnxfst 27 | * @since Apr 17, 2015 28 | */ 29 | @JsonRootName("spqrWebSocketServerConfiguration") 30 | public class SPQRWebSocketServerConfiguration implements Serializable { 31 | 32 | private static final long serialVersionUID = 4463978663305593008L; 33 | 34 | /** log4j configuration file */ 35 | @JsonProperty(value="log4jConfigurationFile", required=true) 36 | private String log4jConfigurationFile = null; 37 | /** port the server listens to for incoming requests */ 38 | @JsonProperty(value="port", required=true) 39 | private int port = 8080; 40 | /** threads used by boss event loop group */ 41 | @JsonProperty(value="bossEventGroupThreads", required=true) 42 | private int bossEventGroupThreads = 1; 43 | /** threads used by worker event loop group */ 44 | @JsonProperty(value="workerEventGroupThreads", required=true) 45 | private int workerEventGroupThreads = 1; 46 | /** kafka consumer configuration */ 47 | @JsonProperty(value="kafkaConsumer", required=true) 48 | private SPQRKafkaConsumerConfiguration kafkaConsumer = null; 49 | 50 | public String getLog4jConfigurationFile() { 51 | return log4jConfigurationFile; 52 | } 53 | public void setLog4jConfigurationFile(String log4jConfigurationFile) { 54 | this.log4jConfigurationFile = log4jConfigurationFile; 55 | } 56 | public int getPort() { 57 | return port; 58 | } 59 | public void setPort(int port) { 60 | this.port = port; 61 | } 62 | public SPQRKafkaConsumerConfiguration getKafkaConsumer() { 63 | return kafkaConsumer; 64 | } 65 | public void setKafkaConsumer(SPQRKafkaConsumerConfiguration kafkaConsumer) { 66 | this.kafkaConsumer = kafkaConsumer; 67 | } 68 | public int getBossEventGroupThreads() { 69 | return bossEventGroupThreads; 70 | } 71 | public void setBossEventGroupThreads(int bossEventGroupThreads) { 72 | this.bossEventGroupThreads = bossEventGroupThreads; 73 | } 74 | public int getWorkerEventGroupThreads() { 75 | return workerEventGroupThreads; 76 | } 77 | public void setWorkerEventGroupThreads(int workerEventGroupThreads) { 78 | this.workerEventGroupThreads = workerEventGroupThreads; 79 | } 80 | 81 | 82 | } 83 | -------------------------------------------------------------------------------- /spqr-websocket-server/src/main/scripts/spqr-websocket-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BASEDIR=$(dirname $0) 4 | PIDFILE=$BASEDIR/pidfile 5 | 6 | if [ -z "$1" ]; then 7 | # write error message 8 | echo "usage: spqr-websocket-server.sh {start|stop} " 9 | exit 1 10 | fi 11 | 12 | if [ "$1" = "stop" ]; then 13 | 14 | # check for running service 15 | if [ ! -f "$PIDFILE" ]; then 16 | echo "spqr websocket server is not running!" 17 | exit 1 18 | fi 19 | 20 | kill -9 `cat $PIDFILE` 21 | rm $PIDFILE 22 | echo "spqr websocket server shut down!" 23 | exit 0 24 | fi 25 | 26 | if [ -z "$2" ]; then 27 | # write error message 28 | echo "usage: spqr-websocket-server.sh {start|stop} " 29 | exit 1 30 | fi 31 | 32 | if [ "$1" = "start" ]; then 33 | 34 | # check for running service 35 | if [ -f "$PIDFILE" ]; then 36 | echo "spqr websocket server already running! process id: `cat $PIDFILE`" 37 | exit 1 38 | fi 39 | 40 | CLASSPATH=.:$BASEDIR/../lib/* 41 | java -d64 -server -XX:MaxPermSize=512M -XX:MaxGCPauseMillis=500 -XX:+UseG1GC -Xms1G -cp $CLASSPATH com.ottogroup.bi.spqr.websocket.server.SPQRWebSocketServer -c $2 & 42 | echo $! > $PIDFILE 43 | echo "spqr websocket server running. process id: `cat $PIDFILE`" 44 | exit 0 45 | fi 46 | 47 | if [ "$1" = "restart" ]; then 48 | # check for running service 49 | if [ -f "$PIDFILE" ]; then 50 | kill -9 `cat $PIDFILE` 51 | rm $PIDFILE 52 | echo "spqr websocket server shut down!" 53 | fi 54 | 55 | CLASSPATH=.:$BASEDIR/../lib/* 56 | java -d64 -server -XX:MaxPermSize=512M -XX:MaxGCPauseMillis=500 -XX:+UseG1GC -Xms1G -cp $CLASSPATH com.ottogroup.bi.spqr.websocket.server.SPQRWebSocketServer -c $2 & 57 | echo $! > $PIDFILE 58 | echo "spqr websocket server running. process id: `cat $PIDFILE`" 59 | exit 0 60 | fi -------------------------------------------------------------------------------- /spqr-websocket-server/src/test/java/com/ottogroup/bi/spqr/websocket/server/SPQRWebSocketServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.spqr.websocket.server; 17 | 18 | import java.io.ByteArrayInputStream; 19 | import java.io.IOException; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | import com.fasterxml.jackson.core.JsonParseException; 25 | import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; 26 | 27 | /** 28 | * Test case for {@link SPQRWebSocketServer} 29 | * @author mnxfst 30 | * @since Apr 17, 2015 31 | */ 32 | public class SPQRWebSocketServerTest { 33 | 34 | /** 35 | * Test case for {@link SPQRWebSocketServer#run(String)} being provided an empty 36 | * file reference 37 | */ 38 | @Test 39 | public void testRun_withEmptyFileReference() throws Exception { 40 | try { 41 | new SPQRWebSocketServer().run(""); 42 | Assert.fail("Invalid input"); 43 | } catch(IOException e) { 44 | // expected 45 | } 46 | } 47 | 48 | /** 49 | * Test case for {@link SPQRWebSocketServer#run(String)} being provided an invalid 50 | * file reference 51 | */ 52 | @Test 53 | public void testRun_withInvalidFileReference() throws Exception { 54 | try { 55 | new SPQRWebSocketServer().run("/no/such/folder/no-such-file-"+System.currentTimeMillis()+"-.cfg"); 56 | Assert.fail("Invalid input"); 57 | } catch(IOException e) { 58 | // expected 59 | } 60 | } 61 | 62 | /** 63 | * Test case for {@link SPQRWebSocketServer#run(java.io.InputStream)} being provided a 64 | * stream holding a non-JSON string 65 | */ 66 | @Test 67 | public void testRun_withStreamHoldingNonJSONString() throws Exception { 68 | try { 69 | new SPQRWebSocketServer().run(new ByteArrayInputStream("no-json".getBytes())); 70 | Assert.fail("Invalid input"); 71 | } catch(JsonParseException e) { 72 | // expected 73 | } 74 | } 75 | 76 | /** 77 | * Test case for {@link SPQRWebSocketServer#run(java.io.InputStream)} being provided a 78 | * stream holding a JSON string which does not comply with expected format 79 | */ 80 | @Test 81 | public void testRun_withStreamHoldingInvalidJSONFormat() throws Exception { 82 | try { 83 | new SPQRWebSocketServer().run(new ByteArrayInputStream("{\"field\":\"value\"}".getBytes())); 84 | Assert.fail("Invalid input"); 85 | } catch(UnrecognizedPropertyException e) { 86 | // expected 87 | } 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /spqr-websocket-server/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n --------------------------------------------------------------------------------