├── README.md ├── CHANGELOG.md ├── 3rdparty.md ├── .gitignore ├── jetstream-esper ├── src │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── ebay │ │ │ └── jetstream │ │ │ ├── event │ │ │ └── processor │ │ │ │ └── esper │ │ │ │ ├── annotation │ │ │ │ ├── BroadCast.java │ │ │ │ ├── OutputTo.java │ │ │ │ ├── PublishOn.java │ │ │ │ ├── CreateDimension.java │ │ │ │ ├── ClusterAffinityTag.java │ │ │ │ ├── OutputToWithNewStreamName.java │ │ │ │ ├── DBInfo.java │ │ │ │ ├── metadata │ │ │ │ │ ├── BroadCastMetadata.java │ │ │ │ │ ├── DBInfoAnnotationMetadata.java │ │ │ │ │ ├── AffinityAnnotationMetadata.java │ │ │ │ │ ├── PublishOnAnnotationMetadata.java │ │ │ │ │ ├── OutputToAnnotationMetadata.java │ │ │ │ │ ├── SendMailAnnotationMetadata.java │ │ │ │ │ └── OutputToNewStreamMetadata.java │ │ │ │ ├── SendMail.java │ │ │ │ ├── config │ │ │ │ │ ├── DefaultAnnotationConfiguration.java │ │ │ │ │ └── AnnotationConfigLoader.java │ │ │ │ ├── listener │ │ │ │ │ ├── BroadCastAnnotationListener.java │ │ │ │ │ ├── DBInfoAnnotationListener.java │ │ │ │ │ ├── AffinityAnnotationListener.java │ │ │ │ │ ├── OutputToAnnotationListener.java │ │ │ │ │ ├── PublishOnAnnotationListener.java │ │ │ │ │ ├── OutputToNewStreamListener.java │ │ │ │ │ └── SendMailAnnotationListener.java │ │ │ │ └── processor │ │ │ │ │ ├── BroadCastAnnotationProcessor.java │ │ │ │ │ ├── DimensionAnnotationProcessor.java │ │ │ │ │ ├── AffinityAnnotationProcessor.java │ │ │ │ │ ├── AnnotationProcessorFacade.java │ │ │ │ │ ├── PublishOnAnnotationProcessor.java │ │ │ │ │ ├── SendMailAnnotationProcessor.java │ │ │ │ │ ├── OutputToNewStreamProcessor.java │ │ │ │ │ ├── DBInfoAnnotationProcessor.java │ │ │ │ │ └── OutputToAnnotationProcessor.java │ │ │ │ ├── EventListener.java │ │ │ │ ├── DataSourceConfig.java │ │ │ │ ├── StringEventType.java │ │ │ │ ├── ClassEventType.java │ │ │ │ ├── AbstractEventType.java │ │ │ │ ├── EsperEngineAnnotationMetadata.java │ │ │ │ ├── aggregates │ │ │ │ ├── CardinalityAggregator.java │ │ │ │ ├── CardinalityAggregatorFactory.java │ │ │ │ ├── QuantileAggregator.java │ │ │ │ ├── TopKAggregatorFactory.java │ │ │ │ ├── QuantileAggregatorFactory.java │ │ │ │ └── TopKAggregator.java │ │ │ │ ├── EsperDeclaredEvents.java │ │ │ │ ├── EsperEventConverter.java │ │ │ │ ├── ProcessEventRequest.java │ │ │ │ ├── ProcessDestroyEngineRequest.java │ │ │ │ ├── JetstreamExceptionHandlerFactory.java │ │ │ │ ├── EsperWrappedEventConverter.java │ │ │ │ ├── CreateDimensionGenerator.java │ │ │ │ ├── EsperDefaultEventConverter.java │ │ │ │ ├── AffinityKeyGenerator.java │ │ │ │ ├── MapEventType.java │ │ │ │ ├── EsperExceptionHandler.java │ │ │ │ ├── EPL.java │ │ │ │ └── OnDemandQuery.java │ │ │ └── epl │ │ │ ├── EntityType.java │ │ │ ├── HistogramAggregatorFactory.java │ │ │ ├── HistogramAggregator.java │ │ │ ├── AttributesAliasesConfiguration.java │ │ │ ├── JetstreamSideEventIdGenerator.java │ │ │ ├── EPLUtils.java │ │ │ └── ClipBoard.java │ └── test │ │ └── java │ │ └── com │ │ └── ebay │ │ └── jetstream │ │ ├── event │ │ ├── processor │ │ │ └── esper │ │ │ │ ├── EsperTestEventField.java │ │ │ │ ├── raw │ │ │ │ ├── EsperTestAggregationListener.java │ │ │ │ ├── EsperTestStatement.java │ │ │ │ ├── EsperTestAggregationRunnable.java │ │ │ │ ├── EsperTestSubscriber.java │ │ │ │ ├── EsperTestAggregationStatement.java │ │ │ │ ├── EsperTestListener.java │ │ │ │ ├── EsperTestConfig.xml │ │ │ │ ├── log4j.xml │ │ │ │ ├── EsperTestRunnable.java │ │ │ │ └── EsperTestConfigurationValidator.java │ │ │ │ ├── ESPTestListenerForStringUtilsTest.java │ │ │ │ ├── EsperStreamTest.java │ │ │ │ ├── ESPTestAggregationSink.java │ │ │ │ ├── ESPTestSubscriber.java │ │ │ │ ├── ESPTestAggregationListener.java │ │ │ │ ├── ESPTestSink.java │ │ │ │ ├── ESPTestRunnable.java │ │ │ │ ├── ESPTestListener.java │ │ │ │ ├── log4j.xml │ │ │ │ ├── ESPEventProcessorTest.java │ │ │ │ ├── advice │ │ │ │ ├── EsperProcessorTest.java │ │ │ │ └── EsperConfig.xml │ │ │ │ └── EsperStreamTest.xml │ │ └── esper │ │ │ └── customaggregate │ │ │ ├── MyConcatAggregationFunctionFactory.java │ │ │ ├── MySecondConcatAggregationFunctionFactory.java │ │ │ ├── MyConcatAggregationFunction.java │ │ │ ├── MySecondConcatAggregationFunction.java │ │ │ ├── EventListener.java │ │ │ ├── ESPAggrTest.xml │ │ │ └── ESPAggrTest.java │ │ ├── epl │ │ ├── EPLUtilitiesTest.java │ │ ├── EPLCompileTest.java │ │ └── sampleEPL.txt │ │ └── esper │ │ └── annotations │ │ └── EsperAnnotationTest.xml └── pom.xml └── pom.xml /README.md: -------------------------------------------------------------------------------- 1 | # jetstream-esper 2 | 3 | Jetstream integration with Esper - CEP Engine. EsperEngine is been implemented as a processor. 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 4.1.0 (2015-07-30) 2 | 3 | * Upgrade pom to jetstream 4.1.0. 4 | * Upgrade esper to 5.2.0. 5 | 6 | # 4.0.2 (2015-02-23) 7 | 8 | eBay opensource Pulsar, first public release 9 | -------------------------------------------------------------------------------- /3rdparty.md: -------------------------------------------------------------------------------- 1 | # External dependencies 2 | 3 | ## GPLV2 4 | 5 | [License link](http://opensource.org/licenses/gpl-2.0.php) 6 | 7 | 1. Esper - http://esper.codehaus.org/about/license/license.html 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # maven target 2 | target 3 | target/* 4 | */target/* 5 | 6 | # intellij files 7 | /.idea 8 | **.iml 9 | **.patch 10 | 11 | # eclipse files 12 | .metadata 13 | .project 14 | .settings 15 | .prefs 16 | .classpath 17 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/BroadCast.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | public @interface BroadCast { 9 | 10 | } 11 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/OutputTo.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | public @interface OutputTo { 9 | 10 | String value() default ""; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/PublishOn.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | public @interface PublishOn { 9 | 10 | String topics() default ""; 11 | String urls() default ""; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/CreateDimension.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | public @interface CreateDimension { 9 | 10 | String name() default ""; 11 | String dimensionspan() default ""; 12 | } 13 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/ClusterAffinityTag.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | public @interface ClusterAffinityTag { 9 | 10 | String colname() default ""; 11 | CreateDimension dimension() default @CreateDimension(); 12 | } 13 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/epl/EntityType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | /** 9 | * The EntityType indicates whether the guid is for an identity in TIS. In case of link we identify it by pair of guids. 10 | */ 11 | public enum EntityType { 12 | IDENTITY, /* GraphType values: */USER, ASSET, TRANSACTION, COMMUNICATION 13 | } -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EventListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import com.espertech.esper.client.UpdateListener; 9 | 10 | /** 11 | * Event Stream Processor listener interface 12 | * 13 | * @author snikolaev 14 | */ 15 | public interface EventListener extends UpdateListener { 16 | } 17 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/EsperTestEventField.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.io.Serializable; 9 | 10 | public class EsperTestEventField implements Serializable { 11 | private static final long serialVersionUID = 1L; 12 | 13 | @Override 14 | public String toString() { 15 | return "This is Esper Test Event Field object"; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/OutputToWithNewStreamName.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | /** 9 | * - Annotation is to change the existing stream name to new name 10 | * @author rmuthupandian 11 | * 12 | */ 13 | public @interface OutputToWithNewStreamName { 14 | 15 | String sinks() default ""; 16 | 17 | String newname() default ""; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/DBInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | public @interface DBInfo { 9 | 10 | public enum WriteMode {INSERT_ONLY, UPSERT, UPDATE_ONLY} 11 | String table() default ""; 12 | String sumfields() default ""; 13 | String minfields() default ""; 14 | String maxfields() default ""; 15 | WriteMode mode() default WriteMode.UPSERT; 16 | 17 | } 18 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/metadata/BroadCastMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.metadata; 7 | 8 | public class BroadCastMetadata { 9 | 10 | private boolean isBroadcast = false; 11 | 12 | public boolean isBroadcast() { 13 | return isBroadcast; 14 | } 15 | 16 | public void setBroadcast(boolean isBroadcast) { 17 | this.isBroadcast = isBroadcast; 18 | } 19 | 20 | 21 | } 22 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/metadata/DBInfoAnnotationMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.metadata; 7 | 8 | import java.util.Map; 9 | 10 | public class DBInfoAnnotationMetadata { 11 | 12 | private Map mapMetaData; 13 | 14 | public Map getMapMetaData() { 15 | return mapMetaData; 16 | } 17 | public void setMapMetaData(Map mapMetaData) { 18 | this.mapMetaData = mapMetaData; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/SendMail.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation; 7 | 8 | 9 | public @interface SendMail { 10 | String alertList() default ""; 11 | String sendFrom() default ""; 12 | String alertSeverity() default "SEVERE"; 13 | String mailServer() default ""; 14 | String eventFields() default ""; 15 | String mailContent() default "Esper query needs your attention!!"; 16 | String mailSubject() default "Notification from Esper query annotation.."; 17 | } 18 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/DataSourceConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | 9 | public class DataSourceConfig { 10 | 11 | private String m_strDsName; 12 | private String m_strEplRef; 13 | 14 | public void setDsName(String strName) { 15 | //m_strDsName = DalJdbcDriver.JDBC_PREFIX + strName; 16 | } 17 | 18 | public void setEplRefName(String strName) { 19 | m_strEplRef = strName; 20 | } 21 | 22 | String getDsName() { 23 | return m_strDsName; 24 | } 25 | 26 | String getEplRefName() { 27 | return m_strEplRef; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/StringEventType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | public class StringEventType extends AbstractEventType { 9 | 10 | private static final long serialVersionUID = 1784746590670293787L; 11 | private String m_strEventClassName; 12 | 13 | public String getEventClassName() { 14 | return m_strEventClassName; 15 | } 16 | 17 | public void setEventClassName(String strEventClassName) { 18 | m_strEventClassName = strEventClassName; 19 | } 20 | 21 | @Override 22 | protected Object getEventDefinition() { 23 | return getEventClassName(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/ClassEventType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | /** 9 | * ClassEvent is used to declare an event that is an instance of an arbitrary java class, that is, a POJO. 10 | * 11 | * @author msikes 12 | */ 13 | public class ClassEventType extends StringEventType { 14 | 15 | private static final long serialVersionUID = 1L; 16 | 17 | @Override 18 | protected Class getEventDefinition() { 19 | Class clazz = null; 20 | try { 21 | clazz = Class.forName(getEventClassName()); 22 | } 23 | catch (ClassNotFoundException e) { 24 | throw new IllegalArgumentException("Unknown class: " + getEventClassName(), e); 25 | } 26 | return clazz; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestAggregationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.raw; 7 | 8 | import com.espertech.esper.client.EventBean; 9 | import com.espertech.esper.client.UpdateListener; 10 | 11 | public class EsperTestAggregationListener implements UpdateListener { 12 | 13 | private int count = 0; 14 | 15 | public EsperTestAggregationListener() { 16 | } 17 | 18 | int getCount() { 19 | return count; 20 | } 21 | 22 | public void update(EventBean[] newEvents, EventBean[] oldEvents) { 23 | EventBean event = newEvents[0]; 24 | count++; 25 | System.out.println("*** event: " + event + " guid: " + event.get("guid") + ", RESULT: " + event.get("RESULT")); //KEEPME 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPTestListenerForStringUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.List; 9 | 10 | import com.espertech.esper.client.EventBean; 11 | 12 | public class ESPTestListenerForStringUtilsTest implements EventListener { 13 | 14 | private int count = 0; 15 | private List topicList; 16 | 17 | int getCount() { 18 | return count; 19 | } 20 | 21 | public List getTopicList() { 22 | return topicList; 23 | } 24 | 25 | @SuppressWarnings("unchecked") 26 | public void update(EventBean[] newEvents, EventBean[] oldEvents) { 27 | count++; 28 | EventBean event = newEvents[0]; 29 | topicList = (List) event.get("TopicList"); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/metadata/AffinityAnnotationMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.metadata; 7 | 8 | import java.util.LinkedList; 9 | import java.util.List; 10 | 11 | import com.ebay.jetstream.event.processor.esper.AffinityKeyGenerator; 12 | 13 | public class AffinityAnnotationMetadata { 14 | 15 | private List keygen = new LinkedList(); 16 | 17 | public void setKeygen(List keygen) { 18 | this.keygen = keygen; 19 | } 20 | 21 | public void addToKeyGenList(AffinityKeyGenerator keygen){ 22 | this.keygen.add(keygen); 23 | } 24 | 25 | public List getAffinityKeyGen(){ 26 | return this.keygen; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/metadata/PublishOnAnnotationMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.metadata; 7 | 8 | public class PublishOnAnnotationMetadata { 9 | 10 | private String urls; 11 | private String topics; 12 | private String[] m_aUrls; 13 | private String[] m_aTopics; 14 | 15 | 16 | public void setUrls(String urls) { 17 | this.urls = urls; 18 | this.m_aUrls = ( urls != null) ? urls.split(",") : null; 19 | 20 | } 21 | public void setTopics(String topics) { 22 | this.topics = topics; 23 | m_aTopics = (topics != null) ? topics.split(",") : null; 24 | } 25 | 26 | public String[] getPublishUrls() { 27 | return m_aUrls; 28 | } 29 | 30 | public String[] getPublishTopics() { 31 | return m_aTopics; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.raw; 7 | 8 | import com.espertech.esper.client.EPAdministrator; 9 | import com.espertech.esper.client.EPStatement; 10 | import com.espertech.esper.client.UpdateListener; 11 | 12 | public class EsperTestStatement { 13 | private EPStatement statement; 14 | 15 | public EsperTestStatement(EPAdministrator admin) { 16 | String stmt = "select id, field1, field2, field3 from EsperTestEvent"; 17 | statement = admin.createEPL(stmt); 18 | } 19 | 20 | public void addListener(UpdateListener listener) { 21 | statement.addListener(listener); 22 | } 23 | 24 | public void setSubscriber (EsperTestSubscriber subscriber) { 25 | statement.setSubscriber(subscriber); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/AbstractEventType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.io.Serializable; 9 | 10 | import com.ebay.jetstream.xmlser.XSerializable; 11 | 12 | /** 13 | * Events need to be declared for Esper processors to recognize them. 14 | * AbstractEventType is a base for declared events. 15 | * 16 | * @author trobison, derived from original work by msikes 17 | */ 18 | public abstract class AbstractEventType implements Serializable, XSerializable { 19 | 20 | private static final long serialVersionUID = -1348054045746281953L; 21 | private String m_strEventAlias; 22 | 23 | public String getEventAlias() { 24 | return m_strEventAlias; 25 | } 26 | 27 | public void setEventAlias(String strEventAlias) { 28 | m_strEventAlias = strEventAlias; 29 | } 30 | 31 | protected abstract Object getEventDefinition(); 32 | } 33 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/config/DefaultAnnotationConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.config; 7 | 8 | import java.util.List; 9 | 10 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 11 | 12 | /** 13 | * - Container class to hold list of AnnotationConfiguration. 14 | * @author rmuthupandian 15 | * 16 | */ 17 | public class DefaultAnnotationConfiguration { 18 | 19 | private List defaultAnnotationConfiguration; 20 | 21 | public List getDefaultAnnotationConfiguration() { 22 | return defaultAnnotationConfiguration; 23 | } 24 | 25 | public void setDefaultAnnotationConfiguration( 26 | List defaultAnnotationConfiguration) { 27 | this.defaultAnnotationConfiguration = defaultAnnotationConfiguration; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/epl/HistogramAggregatorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import java.util.Map; 9 | 10 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 11 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 12 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 13 | 14 | public class HistogramAggregatorFactory implements AggregationFunctionFactory { 15 | 16 | @Override 17 | public void setFunctionName(String functionName) { 18 | // nothing needed here 19 | } 20 | 21 | @Override 22 | public void validate(AggregationValidationContext validationContext) { 23 | // TODO ADD VALIDATAION 24 | } 25 | 26 | @Override 27 | public AggregationMethod newAggregator() { 28 | return new HistogramAggregator(); 29 | } 30 | 31 | @Override 32 | public Class getValueType() { 33 | return Map.class; 34 | } 35 | } 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/metadata/OutputToAnnotationMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.metadata; 7 | 8 | import java.util.LinkedList; 9 | import java.util.List; 10 | 11 | import com.ebay.jetstream.event.EventSink; 12 | 13 | public class OutputToAnnotationMetadata { 14 | 15 | private List sinklist = new LinkedList(); 16 | private String strStreamName = ""; 17 | private EventSink[] sinks; 18 | 19 | public String getStreamName() { 20 | return strStreamName; 21 | } 22 | 23 | public void setStreamName(String strStreamName) { 24 | this.strStreamName = strStreamName; 25 | } 26 | 27 | public void addToSinkList(List theSinks) { 28 | sinklist.addAll(theSinks); 29 | sinks = new EventSink[sinklist.size()]; 30 | sinks = sinklist.toArray(sinks); 31 | } 32 | 33 | public EventSink[] getSinks() { 34 | return sinks; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/metadata/SendMailAnnotationMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.metadata; 7 | 8 | import com.ebay.jetstream.notification.MailConfiguration; 9 | 10 | public class SendMailAnnotationMetadata extends MailConfiguration { 11 | private String eventFields; 12 | private String mailContent; 13 | private String mailSubject; 14 | 15 | public String getEventFields() { 16 | return eventFields; 17 | } 18 | 19 | public void setEventFields(String eventFields) { 20 | this.eventFields = eventFields; 21 | } 22 | 23 | public String getMailSubject() { 24 | return mailSubject; 25 | } 26 | 27 | public void setMailSubject(String mailSubject) { 28 | this.mailSubject = mailSubject; 29 | } 30 | 31 | public String getMailContent() { 32 | return mailContent; 33 | } 34 | 35 | public void setMailContent(String mailContent) { 36 | this.mailContent = mailContent; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/EsperStreamTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.Map; 9 | 10 | import org.junit.Ignore; 11 | 12 | import com.ebay.jetstream.config.ApplicationInformation; 13 | import com.ebay.jetstream.config.Configuration; 14 | import com.ebay.jetstream.config.RootConfiguration; 15 | 16 | /** 17 | * @author msikes 18 | * 19 | */ 20 | public class EsperStreamTest { 21 | private static final Configuration s_springConfiguration = new RootConfiguration(new ApplicationInformation( 22 | "EsperStreamTest", "1.0"), new String[] { Configuration.getClasspathContext(EsperStreamTest.class, null) }); 23 | 24 | public static Map tagEvent(Map event, String name, Object value) { 25 | event.put(name, value); 26 | return event; 27 | } 28 | 29 | @Ignore 30 | public void test() throws Exception { 31 | s_springConfiguration.start(); 32 | Thread.sleep(100000); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPTestAggregationSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import com.ebay.jetstream.event.EventException; 11 | import com.ebay.jetstream.event.EventSink; 12 | import com.ebay.jetstream.event.JetstreamEvent; 13 | 14 | public class ESPTestAggregationSink implements EventSink { 15 | 16 | private int count = 0; 17 | 18 | public String getBeanName() { 19 | // TODO Auto-generated method stub 20 | return null; 21 | } 22 | 23 | int getCount() { 24 | return count; 25 | } 26 | 27 | public void sendEvent(JetstreamEvent event) throws EventException { 28 | 29 | Integer id = (Integer) event.get("field_id"); 30 | Double result = (Double) event.get("AggregatedResult"); 31 | 32 | // AVG of 3 values: id and id^2 and id^3 33 | assertEquals("For field_id=" + id, (id + id * id + id * id * id) / 3., result, 1.e-06); 34 | 35 | count++; 36 | 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/metadata/OutputToNewStreamMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.metadata; 7 | 8 | import java.util.LinkedList; 9 | import java.util.List; 10 | 11 | import com.ebay.jetstream.event.EventSink; 12 | 13 | /** 14 | * - Metadata class holds new name for the stream to change. 15 | * @author rmuthupandian 16 | * 17 | */ 18 | public class OutputToNewStreamMetadata { 19 | 20 | public String name ; 21 | private List sinklist = new LinkedList(); 22 | private EventSink[] sinks; 23 | 24 | public void addToSinkList(List theSinks) { 25 | sinklist.addAll(theSinks); 26 | sinks = new EventSink[sinklist.size()]; 27 | sinks = sinklist.toArray(sinks); 28 | } 29 | 30 | public EventSink[] getSinks() { 31 | return sinks; 32 | } 33 | 34 | public String getStreamName(){ 35 | return this.name; 36 | } 37 | 38 | public void setStreamName(String newName){ 39 | this.name = newName; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestAggregationRunnable.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.raw; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import com.espertech.esper.client.EPServiceProvider; 12 | 13 | public class EsperTestAggregationRunnable implements Runnable { 14 | 15 | private final EPServiceProvider m_engine; 16 | private final int m_id; 17 | 18 | public EsperTestAggregationRunnable(EPServiceProvider engine, int id) { 19 | m_engine = engine; 20 | m_id = id; 21 | } 22 | 23 | public void run() { 24 | Thread thread = Thread.currentThread(); 25 | final long workerId = thread.getId(); 26 | Map map = generateNVP(m_id, workerId); 27 | EsperTest.doSendAggrEvent(m_engine.getEPRuntime(), workerId, map); 28 | } 29 | 30 | private Map generateNVP(int id, long workerId) { 31 | HashMap map = new HashMap(); 32 | map.put("workerId", workerId); 33 | map.put("guid", id); 34 | return map; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EsperEngineAnnotationMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import com.ebay.jestream.event.annotation.EngineMetadata; 13 | import com.espertech.esper.client.soda.AnnotationPart; 14 | import com.espertech.esper.client.soda.EPStatementObjectModel; 15 | 16 | public class EsperEngineAnnotationMetadata implements EngineMetadata { 17 | 18 | private Map> partsMap = new HashMap>(); 19 | private EPStatementObjectModel model ; 20 | 21 | public void setAnnotationPartsMap(Map> annotPartsMap){ 22 | this.partsMap = annotPartsMap; 23 | } 24 | 25 | public Map> getAnnotationPartsMap(){ 26 | return partsMap; 27 | } 28 | 29 | public void setStatementObjectModel(EPStatementObjectModel model){ 30 | this.model = model; 31 | } 32 | 33 | public EPStatementObjectModel getStatementObjectModel(){ 34 | return model; 35 | } 36 | 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPTestSubscriber.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.List; 9 | import java.util.SortedSet; 10 | import java.util.TreeSet; 11 | 12 | import org.apache.commons.logging.Log; 13 | import org.apache.commons.logging.LogFactory; 14 | 15 | import com.ebay.jetstream.event.processor.esper.raw.EsperTestListener; 16 | 17 | public class ESPTestSubscriber { 18 | private static final Log testLogger = LogFactory.getLog(EsperTestListener.class); 19 | private final SortedSet ids = new TreeSet(); 20 | private int count = 0; 21 | 22 | public ESPTestSubscriber() { 23 | testLogger.info("subscribed"); 24 | } 25 | 26 | public int getCount() { 27 | return count; 28 | } 29 | 30 | public SortedSet getIds() { 31 | return ids; 32 | } 33 | 34 | public synchronized void update(Integer id, Double field1, List field2, EsperTestEventField field3) { 35 | count++; 36 | if (id == null) { 37 | testLogger.error("Id is null!"); 38 | } 39 | else { 40 | ids.add(id); 41 | } 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/esper/customaggregate/MyConcatAggregationFunctionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.esper.customaggregate; 7 | 8 | 9 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 10 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 11 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 12 | 13 | 14 | public class MyConcatAggregationFunctionFactory implements AggregationFunctionFactory { 15 | 16 | public void validate(AggregationValidationContext validationContext) { 17 | if ((validationContext.getParameterTypes().length != 1) || 18 | (validationContext.getParameterTypes()[0] != String.class)) { 19 | throw new IllegalArgumentException("Concat aggregation requires a single parameter of type String"); 20 | } 21 | } 22 | 23 | public Class getValueType() { 24 | return String.class; 25 | } 26 | 27 | public void setFunctionName(String functionName) { 28 | // not required here 29 | } 30 | 31 | public AggregationMethod newAggregator() { 32 | return new MyConcatAggregationFunction(); 33 | } 34 | } -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/esper/customaggregate/MySecondConcatAggregationFunctionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.esper.customaggregate; 7 | 8 | 9 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 10 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 11 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 12 | 13 | public class MySecondConcatAggregationFunctionFactory implements AggregationFunctionFactory { 14 | 15 | public void validate(AggregationValidationContext validationContext) { 16 | if ((validationContext.getParameterTypes().length != 1) || 17 | (validationContext.getParameterTypes()[0] != String.class)) { 18 | throw new IllegalArgumentException("Concat aggregation requires a single parameter of type String"); 19 | } 20 | } 21 | 22 | public Class getValueType() { 23 | return String.class; 24 | } 25 | 26 | public void setFunctionName(String functionName) { 27 | // not required here 28 | } 29 | 30 | public AggregationMethod newAggregator() { 31 | return new MySecondConcatAggregationFunction(); 32 | } 33 | } -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestSubscriber.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.raw; 7 | 8 | import java.util.List; 9 | import java.util.SortedSet; 10 | import java.util.TreeSet; 11 | 12 | import org.apache.commons.logging.Log; 13 | import org.apache.commons.logging.LogFactory; 14 | 15 | import com.ebay.jetstream.event.processor.esper.EsperTestEventField; 16 | 17 | public class EsperTestSubscriber { 18 | 19 | private static final Log log = LogFactory.getLog(EsperTestSubscriber.class); 20 | private static SortedSet ids = new TreeSet(); 21 | private int count = 0; 22 | 23 | public int getCount() { 24 | return count; 25 | } 26 | 27 | public SortedSet getIds() { 28 | return ids; 29 | } 30 | 31 | // yes, unlike listener's update this one has to be synchronized 32 | // TODO check this for future Esper versions 33 | public synchronized void update(Integer id, String field1, List field2, EsperTestEventField field3) { 34 | count++; 35 | if (id == null) { 36 | log.error("Id is null!"); 37 | } 38 | else { 39 | ids.add(id); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPTestAggregationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import com.espertech.esper.client.EventBean; 11 | 12 | public class ESPTestAggregationListener implements EventListener { 13 | 14 | private int count = 0; 15 | private int newCount = 0; 16 | private int oldCount = 0; 17 | 18 | int getCount() { 19 | return count; 20 | } 21 | 22 | int getNewCount() { 23 | return newCount; 24 | } 25 | 26 | int getOldCount() { 27 | return oldCount; 28 | } 29 | 30 | public void update(EventBean[] newEvents, EventBean[] oldEvents) { 31 | EventBean event = newEvents[0]; 32 | Integer id = (Integer) event.get("field_id"); 33 | Double result = (Double) event.get("AggregatedResult"); 34 | 35 | // AVG of 3 values: id and id^2 and id^3 36 | assertEquals("For field_id=" + id, (id + id * id + id * id * id) / 3., result, 1.e-06); 37 | 38 | count++; 39 | if (newEvents != null) 40 | newCount += newEvents.length; 41 | if (oldEvents != null) 42 | oldCount += oldEvents.length; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPTestSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.SortedSet; 9 | import java.util.TreeSet; 10 | 11 | import org.apache.commons.logging.Log; 12 | import org.apache.commons.logging.LogFactory; 13 | 14 | import com.ebay.jetstream.event.EventException; 15 | import com.ebay.jetstream.event.EventSink; 16 | import com.ebay.jetstream.event.JetstreamEvent; 17 | 18 | public class ESPTestSink implements EventSink { 19 | private static final Log testLogger = LogFactory.getLog(ESPTestSink.class); 20 | 21 | private final SortedSet ids = new TreeSet(); 22 | private int count = 0; 23 | 24 | public String getBeanName() { 25 | // TODO Auto-generated method stub 26 | return null; 27 | } 28 | 29 | public int getCount() { 30 | return count; 31 | } 32 | 33 | public SortedSet getIds() { 34 | return ids; 35 | } 36 | 37 | public void sendEvent(JetstreamEvent event) throws EventException { 38 | count++; 39 | Integer id = (Integer) event.get("id"); 40 | if (id != null) { 41 | ids.add(id); 42 | } 43 | testLogger.debug("id " + id); 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/aggregates/CardinalityAggregator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.aggregates; 7 | 8 | import com.clearspring.analytics.stream.cardinality.HyperLogLog; 9 | import com.clearspring.analytics.stream.cardinality.ICardinality; 10 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 11 | 12 | /** 13 | * @author shmurthy@ebay.com Esper aggregation method for computing 14 | * the cardinality of elements provided to this aggregator 15 | * 16 | * Usage: select cardinality(element) as uniqueElement from stream 17 | */ 18 | 19 | public class CardinalityAggregator implements AggregationMethod { 20 | 21 | ICardinality m_card = new HyperLogLog(24); 22 | 23 | @Override 24 | public void enter(Object value) { 25 | 26 | m_card.offer(value); 27 | 28 | } 29 | 30 | @Override 31 | public void leave(Object value) { 32 | // nothing to do 33 | 34 | } 35 | 36 | @Override 37 | public Object getValue() { 38 | 39 | return m_card.cardinality(); 40 | } 41 | 42 | @Override 43 | public Class getValueType() { 44 | 45 | return Long.class; 46 | } 47 | 48 | @Override 49 | public void clear() { 50 | } 51 | 52 | 53 | } 54 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/esper/customaggregate/MyConcatAggregationFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.esper.customaggregate; 7 | 8 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 9 | 10 | public class MyConcatAggregationFunction implements AggregationMethod { 11 | 12 | private static char DELIMITER = '1'; 13 | private StringBuilder builder; 14 | private String delimiter; 15 | 16 | public MyConcatAggregationFunction() { 17 | builder = new StringBuilder(); 18 | delimiter = ""; 19 | } 20 | 21 | @Override 22 | public void enter(Object value) { 23 | if (value != null) { 24 | builder.append(delimiter); 25 | builder.append(value.toString()); 26 | delimiter = Character.toString(DELIMITER); 27 | } 28 | } 29 | 30 | @Override 31 | public void leave(Object value) { 32 | if (value != null) { 33 | builder.setLength(0); 34 | } 35 | } 36 | 37 | @Override 38 | public Object getValue() { 39 | return builder.toString(); 40 | } 41 | 42 | @Override 43 | public Class getValueType() { 44 | return String.class; 45 | } 46 | 47 | @Override 48 | public void clear() { 49 | builder = new StringBuilder(); 50 | delimiter = ""; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EsperDeclaredEvents.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.io.Serializable; 9 | import java.util.ArrayList; 10 | import java.util.Collection; 11 | 12 | import com.ebay.jetstream.config.AbstractNamedBean; 13 | import com.ebay.jetstream.xmlser.XSerializable; 14 | 15 | /** 16 | * @author trobison, derived from original work of msikes 17 | */ 18 | public final class EsperDeclaredEvents extends AbstractNamedBean implements Serializable, XSerializable { 19 | 20 | private static final long serialVersionUID = -8107344168985484754L; 21 | private final Collection m_listEventTypes = new ArrayList(); 22 | 23 | /** 24 | * Gets all registered Event Types with their fields 25 | * 26 | * @return the collection of event types 27 | */ 28 | @SuppressWarnings("unchecked") 29 | public Collection getEventTypes() { 30 | return (Collection)((ArrayList)m_listEventTypes).clone(); 31 | } 32 | 33 | public void setEventTypes(Collection eventTypes) { 34 | m_listEventTypes.clear(); 35 | m_listEventTypes.addAll(eventTypes); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EsperEventConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import com.ebay.jetstream.event.EventException; 9 | import com.ebay.jetstream.event.JetstreamEvent; 10 | import com.espertech.esper.client.EventBean; 11 | 12 | /** 13 | * This interface allows pluggable Esper EventBean to JetstreamEvent conversion. Esper always sends events as EventBeans, 14 | * and supports several types of beans. Jetstream expects JetstreamEvents always, which is an extension of Map. 16 | * 17 | * @author msikes 18 | */ 19 | public interface EsperEventConverter { 20 | 21 | /** 22 | * Convert from an EventBean to a JetstreamEvent. If the event bean isn't supported, null may be returned for the event 23 | * to be ignored: sub-classes may then attempt their own conversions. 24 | * 25 | * @param bean 26 | * the event from Esper to be converted. 27 | * @return the event, converted to a JetstreamEvent. 28 | * @throws EventException 29 | * if the event cannot be converted but the event should not be ignored. An exception is logged. 30 | */ 31 | JetstreamEvent getJetstreamEvent(EventBean bean) throws EventException; 32 | } 33 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/esper/customaggregate/MySecondConcatAggregationFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.esper.customaggregate; 7 | 8 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 9 | 10 | public class MySecondConcatAggregationFunction implements AggregationMethod { 11 | 12 | private static char DELIMITER = '2'; 13 | private StringBuilder builder; 14 | private String delimiter; 15 | 16 | public MySecondConcatAggregationFunction() { 17 | builder = new StringBuilder(); 18 | delimiter = ""; 19 | } 20 | 21 | @Override 22 | public void enter(Object value) { 23 | if (value != null) { 24 | builder.append(delimiter); 25 | builder.append(value.toString()); 26 | delimiter = Character.toString(DELIMITER); 27 | } 28 | } 29 | 30 | @Override 31 | public void leave(Object value) { 32 | if (value != null) { 33 | builder.setLength(0); 34 | } 35 | } 36 | 37 | @Override 38 | public Object getValue() { 39 | return builder.toString(); 40 | } 41 | 42 | @Override 43 | public Class getValueType() { 44 | return String.class; 45 | } 46 | 47 | @Override 48 | public void clear() { 49 | builder = new StringBuilder(); 50 | delimiter = ""; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPTestRunnable.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.ArrayList; 9 | import java.util.HashMap; 10 | 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | 13 | public class ESPTestRunnable implements Runnable { 14 | 15 | private final EsperProcessor m_processor; 16 | private final int m_id; 17 | 18 | public ESPTestRunnable(EsperProcessor processor, int id) { 19 | m_processor = processor; 20 | m_id = id; 21 | } 22 | 23 | public void run() { 24 | JetstreamEvent event = generateNVP(); 25 | event.setEventType("ESPTestEvent1"); 26 | m_processor.sendEvent(event); 27 | } 28 | 29 | private JetstreamEvent generateNVP() { 30 | HashMap map = new HashMap(); 31 | String field1 = "456.789"; 32 | ArrayList field2 = new ArrayList(); 33 | EsperTestEventField field3 = new EsperTestEventField(); 34 | field2.add("Field 2, value 1"); 35 | field2.add("Field 2, value 2"); 36 | map.put("id", m_id); 37 | map.put("field1", field1); 38 | map.put("field2", field2); 39 | map.put("field3", field3); 40 | return new JetstreamEvent(map); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/ProcessEventRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | import com.ebay.jetstream.event.MonitorableStatCollector; 13 | import com.ebay.jetstream.event.processor.EventProcessRequest; 14 | import com.ebay.jetstream.event.processor.esper.EsperProcessor.EsperInternals; 15 | 16 | /** 17 | * @author shmurthy@ebay.com - worker request to execute DB TXN 18 | */ 19 | public class ProcessEventRequest extends EventProcessRequest { 20 | 21 | private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.jetstream.event.processor.esper"); 22 | private EsperInternals m_esperInternals; 23 | 24 | /** 25 | */ 26 | public ProcessEventRequest(EsperProcessor.EsperInternals esp, JetstreamEvent event, 27 | MonitorableStatCollector parent) { 28 | super(event, parent); 29 | m_esperInternals = esp; 30 | } 31 | 32 | @Override 33 | protected void processEvent(JetstreamEvent event) throws Exception { 34 | m_esperInternals.processEvent(event); 35 | } 36 | 37 | protected EsperInternals getEsperInternals() { 38 | return m_esperInternals; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/listener/BroadCastAnnotationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.listener; 7 | 8 | import com.ebay.jestream.event.annotation.AnnotationListener; 9 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 10 | import com.ebay.jetstream.event.JetstreamEvent; 11 | import com.ebay.jetstream.event.JetstreamReservedKeys; 12 | import com.ebay.jetstream.event.processor.esper.annotation.BroadCast; 13 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.BroadCastMetadata; 14 | 15 | public class BroadCastAnnotationListener implements AnnotationListener{ 16 | 17 | private static final String ANNO_KEY = BroadCast.class.getSimpleName(); 18 | 19 | @Override 20 | public JetstreamEvent processMetaInformation(JetstreamEvent event, 21 | StatementAnnotationInfo annotationInfo) { 22 | 23 | BroadCastMetadata metadata = (BroadCastMetadata) annotationInfo.getAnnotationInfo(ANNO_KEY); 24 | if(metadata != null && metadata.isBroadcast()) 25 | event.addMetaData(JetstreamReservedKeys.EventBroadCast.toString(), String.valueOf(true)); 26 | 27 | return event; 28 | } 29 | 30 | @Override 31 | public Class getAnnotationClass() { 32 | return BroadCast.class; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/listener/DBInfoAnnotationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.listener; 7 | 8 | import com.ebay.jestream.event.annotation.AnnotationListener; 9 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 10 | import com.ebay.jetstream.event.JetstreamEvent; 11 | import com.ebay.jetstream.event.JetstreamReservedKeys; 12 | import com.ebay.jetstream.event.processor.esper.annotation.DBInfo; 13 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.DBInfoAnnotationMetadata; 14 | 15 | public class DBInfoAnnotationListener implements AnnotationListener { 16 | private static final String ANNO_KEY = DBInfo.class.getSimpleName(); 17 | 18 | @Override 19 | public JetstreamEvent processMetaInformation(JetstreamEvent event, 20 | StatementAnnotationInfo annotationInfo) { 21 | 22 | DBInfoAnnotationMetadata anntmetadata = (DBInfoAnnotationMetadata) annotationInfo.getAnnotationInfo(ANNO_KEY); 23 | if (anntmetadata != null){ 24 | event.put(JetstreamReservedKeys.EventMetaData.toString(), anntmetadata.getMapMetaData()); 25 | } 26 | 27 | return event; 28 | } 29 | 30 | @Override 31 | public Class getAnnotationClass() { 32 | return DBInfo.class; 33 | } 34 | 35 | 36 | 37 | } 38 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/listener/AffinityAnnotationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.listener; 7 | 8 | import com.ebay.jestream.event.annotation.AnnotationListener; 9 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 10 | import com.ebay.jetstream.event.JetstreamEvent; 11 | import com.ebay.jetstream.event.processor.esper.AffinityKeyGenerator; 12 | import com.ebay.jetstream.event.processor.esper.annotation.ClusterAffinityTag; 13 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.AffinityAnnotationMetadata; 14 | 15 | public class AffinityAnnotationListener implements AnnotationListener { 16 | private static final String ANNO_KEY = ClusterAffinityTag.class.getSimpleName(); 17 | 18 | @Override 19 | public JetstreamEvent processMetaInformation(JetstreamEvent event, 20 | StatementAnnotationInfo annotationInfo) { 21 | 22 | AffinityAnnotationMetadata metadata = (AffinityAnnotationMetadata) annotationInfo 23 | .getAnnotationInfo(ANNO_KEY); 24 | 25 | for (AffinityKeyGenerator gen : metadata.getAffinityKeyGen()) 26 | gen.setEventFields(event); 27 | 28 | return event; 29 | } 30 | 31 | @Override 32 | public Class getAnnotationClass() { 33 | return ClusterAffinityTag.class; 34 | } 35 | 36 | 37 | 38 | } 39 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/ProcessDestroyEngineRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | import com.ebay.jetstream.event.MonitorableStatCollector; 13 | import com.ebay.jetstream.event.processor.esper.EsperProcessor.EsperInternals; 14 | 15 | 16 | public class ProcessDestroyEngineRequest extends ProcessEventRequest { 17 | 18 | private static Logger logger = LoggerFactory.getLogger("com.ebay.jetstream.event.processor.esper.EsperProcessor"); 19 | 20 | 21 | public ProcessDestroyEngineRequest(EsperInternals esp) { 22 | super(esp, null, null); 23 | } 24 | 25 | @Override 26 | protected void processEvent(JetstreamEvent event) throws Exception { 27 | getEsperInternals().getEsperService().getEPRuntime().getEventSender("EsperEndEvent").sendEvent(new Object()); 28 | try { 29 | Thread.sleep(100); 30 | }catch (InterruptedException e) { 31 | logger.error( e.getLocalizedMessage() , e); 32 | } 33 | getEsperInternals().getEsperService().destroy(); 34 | getEsperInternals().clear(); 35 | } 36 | 37 | protected void afterEventProcessed(JetstreamEvent event, MonitorableStatCollector stats) { 38 | // do nothing but log 39 | logger.info( "Destroyed stale Esper engine"); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/listener/OutputToAnnotationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.listener; 7 | 8 | import com.ebay.jestream.event.annotation.AnnotationListener; 9 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 10 | import com.ebay.jetstream.event.EventSink; 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | import com.ebay.jetstream.event.processor.esper.annotation.OutputTo; 13 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.OutputToAnnotationMetadata; 14 | 15 | public class OutputToAnnotationListener implements AnnotationListener { 16 | private static final String ANNO_KEY = OutputTo.class.getSimpleName(); 17 | 18 | @Override 19 | public JetstreamEvent processMetaInformation(JetstreamEvent event, 20 | StatementAnnotationInfo annotationInfo) { 21 | 22 | OutputToAnnotationMetadata anntmetadata = (OutputToAnnotationMetadata) annotationInfo.getAnnotationInfo(ANNO_KEY); 23 | if (anntmetadata.getStreamName() != null) 24 | event.setEventType(anntmetadata.getStreamName()); 25 | 26 | EventSink[] sinks = anntmetadata.getSinks(); 27 | if (sinks != null) 28 | event.put("sink.array", sinks); 29 | 30 | return event; 31 | 32 | } 33 | 34 | @Override 35 | public Class getAnnotationClass() { 36 | return OutputTo.class; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestAggregationStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.raw; 7 | 8 | import com.espertech.esper.client.EPAdministrator; 9 | import com.espertech.esper.client.EPStatement; 10 | import com.espertech.esper.client.UpdateListener; 11 | 12 | public class EsperTestAggregationStatement { 13 | 14 | private final EPStatement statement; 15 | 16 | public EsperTestAggregationStatement(EPAdministrator admin) { 17 | admin.createEPL("INSERT INTO S SELECT 1.1+guid AS V, workerId, guid FROM EsperTestAggregationEvent"); 18 | admin.createEPL("INSERT INTO S SELECT 2.2+guid AS V, workerId, guid FROM EsperTestAggregationEvent"); 19 | admin.createEPL("INSERT INTO S SELECT 3.3+guid AS V, workerId, guid FROM EsperTestAggregationEvent"); 20 | admin.createEPL("INSERT INTO S SELECT 4.4+guid AS V, workerId, guid FROM EsperTestAggregationEvent"); 21 | admin.createEPL("CREATE WINDOW SW.win:keepall() AS (V double, workerId long, guid int)"); 22 | admin.createEPL("INSERT INTO SW SELECT V, workerId, guid FROM S"); 23 | admin.createEPL("ON CleanupWindowEvent DELETE FROM SW WHERE SW.workerId = CleanupWindowEvent.workerId"); 24 | statement = admin.createEPL("SELECT * FROM OutputEvent"); 25 | } 26 | 27 | public void addListener(UpdateListener listener) { 28 | statement.addListener(listener); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/JetstreamExceptionHandlerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.Map; 9 | 10 | import com.ebay.jetstream.config.RootConfiguration; 11 | import com.espertech.esper.client.hook.ExceptionHandler; 12 | import com.espertech.esper.client.hook.ExceptionHandlerFactory; 13 | import com.espertech.esper.client.hook.ExceptionHandlerFactoryContext; 14 | 15 | /** 16 | * This factory class serves Exceptionhandler instance to the Engine to capture 17 | * RuntimeExceptions from EPL processing. Since multiple EsperPorcessors are 18 | * available, all EsperProcessors are fetched from RootConfiguration. Each 19 | * EsperProcessor holds ExceptionHandler. 20 | * 21 | * @author rmuthupandian 22 | * 23 | */ 24 | 25 | public class JetstreamExceptionHandlerFactory implements 26 | ExceptionHandlerFactory { 27 | 28 | Map esperprocessors = RootConfiguration 29 | .getConfiguration().getBeansOfType(EsperProcessor.class); 30 | 31 | @Override 32 | public ExceptionHandler getHandler(ExceptionHandlerFactoryContext context) { 33 | for (EsperProcessor proc : esperprocessors.values()) { 34 | if (proc.getEngineURI() != null 35 | && proc.getEngineURI().equals(context.getEngineURI())) { 36 | return proc.getEsperExceptionHandler(); 37 | } 38 | } 39 | return null; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/listener/PublishOnAnnotationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.listener; 7 | 8 | import com.ebay.jestream.event.annotation.AnnotationListener; 9 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 10 | import com.ebay.jetstream.event.JetstreamEvent; 11 | import com.ebay.jetstream.event.processor.esper.annotation.PublishOn; 12 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.PublishOnAnnotationMetadata; 13 | 14 | public class PublishOnAnnotationListener implements AnnotationListener { 15 | private static final String ANNO_KEY = PublishOn.class.getSimpleName(); 16 | 17 | @Override 18 | public JetstreamEvent processMetaInformation(JetstreamEvent event, 19 | StatementAnnotationInfo annotationInfo) { 20 | 21 | PublishOnAnnotationMetadata anntmetadata = (PublishOnAnnotationMetadata) annotationInfo 22 | .getAnnotationInfo(ANNO_KEY); 23 | 24 | String[] aPublishTopics = anntmetadata.getPublishTopics(); 25 | if (aPublishTopics != null) 26 | event.setForwardingTopics(aPublishTopics); 27 | 28 | String[] aPublishUrls = anntmetadata.getPublishUrls(); 29 | if (aPublishUrls != null) 30 | event.setForwardingUrls(aPublishUrls); 31 | 32 | return event; 33 | } 34 | 35 | @Override 36 | public Class getAnnotationClass() { 37 | return PublishOn.class; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/aggregates/CardinalityAggregatorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.aggregates; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 12 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 13 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 14 | 15 | /** 16 | * @author shmurthy@ebay.com Esper aggregation Function factory for 17 | * CardinalityAggregator 18 | * 19 | * 20 | */ 21 | 22 | public class CardinalityAggregatorFactory implements AggregationFunctionFactory { 23 | 24 | private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.jetstream.event.processor.esper.aggregates"); 25 | 26 | 27 | @Override 28 | public void setFunctionName(String functionName) { 29 | 30 | } 31 | 32 | @Override 33 | public void validate(AggregationValidationContext validationContext) { 34 | if (validationContext.getParameterTypes().length != 1) 35 | LOGGER.error( "Cardinality Aggregation Function requires 1 parameter of type Double)"); 36 | 37 | } 38 | 39 | @Override 40 | public AggregationMethod newAggregator() { 41 | 42 | return new CardinalityAggregator(); 43 | } 44 | 45 | @Override 46 | public Class getValueType() { 47 | 48 | return Long.class; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.raw; 7 | 8 | import java.util.SortedSet; 9 | import java.util.TreeSet; 10 | 11 | import org.apache.commons.logging.Log; 12 | import org.apache.commons.logging.LogFactory; 13 | 14 | import com.espertech.esper.client.EventBean; 15 | import com.espertech.esper.client.UpdateListener; 16 | 17 | public class EsperTestListener implements UpdateListener { 18 | 19 | private static final Log log = LogFactory.getLog(EsperTestListener.class); 20 | 21 | private final SortedSet ids = new TreeSet(); 22 | private int count = 0; 23 | private int newCount = 0; 24 | private int oldCount = 0; 25 | 26 | int getCount() { 27 | return count; 28 | } 29 | 30 | public SortedSet getIds() { 31 | return ids; 32 | } 33 | 34 | int getNewCount() { 35 | return newCount; 36 | } 37 | 38 | int getOldCount() { 39 | return oldCount; 40 | } 41 | 42 | public void update(EventBean[] newEvents, EventBean[] oldEvents) { 43 | EventBean event = newEvents[0]; 44 | Integer id = (Integer) event.get("id"); 45 | 46 | ids.add(id); 47 | log.debug("id " + id); 48 | 49 | count++; 50 | if (newEvents != null) 51 | newCount += newEvents.length; 52 | if (oldEvents != null) 53 | oldCount += oldEvents.length; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestConfig.xml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/aggregates/QuantileAggregator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.aggregates; 7 | 8 | import com.clearspring.analytics.stream.quantile.TDigest; 9 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 10 | 11 | /** 12 | * @author shmurthy@ebay.com Esper aggregation method for computing 13 | * the Quantile of element of type double provided to this aggregator 14 | * 15 | * Usage: select percentile(quantile, element) as percentile from stream 16 | * quantile must be between 0 - 1. 17 | */ 18 | 19 | public class QuantileAggregator implements AggregationMethod { 20 | 21 | TDigest m_digest; 22 | double m_quantile; 23 | double m_compression = 10.0; 24 | 25 | @Override 26 | public void enter(Object value) { 27 | 28 | Object[] params = (Object[]) value; 29 | 30 | if(m_digest == null) { 31 | m_quantile = (Double) params[0]; 32 | m_digest = new TDigest(m_compression); 33 | } 34 | 35 | m_digest.add((Double) params[1]); 36 | 37 | } 38 | 39 | @Override 40 | public void leave(Object value) { 41 | // nothing to do 42 | } 43 | 44 | @Override 45 | public Object getValue() { 46 | 47 | return m_digest.quantile(m_quantile); 48 | } 49 | 50 | @Override 51 | public Class getValueType() { 52 | 53 | return Double.class; 54 | } 55 | 56 | @Override 57 | public void clear() { 58 | m_digest = null; 59 | 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/aggregates/TopKAggregatorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.aggregates; 7 | 8 | import java.util.Map; 9 | 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 14 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 15 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 16 | 17 | /** 18 | * @author shmurthy@ebay.com Esper aggregation Function factory for 19 | * TopKStringAggregator 20 | * 21 | * 22 | */ 23 | 24 | public class TopKAggregatorFactory implements AggregationFunctionFactory { 25 | 26 | private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.jetstream.event.processor.esper.aggregates"); 27 | 28 | @Override 29 | public void setFunctionName(String functionName) { 30 | // nothing needed here 31 | } 32 | 33 | // @Override 34 | public void validate(AggregationValidationContext validationContext) { 35 | if (validationContext.getParameterTypes().length != 3) 36 | LOGGER.error( "TopK Aggregation Function requires 3 parameters viz. max capacity, # top elements and element - topN(maxCapacit, topElements, element)"); 37 | } 38 | 39 | @Override 40 | public AggregationMethod newAggregator() { 41 | 42 | return new TopKAggregator(); 43 | } 44 | 45 | @Override 46 | public Class getValueType() { 47 | 48 | return Map.class; 49 | 50 | } 51 | 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/aggregates/QuantileAggregatorFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.aggregates; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 12 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 13 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 14 | 15 | /** 16 | * @author shmurthy@ebay.com Esper aggregation Function factory for 17 | * QuantileAggregator 18 | * 19 | * 20 | */ 21 | 22 | public class QuantileAggregatorFactory implements AggregationFunctionFactory { 23 | private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.jetstream.event.processor.esper.aggregates"); 24 | 25 | @Override 26 | public void setFunctionName(String functionName) { 27 | // TODO Auto-generated method stub 28 | 29 | } 30 | 31 | @Override 32 | public void validate(AggregationValidationContext validationContext) { 33 | if ((validationContext.getParameterTypes().length != 1) && 34 | (validationContext.getParameterTypes()[0] != Double.class)) 35 | LOGGER.error( "Invalid Type of Input : Quantile Aggregation Function requires 1 parameter of type Double)"); 36 | 37 | } 38 | 39 | @Override 40 | public AggregationMethod newAggregator() { 41 | 42 | return new QuantileAggregator(); 43 | } 44 | 45 | @Override 46 | public Class getValueType() { 47 | 48 | return Double.class; 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/listener/OutputToNewStreamListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.listener; 7 | 8 | /** 9 | * - Listener class gets new stream name from ChangeStreamNameMetadata and injects into JetstreamEvent 10 | * @author rmuthupandian 11 | */ 12 | import com.ebay.jestream.event.annotation.AnnotationListener; 13 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 14 | import com.ebay.jetstream.event.EventSink; 15 | import com.ebay.jetstream.event.JetstreamEvent; 16 | import com.ebay.jetstream.event.processor.esper.annotation.OutputToWithNewStreamName; 17 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.OutputToNewStreamMetadata; 18 | 19 | public class OutputToNewStreamListener implements AnnotationListener { 20 | 21 | private static final String ANNO_KEY = OutputToWithNewStreamName.class.getSimpleName(); 22 | 23 | @Override 24 | public JetstreamEvent processMetaInformation(JetstreamEvent event, 25 | StatementAnnotationInfo annotationInfo) { 26 | 27 | OutputToNewStreamMetadata anntmetadata = (OutputToNewStreamMetadata) annotationInfo 28 | .getAnnotationInfo(ANNO_KEY); 29 | 30 | String newEventType = anntmetadata.getStreamName(); 31 | 32 | event.setEventType(newEventType); 33 | 34 | EventSink[] sinks = anntmetadata.getSinks(); 35 | if (sinks != null) 36 | event.put("sink.array", sinks); 37 | 38 | return event; 39 | } 40 | 41 | @Override 42 | public Class getAnnotationClass() { 43 | return OutputToWithNewStreamName.class; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/raw/EsperTestRunnable.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.raw; 7 | 8 | import java.util.ArrayList; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | import com.ebay.jetstream.event.processor.esper.EsperTestEventField; 13 | import com.espertech.esper.client.EPRuntime; 14 | import com.espertech.esper.client.EPServiceProvider; 15 | import com.espertech.esper.client.time.CurrentTimeEvent; 16 | 17 | public class EsperTestRunnable implements Runnable { 18 | private final EPServiceProvider m_engine; 19 | private final int m_id; 20 | 21 | public EsperTestRunnable(EPServiceProvider engine, int id) { 22 | m_engine = engine; 23 | m_id = id; 24 | } 25 | 26 | public void run() { 27 | Map map = generateNVP(m_id); 28 | EPRuntime runtime = m_engine.getEPRuntime(); 29 | runtime.sendEvent(new CurrentTimeEvent(System.nanoTime() / 1000)); 30 | runtime.sendEvent(map, "EsperTestEvent"); 31 | } 32 | 33 | private Map generateNVP(int id) { 34 | HashMap map = new HashMap(); 35 | String field1 = "Field 1 value"; 36 | ArrayList field2 = new ArrayList(); 37 | EsperTestEventField field3 = new EsperTestEventField(); 38 | field2.add("Field 2, value 1"); 39 | field2.add("Field 2, value 2"); 40 | map.put("id", id); 41 | map.put("field0", new Double(id)); 42 | map.put("field1", field1); 43 | map.put("field2", field2); 44 | map.put("field3", field3); 45 | return map; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPTestListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.HashMap; 9 | import java.util.SortedSet; 10 | import java.util.TreeSet; 11 | 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import com.espertech.esper.client.EventBean; 16 | 17 | public class ESPTestListener implements EventListener { 18 | 19 | private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.jetstream.config"); 20 | 21 | private final SortedSet ids = new TreeSet(); 22 | private int count = 0; 23 | private int newCount = 0; 24 | private int oldCount = 0; 25 | 26 | int getCount() { 27 | return count; 28 | } 29 | 30 | public SortedSet getIds() { 31 | return ids; 32 | } 33 | 34 | int getNewCount() { 35 | return newCount; 36 | } 37 | 38 | int getOldCount() { 39 | return oldCount; 40 | } 41 | 42 | @SuppressWarnings("unchecked") 43 | public void update(EventBean[] newEvents, EventBean[] oldEvents) { 44 | EventBean event = newEvents[0]; 45 | Integer id = (Integer) event.get("id"); 46 | ids.add(id); 47 | 48 | LOGGER.info( "**Listener id " + id); 49 | 50 | count++; 51 | if (newEvents != null) 52 | newCount += newEvents.length; 53 | if (oldEvents != null) 54 | oldCount += oldEvents.length; 55 | 56 | HashMap hm = (HashMap) event.getUnderlying(); 57 | if (!(hm.get("field1") instanceof Double)) 58 | throw new IllegalStateException("expecting a Double"); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | com.ebay.jetstream 6 | jetstream-parent 7 | 4.1.1-SNAPSHOT 8 | 9 | 10 | com.ebay.jetstream 11 | jetstream-esper-root 12 | 4.1.1-SNAPSHOT 13 | 4.0.0 14 | pom 15 | jetstream-esper-root 16 | Jetstream root POM 17 | 18 | 19 | 20 | GNU GENERAL PUBLIC LICENSE Version 2, June 1991 21 | http://www.gnu.org/licenses/gpl-2.0.txt 22 | repo 23 | 24 | 25 | 26 | 27 | jetstream-esper 28 | 29 | 30 | 31 | scm:git:git@github.com:pulsarIO/jetstream-esper.git 32 | scm:git:git@github.com:pulsarIO/jetstream-esper.git 33 | scm:git:git@github.com:pulsarIO/jetstream-esper.git 34 | HEAD 35 | 36 | 37 | 38 | 39 | ossrh 40 | https://oss.sonatype.org/content/repositories/snapshots 41 | 42 | 43 | ossrh 44 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/epl/HistogramAggregator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 12 | 13 | public class HistogramAggregator implements AggregationMethod { 14 | 15 | private String[] m_fields; 16 | private Map[] m_maps; 17 | 18 | @Override 19 | public void enter(Object value) { 20 | Object[] aParts = (Object[])value; 21 | if (m_maps == null) { 22 | m_fields = new String[aParts.length / 2]; 23 | m_maps = new Map[aParts.length / 2]; 24 | for (int i = 0, j = 0; i < m_fields.length; i++, j++) { 25 | m_maps[i] = new HashMap(); 26 | m_fields[i] = (String)aParts[i + j]; 27 | } 28 | } 29 | 30 | for (int i = 0, j = 1; i < m_maps.length; i++, j++) { 31 | Object objValue = aParts[i + j]; 32 | if (objValue != null) { 33 | Integer num = m_maps[i].get(objValue); 34 | if (num == null) 35 | num = Integer.valueOf(0); 36 | m_maps[i].put(objValue, Integer.valueOf(num + 1)); 37 | } 38 | } 39 | } 40 | 41 | @Override 42 | public void leave(Object value) { 43 | 44 | } 45 | 46 | @Override 47 | public Map getValue() { 48 | Map event = new HashMap(); 49 | for (int i = 0; i < m_fields.length; i++) 50 | event.put(m_fields[i], m_maps[i]); 51 | return event; 52 | } 53 | 54 | @Override 55 | public Class getValueType() { 56 | return Map.class; 57 | } 58 | 59 | @Override 60 | public void clear() { 61 | m_fields = null; 62 | m_maps = null; 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EsperWrappedEventConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import com.ebay.jetstream.event.EventException; 9 | import com.ebay.jetstream.event.JetstreamEvent; 10 | import com.espertech.esper.client.EventBean; 11 | import com.espertech.esper.event.WrapperEventBean; 12 | import com.espertech.esper.event.WrapperEventType; 13 | 14 | /** 15 | * This converter serves cases like 16 | * 17 | * INSERT INTO metricUpdate SELECT Guid, EntityType, ate.* FROM metricsDb, applyTransaction ate GROUP BY Guid; 18 | * 19 | * SELECT * FROM metricUpdate; 20 | * 21 | * @author snikolaev 22 | * 23 | */ 24 | public class EsperWrappedEventConverter extends EsperDefaultEventConverter { 25 | 26 | @SuppressWarnings("unchecked") 27 | @Override 28 | public JetstreamEvent getJetstreamEvent(EventBean bean) throws EventException { 29 | JetstreamEvent event = super.getJetstreamEvent(bean); 30 | if (event == null && bean instanceof WrapperEventBean) { // trying to unwrap it 31 | WrapperEventBean wrappedBean = (WrapperEventBean) bean; 32 | WrapperEventType eventType = (WrapperEventType) wrappedBean.getEventType(); 33 | JetstreamEvent nestedEvent = super.getJetstreamEvent(wrappedBean.getUnderlyingEvent()); 34 | if (nestedEvent != null) { 35 | nestedEvent.putAll(wrappedBean.getUnderlyingMap()); // outer map has priority, i.e. it overrides similar fields 36 | event = new JetstreamEvent(eventType.getName(), null, nestedEvent); // and we keep original event type 37 | } 38 | else { 39 | event = new JetstreamEvent(eventType.getName(), null, wrappedBean.getUnderlyingMap()); 40 | } 41 | } 42 | return event; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/epl/AttributesAliasesConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | import org.springframework.beans.factory.BeanNameAware; 12 | import org.springframework.beans.factory.InitializingBean; 13 | import org.springframework.beans.factory.NamedBean; 14 | import org.springframework.jmx.export.annotation.ManagedResource; 15 | 16 | import com.ebay.jetstream.management.Management; 17 | import com.ebay.jetstream.xmlser.XSerializable; 18 | import com.espertech.esper.client.Configuration; 19 | 20 | @ManagedResource(objectName = "Event/Processor", description = "Attributes Aliases Configuration") 21 | public class AttributesAliasesConfiguration extends Configuration implements NamedBean, BeanNameAware, XSerializable, 22 | InitializingBean { 23 | 24 | private static final long serialVersionUID = -8212117594421994155L; 25 | private static String id; 26 | 27 | public static String getId() { 28 | return id; 29 | } 30 | 31 | private String name; 32 | private Map> aliasesMap; 33 | 34 | public AttributesAliasesConfiguration() { 35 | } 36 | 37 | public void afterPropertiesSet() throws Exception { 38 | Management.removeBeanOrFolder(getBeanName(), this); 39 | Management.addBean(getBeanName(), this); 40 | } 41 | 42 | public Map> getAliasesMap() { 43 | return aliasesMap; 44 | } 45 | 46 | public String getBeanName() { 47 | return name; 48 | } 49 | 50 | public void setAliasesMap(Map> aliasesMap) { 51 | this.aliasesMap = aliasesMap; 52 | } 53 | 54 | public void setBeanName(String name) { 55 | this.name = name; 56 | id = name; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/CreateDimensionGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.math.BigInteger; 9 | import java.security.MessageDigest; 10 | import java.security.NoSuchAlgorithmException; 11 | import java.util.List; 12 | 13 | import com.ebay.jetstream.event.JetstreamEvent; 14 | import com.espertech.esper.client.soda.AnnotationAttribute; 15 | 16 | public class CreateDimensionGenerator extends AffinityKeyGenerator { 17 | 18 | private String[] m_aFields; 19 | 20 | public CreateDimensionGenerator(List attributes) { 21 | super(attributes); 22 | for (AnnotationAttribute attr : attributes) { 23 | if ("dimensionspan".equals(attr.getName())) { 24 | String strSpan = (String)attr.getValue(); 25 | if (strSpan != null && strSpan.length() != 0) 26 | m_aFields = strSpan.split(","); 27 | } 28 | } 29 | 30 | if (m_aFields == null) 31 | throw new IllegalArgumentException("@CreateDimension.dimensionspan must be defined"); 32 | 33 | if (getNameAttribute() == null || getNameAttribute().length() == 0) 34 | throw new IllegalArgumentException("@CreateDimension.name must be defined"); 35 | } 36 | 37 | String getFieldName() { 38 | return getNameAttribute(); 39 | } 40 | 41 | String getValue(JetstreamEvent event) { 42 | 43 | MessageDigest md; 44 | try { 45 | md = MessageDigest.getInstance("MD5"); 46 | } 47 | catch (NoSuchAlgorithmException e) { 48 | throw new RuntimeException(e); 49 | } 50 | for (String strField : m_aFields) { 51 | Object objValue = event.get(strField); 52 | if (objValue != null) 53 | md.update(objValue.toString().getBytes()); 54 | } 55 | BigInteger bi = new BigInteger(1, md.digest()); 56 | return bi.toString(16); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EsperDefaultEventConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.Map; 9 | 10 | import com.ebay.jetstream.event.EventException; 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | import com.espertech.esper.client.EventBean; 13 | import com.espertech.esper.client.EventType; 14 | import com.espertech.esper.event.bean.BeanEventType; 15 | import com.espertech.esper.event.map.MapEventType; 16 | 17 | /** 18 | * @author msikes 19 | */ 20 | public class EsperDefaultEventConverter implements EsperEventConverter { 21 | 22 | /* 23 | * Default conversion from MapEventBean to JetstreamEvent. No other event beans are supported: if any is given, null is 24 | * returned. 25 | * 26 | * @see 27 | * com.ebay.jetstream.event.processor.esper.EsperEventConverter#getJetstreamEvent(com.espertech.esper.event.EventBean) 28 | */ 29 | @SuppressWarnings("unchecked") 30 | public JetstreamEvent getJetstreamEvent(EventBean bean) throws EventException { 31 | JetstreamEvent jetstreamEvent = null; 32 | EventType eventType = bean.getEventType(); 33 | if (eventType instanceof MapEventType) { 34 | MapEventType meType = (MapEventType) eventType; 35 | jetstreamEvent = new JetstreamEvent(meType.getName(), null, (Map) bean.getUnderlying()); 36 | } 37 | // 38 | // Consider this: 39 | // INSERT INTO FDL SELECT 40 | // com.ebay.jetstream.identity.client.IdentityClient.findDirectLinks(metricUpdate.Guid, 'user') AS FindDirectLinks 41 | // FROM metricUpdate; 42 | // /*$OUTPUT*/ SELECT FindDirectLinks.* FROM FDL; 43 | // 44 | else if (eventType instanceof BeanEventType) { 45 | BeanEventType beType = (BeanEventType) eventType; 46 | jetstreamEvent = new JetstreamEvent(beType.getName(), null, (Map) bean.getUnderlying()); 47 | } 48 | return jetstreamEvent; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/BroadCastAnnotationProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 13 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 14 | import com.ebay.jestream.event.annotation.EngineMetadata; 15 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 16 | import com.ebay.jetstream.event.EventSink; 17 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 18 | import com.ebay.jetstream.event.processor.esper.annotation.BroadCast; 19 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.BroadCastMetadata; 20 | import com.espertech.esper.client.soda.AnnotationPart; 21 | 22 | public class BroadCastAnnotationProcessor implements AnnotationProcessor { 23 | 24 | public StatementAnnotationInfo process(String statement, 25 | EngineMetadata engineAnntMetadata, 26 | Map annotConfigMap, 27 | StatementAnnotationInfo stmtAnntInfo, 28 | Collection registeredSinkList){ 29 | 30 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata).getAnnotationPartsMap(); 31 | List affinityAnnotationParts = partsMap 32 | .get(BroadCast.class.getSimpleName()); 33 | 34 | BroadCastMetadata metadata = new BroadCastMetadata(); 35 | for (AnnotationPart part : affinityAnnotationParts) { 36 | if (BroadCast.class.getSimpleName().equals(part.getName())) { 37 | metadata.setBroadcast(true); 38 | } 39 | } 40 | stmtAnntInfo.addAnnotationInfo(BroadCast.class.getSimpleName(), metadata); 41 | return stmtAnntInfo; 42 | } 43 | 44 | @Override 45 | public Class getAnnotationClass() { 46 | return BroadCast.class; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/DimensionAnnotationProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 13 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 14 | import com.ebay.jestream.event.annotation.EngineMetadata; 15 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 16 | import com.ebay.jetstream.event.EventSink; 17 | import com.ebay.jetstream.event.processor.esper.CreateDimensionGenerator; 18 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 19 | import com.ebay.jetstream.event.processor.esper.annotation.CreateDimension; 20 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.AffinityAnnotationMetadata; 21 | import com.espertech.esper.client.soda.AnnotationPart; 22 | 23 | public class DimensionAnnotationProcessor implements AnnotationProcessor { 24 | 25 | @Override 26 | public StatementAnnotationInfo process(String statement, 27 | EngineMetadata engineAnntMetadata, 28 | Map annotConfigMap, 29 | StatementAnnotationInfo stmtAnntInfo, 30 | Collection registeredSinkList) { 31 | 32 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata) 33 | .getAnnotationPartsMap(); 34 | List affinityAnnotationParts = partsMap 35 | .get("CreateDimension"); 36 | AffinityAnnotationMetadata metadata = new AffinityAnnotationMetadata(); 37 | for (AnnotationPart part : affinityAnnotationParts) { 38 | metadata.addToKeyGenList(new CreateDimensionGenerator(part 39 | .getAttributes())); 40 | } 41 | 42 | return stmtAnntInfo; 43 | } 44 | 45 | @Override 46 | public Class getAnnotationClass() { 47 | return CreateDimension.class; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/AffinityKeyGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.List; 9 | 10 | import com.ebay.jetstream.event.JetstreamEvent; 11 | import com.ebay.jetstream.event.JetstreamReservedKeys; 12 | import com.ebay.jetstream.event.processor.esper.annotation.CreateDimension; 13 | import com.espertech.esper.client.soda.AnnotationAttribute; 14 | import com.espertech.esper.client.soda.AnnotationPart; 15 | 16 | public class AffinityKeyGenerator { 17 | 18 | private String m_strNameAttribute; 19 | private AffinityKeyGenerator m_gen; 20 | 21 | public AffinityKeyGenerator(List attributes) { 22 | for (AnnotationAttribute attr : attributes) { 23 | if ("colname".equals(attr.getName()) || "name".equals(attr.getName())) 24 | m_strNameAttribute = (String)attr.getValue(); 25 | 26 | if ("dimension".equals(attr.getName())) { 27 | Object objDimension = attr.getValue(); 28 | if (objDimension instanceof AnnotationPart) { 29 | AnnotationPart annnoDimension = (AnnotationPart)objDimension; 30 | if (CreateDimension.class.getSimpleName().equals(annnoDimension.getName())) { 31 | m_gen = new CreateDimensionGenerator(annnoDimension.getAttributes()); 32 | } 33 | } 34 | } 35 | } 36 | } 37 | 38 | public void setEventFields(JetstreamEvent event) { 39 | if (m_gen == null) 40 | event.put(getFieldName(), getValue(event)); 41 | else { 42 | m_gen.setEventFields(event); 43 | event.put(getFieldName(), event.get(m_gen.getFieldName())); 44 | } 45 | } 46 | 47 | String getValue(JetstreamEvent event) { 48 | String strValue = null; 49 | Object objValue = event.get(getNameAttribute()); 50 | if (objValue != null) 51 | strValue = objValue.toString(); 52 | return strValue; 53 | } 54 | 55 | String getFieldName() { 56 | return JetstreamReservedKeys.MessageAffinityKey.toString(); 57 | } 58 | 59 | String getNameAttribute() { 60 | return m_strNameAttribute; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/AffinityAnnotationProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 13 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 14 | import com.ebay.jestream.event.annotation.EngineMetadata; 15 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 16 | import com.ebay.jetstream.event.EventSink; 17 | import com.ebay.jetstream.event.processor.esper.AffinityKeyGenerator; 18 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 19 | import com.ebay.jetstream.event.processor.esper.annotation.ClusterAffinityTag; 20 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.AffinityAnnotationMetadata; 21 | import com.espertech.esper.client.soda.AnnotationPart; 22 | 23 | public class AffinityAnnotationProcessor implements AnnotationProcessor 24 | { 25 | 26 | public StatementAnnotationInfo process(String statement, 27 | EngineMetadata engineAnntMetadata, 28 | Map annotConfigMap, 29 | StatementAnnotationInfo stmtAnntInfo, 30 | Collection registeredSinkList){ 31 | 32 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata).getAnnotationPartsMap(); 33 | AffinityAnnotationMetadata metadata = new AffinityAnnotationMetadata(); 34 | List affinityAnnotationParts = partsMap.get("ClusterAffinityTag"); 35 | for(AnnotationPart part : affinityAnnotationParts) 36 | { 37 | metadata.addToKeyGenList(new AffinityKeyGenerator(part.getAttributes())); 38 | } 39 | 40 | stmtAnntInfo.addAnnotationInfo(ClusterAffinityTag.class.getSimpleName(), metadata); 41 | 42 | return stmtAnntInfo; 43 | } 44 | 45 | @Override 46 | public Class getAnnotationClass() { 47 | return ClusterAffinityTag.class; 48 | } 49 | 50 | 51 | } 52 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/MapEventType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.Collections; 9 | import java.util.Map; 10 | 11 | import com.ebay.jetstream.config.ConfigException; 12 | 13 | /** 14 | * MappedEvent is used to declare an event based on a Map, such as a JetstreamEvent. 15 | * @author trobison, derived from original work by msikes 16 | */ 17 | public class MapEventType extends AbstractEventType { 18 | 19 | private static final long serialVersionUID = -5381179076717992566L; 20 | private Map m_mapEventFields; 21 | 22 | @SuppressWarnings("unchecked") 23 | public Map getEventFields() { 24 | return m_mapEventFields == null ? Collections.EMPTY_MAP : m_mapEventFields; 25 | } 26 | 27 | public void setEventFields(Map mapEventFields) throws ConfigException { 28 | validateFields(mapEventFields); 29 | m_mapEventFields = Collections.unmodifiableMap(mapEventFields); 30 | } 31 | 32 | @Override 33 | protected Map getEventDefinition() { 34 | return getEventFields(); 35 | } 36 | 37 | @SuppressWarnings("unchecked") 38 | private void validateFields(Map mapFields) throws ConfigException { 39 | 40 | for (Map.Entry entry : mapFields.entrySet()) { 41 | Object objValue = entry.getValue(); 42 | 43 | /* 44 | * String must be promoted to Class because Spring won't convert String 45 | * to Class. Map values are validated recursively, null and Class are OK, 46 | * anything else is invalid. 47 | */ 48 | if (objValue == null || objValue instanceof Class) 49 | continue; 50 | 51 | if (objValue instanceof String) { 52 | try { 53 | entry.setValue(Class.forName((String)objValue)); 54 | } 55 | catch (ClassNotFoundException e) { 56 | throw new ConfigException("Mapped field validation failed", e); 57 | } 58 | } 59 | else if (objValue instanceof Map) 60 | validateFields((Map)objValue); 61 | else 62 | throw new ConfigException("not a String, a Class, a Map or null: " + objValue); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/esper/customaggregate/EventListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.esper.customaggregate; 7 | 8 | import java.util.concurrent.atomic.AtomicLong; 9 | 10 | import org.springframework.context.ApplicationEvent; 11 | 12 | import com.ebay.jetstream.event.EventException; 13 | import com.ebay.jetstream.event.processor.esper.EsperEventConverter; 14 | import com.ebay.jetstream.event.processor.esper.EsperEventListener; 15 | import com.ebay.jetstream.event.processor.esper.EsperWrappedEventConverter; 16 | import com.ebay.jetstream.xmlser.XSerializable; 17 | import com.espertech.esper.client.EPServiceProvider; 18 | import com.espertech.esper.client.EPStatement; 19 | import com.espertech.esper.client.EventBean; 20 | import com.espertech.esper.client.StatementAwareUpdateListener; 21 | 22 | public class EventListener extends EsperEventListener implements StatementAwareUpdateListener, XSerializable{ 23 | 24 | private final AtomicLong m_eventReceived = new AtomicLong(0); 25 | 26 | protected static final EsperEventConverter DEFAULT_CONVERTER = new EsperWrappedEventConverter(); 27 | 28 | private EsperEventConverter m_eventConverter = DEFAULT_CONVERTER; 29 | 30 | public EsperEventConverter getEventConverter() { 31 | return m_eventConverter; 32 | } 33 | 34 | @Override 35 | public void update(EventBean[] newEvents, EventBean[] oldEvents, 36 | EPStatement statement, EPServiceProvider epServiceProvider) { 37 | 38 | System.out.println(newEvents); 39 | if(newEvents == null) 40 | return; 41 | 42 | m_eventReceived.incrementAndGet(); 43 | for (EventBean bean : newEvents) { 44 | try { 45 | System.out.println(bean.get("conOutput")); 46 | } 47 | catch (EventException e) { 48 | } 49 | } 50 | 51 | // TODO Auto-generated method stub 52 | 53 | } 54 | 55 | @Override 56 | public void pause() { 57 | // TODO Auto-generated method stub 58 | 59 | } 60 | 61 | @Override 62 | protected void processApplicationEvent(ApplicationEvent event) { 63 | // TODO Auto-generated method stub 64 | 65 | } 66 | 67 | @Override 68 | public void resume() { 69 | // TODO Auto-generated method stub 70 | 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/aggregates/TopKAggregator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.aggregates; 7 | 8 | 9 | import java.util.LinkedHashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | import com.clearspring.analytics.stream.Counter; 14 | import com.clearspring.analytics.stream.StreamSummary; 15 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 16 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 17 | 18 | /** 19 | * @author shmurthy@ebay.com Esper aggregation method for computing 20 | * the topN elements of the elements provided to this aggregator 21 | * 22 | * Usage: select topN(, , element) as uniqueElement from stream 23 | */ 24 | 25 | 26 | public class TopKAggregator implements AggregationMethod { 27 | 28 | StreamSummary m_counter; 29 | Integer m_capacity; 30 | Integer m_topElementCnt = new Integer(10); 31 | 32 | @Override 33 | public void enter(Object value) { 34 | 35 | Object[] params = (Object[]) value; 36 | 37 | if(m_counter == null) { 38 | m_capacity = (Integer) params[0]; 39 | m_topElementCnt = (Integer) params[1]; 40 | m_counter = new StreamSummary(m_capacity.intValue()); 41 | } 42 | 43 | 44 | m_counter.offer(params[2]); 45 | 46 | } 47 | 48 | @Override 49 | public void leave(Object value) { 50 | 51 | } 52 | 53 | @Override 54 | public Object getValue() { 55 | 56 | 57 | Map topN = new LinkedHashMap(); 58 | 59 | if ((m_counter == null)) 60 | return topN; 61 | 62 | List> topCounters = m_counter.topK(m_topElementCnt); 63 | 64 | for (Counter counter : topCounters) { 65 | topN.put(counter.getItem(), counter.getCount()); 66 | } 67 | return topN; 68 | } 69 | 70 | @Override 71 | public Class getValueType() { 72 | 73 | return Map.class; 74 | } 75 | 76 | @Override 77 | public void clear() { 78 | m_counter = null; 79 | 80 | } 81 | 82 | // @Override 83 | public void validate(AggregationValidationContext validationContext) { 84 | 85 | 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/ESPEventProcessorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.ArrayList; 9 | import java.util.HashMap; 10 | 11 | import org.junit.After; 12 | import org.junit.Before; 13 | import org.junit.Ignore; 14 | 15 | import com.ebay.jetstream.config.ApplicationInformation; 16 | import com.ebay.jetstream.config.Configuration; 17 | import com.ebay.jetstream.config.RootConfiguration; 18 | import com.ebay.jetstream.event.EventException; 19 | import com.ebay.jetstream.event.EventSink; 20 | import com.ebay.jetstream.event.JetstreamEvent; 21 | 22 | 23 | public class ESPEventProcessorTest { 24 | protected static final Configuration s_springConfiguration = new RootConfiguration(new ApplicationInformation( 25 | "ESPTest", "0.0"), new String[] { Configuration.getClasspathContext(ESPTest.class, null) }); 26 | 27 | private EsperProcessor m_eventProcessor = null; 28 | 29 | @Before 30 | public void setUp() throws Exception { 31 | m_eventProcessor = (EsperProcessor)s_springConfiguration.getBean("ESPTestProcessorXYZ"); 32 | m_eventProcessor.addEventSink(new EventSink() { 33 | public String getBeanName() { 34 | // TODO Auto-generated method stub 35 | return null; 36 | } 37 | 38 | public void sendEvent(JetstreamEvent event) throws EventException { 39 | System.out.println("Receive event " + event); //KEEPME 40 | } 41 | }); 42 | } 43 | 44 | @After 45 | public void tearDown() throws Exception { 46 | m_eventProcessor = null; 47 | } 48 | 49 | @Ignore 50 | public void testSendEvent() { 51 | HashMap map = new HashMap(); 52 | String field1 = "Field 1 value"; 53 | ArrayList field2 = new ArrayList(); 54 | EsperTestEventField field3 = new EsperTestEventField(); 55 | field2.add("Field 2, value 1"); 56 | field2.add("Field 2, value 2"); 57 | map.put("id", "001"); 58 | map.put("field1", field1); 59 | map.put("field2", field2); 60 | map.put("field3", field3); 61 | JetstreamEvent event = new JetstreamEvent("ESPTestEvent1", null, map); 62 | m_eventProcessor.sendEvent(event); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/AnnotationProcessorFacade.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.Collection; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 14 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 15 | import com.ebay.jestream.event.annotation.EngineMetadata; 16 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 17 | import com.ebay.jetstream.event.EventSink; 18 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 19 | import com.espertech.esper.client.soda.AnnotationPart; 20 | import com.espertech.esper.client.soda.EPStatementObjectModel; 21 | 22 | public class AnnotationProcessorFacade { 23 | 24 | private Map annotationConfigMap = new HashMap(); 25 | 26 | public StatementAnnotationInfo getStatementAnnotationInfo( 27 | List annotationParts, String statement, 28 | EPStatementObjectModel model, 29 | Map> partsMap, 30 | Collection registeredSinkList) throws Exception { 31 | 32 | StatementAnnotationInfo stmtAnntInfo = new StatementAnnotationInfo(); 33 | EngineMetadata engineMetadata = new EsperEngineAnnotationMetadata(); 34 | 35 | ((EsperEngineAnnotationMetadata) engineMetadata) 36 | .setAnnotationPartsMap(partsMap); 37 | ((EsperEngineAnnotationMetadata) engineMetadata) 38 | .setStatementObjectModel(model); 39 | 40 | for (AnnotationPart part : annotationParts) { 41 | if (annotationConfigMap.get(part.getName()) != null) { 42 | AnnotationProcessor annotationProcessor = annotationConfigMap 43 | .get(part.getName()).getProcessor(); 44 | if (annotationProcessor == null) { 45 | continue; 46 | } 47 | stmtAnntInfo = annotationProcessor.process(statement, 48 | engineMetadata, annotationConfigMap, stmtAnntInfo, 49 | registeredSinkList); 50 | } 51 | } 52 | 53 | return stmtAnntInfo; 54 | 55 | } 56 | 57 | public void setAnnotationConfigurationMap( 58 | Map annotationConfigMap) { 59 | this.annotationConfigMap = annotationConfigMap; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/PublishOnAnnotationProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 13 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 14 | import com.ebay.jestream.event.annotation.EngineMetadata; 15 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 16 | import com.ebay.jetstream.event.EventSink; 17 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 18 | import com.ebay.jetstream.event.processor.esper.annotation.PublishOn; 19 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.PublishOnAnnotationMetadata; 20 | import com.espertech.esper.client.soda.AnnotationAttribute; 21 | import com.espertech.esper.client.soda.AnnotationPart; 22 | 23 | public class PublishOnAnnotationProcessor implements AnnotationProcessor { 24 | 25 | @Override 26 | public StatementAnnotationInfo process(String statement, 27 | EngineMetadata engineAnntMetadata, 28 | Map annotConfigMap, 29 | StatementAnnotationInfo stmtAnntInfo, 30 | Collection registeredSinkList) { 31 | 32 | String strTopics = null; 33 | String strUrls = null; 34 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata) 35 | .getAnnotationPartsMap(); 36 | List affinityAnnotationParts = partsMap 37 | .get(PublishOn.class.getSimpleName()); 38 | PublishOnAnnotationMetadata metadata = new PublishOnAnnotationMetadata(); 39 | for (AnnotationPart part : affinityAnnotationParts) { 40 | for (AnnotationAttribute attr : part.getAttributes()) { 41 | if (strTopics == null && "topics".equals(attr.getName())) { 42 | strTopics = (String) attr.getValue(); 43 | metadata.setTopics(strTopics); 44 | } 45 | if (strUrls == null && "urls".equals(attr.getName())) { 46 | 47 | strUrls = (String) attr.getValue(); 48 | metadata.setUrls(strUrls); 49 | } 50 | } 51 | } 52 | stmtAnntInfo.addAnnotationInfo(PublishOn.class.getSimpleName(), 53 | metadata); 54 | return stmtAnntInfo; 55 | } 56 | 57 | @Override 58 | public Class getAnnotationClass() { 59 | return PublishOn.class; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/epl/JetstreamSideEventIdGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import java.security.SecureRandom; 9 | import java.util.Map; 10 | import java.util.concurrent.ConcurrentHashMap; 11 | 12 | import com.ebay.jetstream.util.InetAddressParser; 13 | 14 | public final class JetstreamSideEventIdGenerator { 15 | 16 | private static final Map EVENT_DATA = new ConcurrentHashMap(); 17 | 18 | private JetstreamSideEventIdGenerator() { 19 | // utility class, no need to construct 20 | } 21 | 22 | public static long generateEventId(int nHostIp) { 23 | 24 | EventIdInfo info = EVENT_DATA.get(nHostIp); 25 | if (info == null) { 26 | info = new EventIdInfo(nHostIp); 27 | EVENT_DATA.put(nHostIp, info); 28 | } 29 | 30 | byte[] IP_BYTES = info.m_address; 31 | long nRandomMask = info.m_rand.nextInt(16) & 0xffff; 32 | long lEventId = (System.nanoTime() << 48) | (nRandomMask << 32) | ((long)(info.m_sequence.getAndIncrement() & 0xff) << 24) | ((IP_BYTES[1] & 0xff) << 16) | ((IP_BYTES[2] & 0xff) << 8) | (IP_BYTES[3] & 0xff); 33 | return lEventId < 0 ? -lEventId : lEventId; 34 | } 35 | 36 | private static final class EventIdInfo { 37 | private byte[] m_address; 38 | private final AtomicByte m_sequence = new AtomicByte(); 39 | private final Random m_rand = new Random(); 40 | EventIdInfo(int nHostIp) { 41 | m_address = InetAddressParser.intToAddressBytes(nHostIp); 42 | } 43 | } 44 | 45 | private static final class Random extends SecureRandom { 46 | public int nextInt(int bits) { 47 | return super.next(bits); 48 | } 49 | }; 50 | 51 | 52 | private static final class AtomicByte extends Number { 53 | private byte m_bValue; 54 | 55 | public synchronized byte get() { 56 | return m_bValue; 57 | } 58 | 59 | public synchronized byte getAndIncrement() { 60 | byte current = m_bValue++; //IBM-perf_AtomicLong 61 | return current; 62 | } 63 | 64 | @Override 65 | public int intValue() { 66 | return (int)get(); 67 | } 68 | 69 | @Override 70 | public long longValue() { 71 | return (long)get(); 72 | } 73 | 74 | @Override 75 | public float floatValue() { 76 | return (float)get(); 77 | } 78 | 79 | @Override 80 | public double doubleValue() { 81 | return (double)get(); 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/SendMailAnnotationProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 13 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 14 | import com.ebay.jestream.event.annotation.EngineMetadata; 15 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 16 | import com.ebay.jetstream.event.EventSink; 17 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 18 | import com.ebay.jetstream.event.processor.esper.annotation.SendMail; 19 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.SendMailAnnotationMetadata; 20 | import com.espertech.esper.client.soda.AnnotationAttribute; 21 | import com.espertech.esper.client.soda.AnnotationPart; 22 | 23 | public class SendMailAnnotationProcessor implements AnnotationProcessor { 24 | 25 | @Override 26 | public Class getAnnotationClass() { 27 | return SendMail.class; 28 | } 29 | 30 | @Override 31 | public StatementAnnotationInfo process(String statement, 32 | EngineMetadata engineAnntMetadata, 33 | Map annotConfigMap, 34 | StatementAnnotationInfo stmtAnntInfo, 35 | Collection registeredSinkList) { 36 | 37 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata) 38 | .getAnnotationPartsMap(); 39 | List affinityAnnotationParts = partsMap 40 | .get(SendMail.class.getSimpleName()); 41 | SendMailAnnotationMetadata metadata = new SendMailAnnotationMetadata(); 42 | for (AnnotationPart part : affinityAnnotationParts) { 43 | for (AnnotationAttribute attr : part.getAttributes()) { 44 | if ("alertList".equals(attr.getName())) { 45 | metadata.setAlertList((String) attr.getValue()); 46 | } else if ("mailServer".equals(attr.getName())) { 47 | metadata.setMailServer((String) attr.getValue()); 48 | } else if ("alertSeverity".equals(attr.getName())) { 49 | metadata.setAlertSeverity((String) attr.getValue()); 50 | } else if ("sendFrom".equals(attr.getName())) { 51 | metadata.setSendFrom((String) attr.getValue()); 52 | } else if ("eventFields".equals(attr.getName())) { 53 | String fields = attr.getValue().toString().replace(" ", ""); 54 | metadata.setEventFields(fields); 55 | } else if ("mailContent".equals(attr.getName())) { 56 | metadata.setMailContent((String) attr.getValue()); 57 | } else if ("mailSubject".equals(attr.getName())) { 58 | metadata.setMailSubject((String) attr.getValue()); 59 | } 60 | } 61 | } 62 | stmtAnntInfo 63 | .addAnnotationInfo(SendMail.class.getSimpleName(), metadata); 64 | return stmtAnntInfo; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/listener/SendMailAnnotationListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.listener; 7 | 8 | import java.util.Properties; 9 | 10 | import javax.mail.Message; 11 | import javax.mail.Session; 12 | import javax.mail.Transport; 13 | import javax.mail.internet.InternetAddress; 14 | import javax.mail.internet.MimeMessage; 15 | 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | import com.ebay.jestream.event.annotation.AnnotationListener; 20 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 21 | import com.ebay.jetstream.event.JetstreamEvent; 22 | import com.ebay.jetstream.event.processor.esper.annotation.SendMail; 23 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.SendMailAnnotationMetadata; 24 | 25 | public class SendMailAnnotationListener implements AnnotationListener { 26 | private static final String ANNO_KEY = SendMail.class.getSimpleName(); 27 | 28 | private static final Logger s_logger = LoggerFactory.getLogger("com.ebay.jetstream.notification.MailSender"); 29 | Properties properties; 30 | 31 | @Override 32 | public JetstreamEvent processMetaInformation(JetstreamEvent event, 33 | StatementAnnotationInfo annotationInfo) { 34 | 35 | SendMailAnnotationMetadata anntmetadata = (SendMailAnnotationMetadata) annotationInfo.getAnnotationInfo(ANNO_KEY); 36 | properties = new Properties(); 37 | properties.setProperty("mail.smtp.host", anntmetadata.getMailServer()); 38 | 39 | Session session = Session.getInstance(properties, null); 40 | try{ 41 | MimeMessage message = new MimeMessage(session); 42 | message.addHeader("Content-type", "text/HTML; charset=UTF-8"); 43 | 44 | message.setFrom(new InternetAddress(anntmetadata.getSendFrom())); 45 | 46 | message.setRecipients(Message.RecipientType.TO, 47 | InternetAddress.parse(anntmetadata.getAlertList(), false)); 48 | 49 | String[] fieldList = anntmetadata.getEventFields().split(","); 50 | StringBuffer sb = new StringBuffer(); 51 | sb.append(anntmetadata.getMailContent()).append(".\n"); 52 | for(int i=0; i annotConfigMap, 40 | StatementAnnotationInfo stmtAnntInfo, 41 | Collection registeredSinkList){ 42 | 43 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata).getAnnotationPartsMap(); 44 | List annotationParts = partsMap 45 | .get(OutputToWithNewStreamName.class.getSimpleName()); 46 | 47 | OutputToNewStreamMetadata metadata = new OutputToNewStreamMetadata(); 48 | for (AnnotationPart part : annotationParts) { 49 | if (OutputToWithNewStreamName.class.getSimpleName().equals(part.getName())) { 50 | for (AnnotationAttribute attr : part 51 | .getAttributes()) { 52 | metadata.setStreamName((String)attr.getValue()); 53 | } 54 | } 55 | } 56 | 57 | for (AnnotationPart part : annotationParts) { 58 | List attributes = part.getAttributes(); 59 | if (attributes.size() > 0) { 60 | String[] aSinks = ((String) attributes.get(0).getValue()) 61 | .replaceAll(" ", "").split(","); 62 | List sinklist = new ArrayList(); 63 | for (String strSinkName : aSinks) { 64 | boolean bFound = false; 65 | for (EventSink eventsink : registeredSinkList) { 66 | if (eventsink.getBeanName().equals(strSinkName)) { 67 | sinklist.add(eventsink); 68 | bFound = true; 69 | break; 70 | } 71 | } 72 | if (!bFound) { 73 | throw new IllegalArgumentException("Output Bean '" 74 | + strSinkName + "' is was not found."); 75 | } 76 | } 77 | 78 | if (sinklist.size() > 0) 79 | metadata.addToSinkList(sinklist); 80 | } 81 | 82 | } 83 | stmtAnntInfo.addAnnotationInfo(OutputToWithNewStreamName.class.getSimpleName(), metadata); 84 | return stmtAnntInfo; 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/DBInfoAnnotationProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.Collection; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.regex.Matcher; 13 | import java.util.regex.Pattern; 14 | 15 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 16 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 17 | import com.ebay.jestream.event.annotation.EngineMetadata; 18 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 19 | import com.ebay.jetstream.event.EventSink; 20 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 21 | import com.ebay.jetstream.event.processor.esper.annotation.DBInfo; 22 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.DBInfoAnnotationMetadata; 23 | import com.espertech.esper.client.soda.AnnotationAttribute; 24 | import com.espertech.esper.client.soda.AnnotationPart; 25 | import com.espertech.esper.client.soda.EPStatementObjectModel; 26 | import com.espertech.esper.client.soda.FromClause; 27 | 28 | public class DBInfoAnnotationProcessor implements AnnotationProcessor { 29 | 30 | private static final Pattern FROM_STREAM = Pattern 31 | .compile("(?im:\\sfrom\\s+(\\w+))"); 32 | 33 | @Override 34 | public StatementAnnotationInfo process(String statement, 35 | EngineMetadata engineAnntMetadata, 36 | Map annotConfigMap, 37 | StatementAnnotationInfo stmtAnntInfo, 38 | Collection registeredSinkList) { 39 | 40 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata) 41 | .getAnnotationPartsMap(); 42 | EPStatementObjectModel model = ((EsperEngineAnnotationMetadata) engineAnntMetadata) 43 | .getStatementObjectModel(); 44 | Map mapMeta = new HashMap(); 45 | boolean bAddDbMap = false; 46 | String strParsedEventStream = null; 47 | DBInfoAnnotationMetadata metadata = new DBInfoAnnotationMetadata(); 48 | if (partsMap.get(DBInfo.class.getSimpleName()) != null) { 49 | bAddDbMap = true; 50 | 51 | // Parse out event stream name 52 | FromClause fc = model.getFromClause(); 53 | if (fc != null && fc.getStreams().size() == 1) { 54 | Matcher matcher = FROM_STREAM.matcher(statement); 55 | if (matcher.find()) { 56 | String strFoundStream = matcher.group(1); 57 | if (strFoundStream != null) 58 | strParsedEventStream = strFoundStream; 59 | } 60 | } 61 | 62 | List dbInfoAnnoParts = partsMap.get(DBInfo.class 63 | .getSimpleName()); 64 | for (AnnotationPart annotationPart : dbInfoAnnoParts) { 65 | for (AnnotationAttribute attr : annotationPart.getAttributes()) { 66 | mapMeta.put(attr.getName(), attr.getValue()); 67 | } 68 | } 69 | } 70 | if (bAddDbMap) { 71 | if (strParsedEventStream != null && mapMeta.get("table") == null) { 72 | mapMeta.put("table", strParsedEventStream); 73 | } 74 | mapMeta.put("refresh", Boolean.TRUE); 75 | } else { 76 | mapMeta = null; 77 | } 78 | 79 | metadata.setMapMetaData(mapMeta); 80 | stmtAnntInfo.addAnnotationInfo(DBInfo.class.getSimpleName(), metadata); 81 | 82 | return stmtAnntInfo; 83 | 84 | } 85 | 86 | @Override 87 | public Class getAnnotationClass() { 88 | return DBInfo.class; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/epl/EPLUtilitiesTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import org.junit.Test; 14 | 15 | /** 16 | * @author rmuthupandian 17 | * 18 | */ 19 | public class EPLUtilitiesTest { 20 | 21 | public void clearSets(Map map1, Map map2) { 22 | map1.clear(); 23 | map2.clear(); 24 | } 25 | 26 | @Test 27 | public void testFindIntersection() { 28 | Map map1 = new HashMap(); 29 | Map map2 = new HashMap(); 30 | 31 | // Double test 32 | map1.put("score1", 0.0888676767676767676767676767); 33 | map2.put("score1", 0.0888676767676767676767676767); 34 | 35 | System.out.println(EPLUtilities.findIntersection(map1, map2)); 36 | 37 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 0); // No Change 38 | 39 | map1.put("score1", 0.0); 40 | map2.put("score1", 0.2); 41 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 1); // Value Change 42 | 43 | clearSets(map1, map2); 44 | map1.put("score1", 0.0); 45 | map2.put("score1", 0.1); 46 | map2.put("score2", 0.2); 47 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 2); // Value Change + Addional Key 48 | 49 | // String Test 50 | clearSets(map1, map2); 51 | map1.put("score1", "234"); 52 | map2.put("score1", "234"); 53 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 0); // No change 54 | 55 | clearSets(map1, map2); 56 | map1.put("score1", "234"); 57 | map2.put("score1", "234.4"); 58 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 1); // Value Change 59 | 60 | clearSets(map1, map2); 61 | map1.put("score1", "234"); 62 | map2.put("score1", "234.4"); 63 | map2.put("score2", "234.4"); 64 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 2); // Value Change + Additional Key 65 | 66 | // Long Test 67 | clearSets(map1, map2); 68 | map1.put("score1", 234L); 69 | map2.put("score1", 234L); 70 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 0); // No change 71 | 72 | clearSets(map1, map2); 73 | map1.put("score1", 234L); 74 | map2.put("score1", 235L); 75 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 1); // Value Change 76 | 77 | clearSets(map1, map2); 78 | map1.put("score1", 234L); 79 | map2.put("score1", 235L); 80 | map2.put("score2", 4444L); 81 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 2); // Value Change + Additional Key 82 | 83 | // Float Test 84 | 85 | clearSets(map1, map2); 86 | map1.put("score1", 234.5565f); 87 | map2.put("score1", 234.5565f); 88 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 0); // No change 89 | 90 | clearSets(map1, map2); 91 | map1.put("score1", 234.5f); 92 | map2.put("score1", 235f); 93 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 1); // Value Change 94 | 95 | clearSets(map1, map2); 96 | map1.put("score1", 234.5f); 97 | map2.put("score1", 235f); 98 | map2.put("score2", 666f); 99 | assertEquals(EPLUtilities.findIntersection(map1, map2).size(), 2); // Value Change + Additional Key 100 | 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/processor/OutputToAnnotationProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.processor; 7 | 8 | import java.util.ArrayList; 9 | import java.util.Collection; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.regex.Matcher; 13 | import java.util.regex.Pattern; 14 | 15 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 16 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 17 | import com.ebay.jestream.event.annotation.EngineMetadata; 18 | import com.ebay.jestream.event.annotation.StatementAnnotationInfo; 19 | import com.ebay.jetstream.event.EventSink; 20 | import com.ebay.jetstream.event.processor.esper.EsperEngineAnnotationMetadata; 21 | import com.ebay.jetstream.event.processor.esper.annotation.OutputTo; 22 | import com.ebay.jetstream.event.processor.esper.annotation.metadata.OutputToAnnotationMetadata; 23 | import com.espertech.esper.client.soda.AnnotationAttribute; 24 | import com.espertech.esper.client.soda.AnnotationPart; 25 | import com.espertech.esper.client.soda.EPStatementObjectModel; 26 | import com.espertech.esper.client.soda.FromClause; 27 | 28 | public class OutputToAnnotationProcessor implements AnnotationProcessor { 29 | private static final Pattern FROM_STREAM = Pattern 30 | .compile("(?im:\\sfrom\\s+(\\w+))"); 31 | 32 | @Override 33 | public StatementAnnotationInfo process(String statement, 34 | EngineMetadata engineAnntMetadata, 35 | Map annotConfigMap, 36 | StatementAnnotationInfo stmtAnntInfo, 37 | Collection registeredSinkList) { 38 | 39 | Map> partsMap = ((EsperEngineAnnotationMetadata) engineAnntMetadata) 40 | .getAnnotationPartsMap(); 41 | EPStatementObjectModel model = ((EsperEngineAnnotationMetadata) engineAnntMetadata) 42 | .getStatementObjectModel(); 43 | 44 | List outputToAnnotationParts = partsMap 45 | .get(OutputTo.class.getSimpleName()); 46 | String strParsedEventStream = null; 47 | OutputToAnnotationMetadata metadata = new OutputToAnnotationMetadata(); 48 | 49 | FromClause fc = model.getFromClause(); 50 | if (fc != null && fc.getStreams().size() == 1) { 51 | Matcher matcher = FROM_STREAM.matcher(statement); 52 | if (matcher.find()) { 53 | String strFoundStream = matcher.group(1); 54 | if (strFoundStream != null) 55 | strParsedEventStream = strFoundStream; 56 | } 57 | } 58 | 59 | for (AnnotationPart part : outputToAnnotationParts) { 60 | List attributes = part.getAttributes(); 61 | if (attributes.size() > 0) { 62 | String[] aSinks = ((String) attributes.get(0).getValue()) 63 | .replaceAll(" ", "").split(","); 64 | List sinklist = new ArrayList(); 65 | for (String strSinkName : aSinks) { 66 | boolean bFound = false; 67 | for (EventSink eventsink : registeredSinkList) { 68 | if (eventsink.getBeanName().equals(strSinkName)) { 69 | sinklist.add(eventsink); 70 | bFound = true; 71 | break; 72 | } 73 | } 74 | if (!bFound) { 75 | throw new IllegalArgumentException("Output Bean '" 76 | + strSinkName + "' is was not found."); 77 | } 78 | } 79 | 80 | if (sinklist.size() > 0) 81 | metadata.addToSinkList(sinklist); 82 | } 83 | 84 | } 85 | 86 | metadata.setStreamName(strParsedEventStream); 87 | stmtAnntInfo 88 | .addAnnotationInfo(OutputTo.class.getSimpleName(), metadata); 89 | return stmtAnntInfo; 90 | } 91 | 92 | @Override 93 | public Class getAnnotationClass() { 94 | return OutputTo.class; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/epl/EPLCompileTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import static org.junit.Assert.assertFalse; 9 | import static org.junit.Assert.assertTrue; 10 | 11 | import org.junit.Test; 12 | 13 | import com.ebay.jetstream.event.processor.esper.EPL; 14 | import com.ebay.jetstream.event.processor.esper.EsperConfiguration; 15 | import com.ebay.jetstream.event.processor.esper.EsperProcessor; 16 | import com.espertech.esper.client.EPException; 17 | 18 | public class EPLCompileTest { 19 | 20 | public static class TestEvents { 21 | private long timemillis; 22 | private String ck; 23 | 24 | public TestEvents(String id, long time) { 25 | this.ck = id; 26 | this.timemillis = time; 27 | } 28 | 29 | public long getTimemillis() { 30 | return timemillis; 31 | } 32 | 33 | public String getCk() { 34 | return ck; 35 | } 36 | } 37 | 38 | @Test 39 | public void testMultipleStmtEPL(){ 40 | String EPL1 = " select * from TestEvent; " + 41 | " Select timemillis from TestEvent; " + 42 | " @OutputTo(\"omc\") select * from TestEvent; "; 43 | 44 | EPL epl = new EPL(); 45 | epl.setStatementBlock(EPL1); 46 | EsperProcessor proc = new EsperProcessor(); 47 | EsperConfiguration config = new EsperConfiguration(); 48 | config.addEventType("TestEvent", TestEvents.class); 49 | config.addImport(com.ebay.jetstream.event.processor.esper.annotation.OutputTo.class); 50 | proc.setConfiguration(config); 51 | assertTrue(proc.compileEPL(epl)); 52 | 53 | } 54 | 55 | 56 | @Test 57 | public void testcreateSchemaEPL(){ 58 | String EPL1 = " CREATE SCHEMA SchemaEvent as ( key1 string, key2 string) ;" + 59 | " select * from SchemaEvent; " + 60 | " Select key2 from SchemaEvent; " + 61 | " @OutputTo(\"omc\") select * from SchemaEvent; "; 62 | 63 | EPL epl = new EPL(); 64 | epl.setStatementBlock(EPL1); 65 | EsperProcessor proc = new EsperProcessor(); 66 | EsperConfiguration config = new EsperConfiguration(); 67 | config.addImport(com.ebay.jetstream.event.processor.esper.annotation.OutputTo.class); 68 | proc.setConfiguration(config); 69 | assertTrue(proc.compileEPL(epl)); 70 | } 71 | 72 | @Test(expected=EPException.class) 73 | public void testWrongAnnotationEPL(){ 74 | String EPL1 = " select * from TestEvent; " + 75 | " Select timemillis from TestEvent; " + 76 | " @testing(\"omc\") select * from TestEvent; "; 77 | 78 | EPL epl = new EPL(); 79 | epl.setStatementBlock(EPL1); 80 | EsperProcessor proc = new EsperProcessor(); 81 | EsperConfiguration config = new EsperConfiguration(); 82 | config.addEventType("TestEvent", TestEvents.class); 83 | proc.setConfiguration(config); 84 | assertFalse(proc.compileEPL(epl)); 85 | } 86 | 87 | @Test(expected=EPException.class) 88 | public void testWrongEventTYpeEPL(){ 89 | String EPL1 = " select * from TestEvent; " + 90 | " Select timemillis from TestEvent; " + 91 | " @OutputTo(\"omc\") select * from TestEvent; "; 92 | 93 | EPL epl = new EPL(); 94 | epl.setStatementBlock(EPL1); 95 | EsperProcessor proc = new EsperProcessor(); 96 | EsperConfiguration config = new EsperConfiguration(); 97 | proc.setConfiguration(config); 98 | assertFalse(proc.compileEPL(epl)); 99 | 100 | } 101 | 102 | 103 | 104 | 105 | 106 | } 107 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/esper/customaggregate/ESPAggrTest.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | java.lang.Math 59 | com.ebay.jetstream.epl.EPLUtilities 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/annotation/config/AnnotationConfigLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.annotation.config; 7 | 8 | import java.lang.reflect.Method; 9 | import java.util.ArrayList; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.Set; 14 | 15 | import org.reflections.Reflections; 16 | 17 | import com.ebay.jestream.event.annotation.AnnotationConfiguration; 18 | import com.ebay.jestream.event.annotation.AnnotationListener; 19 | import com.ebay.jestream.event.annotation.AnnotationProcessor; 20 | 21 | 22 | /** 23 | * - loads Annotations and its corresponding processors and listeners classes from the given package name. 24 | * 25 | * @author rmuthupandian 26 | * 27 | */ 28 | public class AnnotationConfigLoader { 29 | 30 | private String listener_package = "com.ebay.jetstream.event.processor.esper.annotation.listener"; 31 | private String processor_package = "com.ebay.jetstream.event.processor.esper.annotation.processor"; 32 | 33 | public DefaultAnnotationConfiguration load_annotation_config() { 34 | DefaultAnnotationConfiguration defaultconfig = new DefaultAnnotationConfiguration(); 35 | List configList = new ArrayList(); 36 | 37 | loadProcessors(); 38 | loadListeners(); 39 | Map, AnnotationProcessor> processors = loadProcessors(); 40 | Map, AnnotationListener> listeners = loadListeners(); 41 | 42 | for (Map.Entry, AnnotationProcessor> procEntry : processors 43 | .entrySet()) { 44 | AnnotationConfiguration config = new AnnotationConfiguration(); 45 | config.setAnnotation(procEntry.getKey().getSimpleName()); 46 | config.setClassName(procEntry.getKey()); 47 | config.setProcessor(procEntry.getValue()); 48 | if (listeners.get(procEntry.getKey()) != null) 49 | config.setListener(listeners.get(procEntry.getKey())); 50 | 51 | configList.add(config); 52 | } 53 | 54 | defaultconfig.setDefaultAnnotationConfiguration(configList); 55 | 56 | return defaultconfig; 57 | } 58 | 59 | private Map, AnnotationProcessor> loadProcessors() { 60 | 61 | Reflections reflections = new Reflections(processor_package); 62 | Set> proc_classes = reflections 63 | .getSubTypesOf(AnnotationProcessor.class); 64 | 65 | Map, AnnotationProcessor> procMap = new HashMap, AnnotationProcessor>(); 66 | for (Class procClass : proc_classes) { 67 | try { 68 | Class[] clzargs = null ; 69 | Method m = procClass.getDeclaredMethod( 70 | "getAnnotationClass", clzargs ); 71 | AnnotationProcessor processor = (AnnotationProcessor) procClass 72 | .newInstance(); 73 | Object[] objargs = null ; 74 | Class clz = (Class) m.invoke(processor, objargs ); 75 | if (clz != null) 76 | procMap.put(clz, processor); 77 | 78 | } catch (Throwable t) { 79 | t.printStackTrace(); 80 | } 81 | } 82 | return procMap; 83 | 84 | } 85 | 86 | private Map, AnnotationListener> loadListeners() { 87 | 88 | Reflections reflections = new Reflections(listener_package); 89 | Set> listener_classes = reflections 90 | .getSubTypesOf(AnnotationListener.class); 91 | 92 | Map, AnnotationListener> listenerMap = new HashMap, AnnotationListener>(); 93 | for (Class listenerClass : listener_classes) { 94 | try { 95 | 96 | Class[] clzargs = null ; 97 | Method m = listenerClass.getDeclaredMethod( 98 | "getAnnotationClass", clzargs ); 99 | AnnotationListener listener = (AnnotationListener) listenerClass 100 | .newInstance(); 101 | Object[] objargs = null ; 102 | Class clz = (Class) m.invoke(listenerClass.newInstance(), objargs ); 103 | if (clz != null) 104 | listenerMap.put(clz, listener); 105 | 106 | } catch (Throwable t) { 107 | t.printStackTrace(); 108 | } 109 | } 110 | return listenerMap; 111 | 112 | } 113 | 114 | } 115 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EsperExceptionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.text.DateFormat; 9 | import java.text.SimpleDateFormat; 10 | import java.util.Date; 11 | import java.util.concurrent.atomic.AtomicBoolean; 12 | 13 | import org.springframework.beans.factory.InitializingBean; 14 | import org.springframework.jmx.export.annotation.ManagedOperation; 15 | import org.springframework.jmx.export.annotation.ManagedResource; 16 | 17 | import com.ebay.jetstream.config.AbstractNamedBean; 18 | import com.ebay.jetstream.counter.LongCounter; 19 | import com.ebay.jetstream.management.Management; 20 | import com.ebay.jetstream.xmlser.XSerializable; 21 | import com.espertech.esper.client.hook.ExceptionHandler; 22 | import com.espertech.esper.client.hook.ExceptionHandlerContext; 23 | 24 | /** 25 | * This class will be called by EsperEngine when there is RuntimeException 26 | * thrown during EPL statement processing. Last Exception will be set for 27 | * debugging purposes. It also counts number of exceptions thrown per second 28 | * with EWMA counter. Based on the exception count, action can be taken. 29 | * 30 | * @author rmuthupandian 31 | * 32 | */ 33 | 34 | @ManagedResource(objectName = "Event/Processor", description = "Esper Exception Handler") 35 | public class EsperExceptionHandler extends AbstractNamedBean implements 36 | ExceptionHandler, XSerializable, InitializingBean { 37 | 38 | private String lastException; 39 | private LongCounter num_exception = new LongCounter(); 40 | private LongCounter exceptionTimeStamp = new LongCounter(); 41 | private AtomicBoolean exceptionStatus = new AtomicBoolean(false); 42 | private AtomicBoolean alertPosted = new AtomicBoolean(false); 43 | DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.sss'Z'"); 44 | Date m_date = new Date(); 45 | private String m_lastExceptionTime; 46 | 47 | public String getLastExceptionTime() { 48 | return m_lastExceptionTime; 49 | } 50 | 51 | public void setLastExceptionTime(String m_lastExceptionTime) { 52 | this.m_lastExceptionTime = m_lastExceptionTime; 53 | } 54 | 55 | //This method is just for exposing to monitoring page 56 | public boolean getAlertPostedStatus() { 57 | return alertPosted.get(); 58 | } 59 | 60 | public boolean isAlertPosted(){ 61 | return alertPosted.get(); 62 | } 63 | 64 | public void setAlertPostedStatus(boolean status){ 65 | alertPosted.set(status); 66 | } 67 | 68 | public boolean IsInException(){ 69 | return exceptionStatus.get(); 70 | } 71 | 72 | //This method is just for exposing to monitoring page - it demands method name to start with getXX 73 | public boolean getIsInException(){ 74 | return exceptionStatus.get(); 75 | } 76 | 77 | public String getLastException() { 78 | return lastException; 79 | } 80 | 81 | public void setLastException(String exception) { 82 | lastException = exception; 83 | } 84 | 85 | @Override 86 | public void handle(ExceptionHandlerContext context) { 87 | num_exception.increment(); 88 | m_date.setTime(System.currentTimeMillis()); 89 | setLastExceptionTime(formatter.format(m_date)); 90 | setLastException(context.getThrowable().getMessage() + " , EPL :" + context.getEpl()); 91 | 92 | if (num_exception.get() == 1) { 93 | exceptionTimeStamp.set( System.currentTimeMillis()); 94 | } 95 | else { 96 | if( (System.currentTimeMillis() - exceptionTimeStamp.get()) > 10000) { 97 | exceptionStatus.set(true); 98 | alertPosted.set(false); 99 | exceptionTimeStamp.set(0); 100 | num_exception.set(0); 101 | } 102 | } 103 | } 104 | 105 | public LongCounter getNumExceptions() { 106 | return num_exception; 107 | } 108 | 109 | 110 | @Override 111 | public void afterPropertiesSet() throws Exception { 112 | Management.addBean(getBeanName(), this); 113 | } 114 | 115 | @ManagedOperation 116 | public void clearAlertStatus(){ 117 | exceptionStatus.set(false); 118 | alertPosted.set(false); 119 | exceptionTimeStamp.set(0); 120 | num_exception.set(0); 121 | } 122 | 123 | public void setExceptionStatus(boolean status) { 124 | exceptionStatus.set(status); 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/epl/EPLUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import java.io.IOException; 9 | import java.io.StringWriter; 10 | import java.io.Writer; 11 | import java.security.SecureRandom; 12 | import java.sql.Timestamp; 13 | import java.util.Date; 14 | import java.util.Map; 15 | import java.util.Random; 16 | 17 | import org.codehaus.jackson.JsonGenerationException; 18 | import org.codehaus.jackson.JsonParseException; 19 | import org.codehaus.jackson.map.JsonMappingException; 20 | import org.codehaus.jackson.map.ObjectMapper; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import com.ebay.jetstream.event.JetstreamEvent; 25 | import com.ebay.jetstream.event.JetstreamReservedKeys; 26 | 27 | public final class EPLUtils { 28 | 29 | private static final Random m_rand = new SecureRandom(); 30 | private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.jetstream.epl.EPLUtils"); 31 | private static volatile int m_nErrorCount; 32 | 33 | private EPLUtils() { 34 | // util class 35 | } 36 | 37 | public static String toJsonString(Object obj) throws JsonGenerationException, JsonMappingException, IOException { 38 | return EPLUtils.toJsonString(obj, false); 39 | } 40 | 41 | public static Timestamp toTimestamp(Object objInParam) { 42 | Timestamp ts = null; 43 | if (objInParam instanceof Timestamp) 44 | ts = (Timestamp)objInParam; 45 | else if (objInParam instanceof Number) 46 | ts = new Timestamp(((Number)objInParam).longValue()); 47 | else if (objInParam instanceof Date) 48 | ts = new Timestamp(((Date)objInParam).getTime()); 49 | else if (objInParam instanceof String) { 50 | if ("now".equalsIgnoreCase((String)objInParam)) 51 | ts = new Timestamp(System.currentTimeMillis()); 52 | else 53 | ts = Timestamp.valueOf((String)objInParam); 54 | } 55 | else if (objInParam == null) 56 | ts = new Timestamp(System.currentTimeMillis()); 57 | return ts; 58 | } 59 | 60 | public static String toJsonString(Object obj, boolean bStripJsReservedWords) throws JsonGenerationException, JsonMappingException, IOException { 61 | String strResult = null; 62 | if (obj != null) { 63 | 64 | if (bStripJsReservedWords && obj instanceof JetstreamEvent) 65 | obj = ((JetstreamEvent)obj).getFilteredEvent(); 66 | 67 | try { 68 | ObjectMapper mapper = new ObjectMapper(); 69 | Writer writer = new StringWriter(); 70 | mapper.writeValue(writer, obj); 71 | strResult = writer.toString(); 72 | } 73 | catch (Throwable t) { 74 | if (m_nErrorCount++ % 10000 == 0 && LOGGER.isErrorEnabled()) 75 | LOGGER.error( "", t); 76 | } 77 | } 78 | return strResult; 79 | } 80 | 81 | public static long getRandomLong(Object objDontCache) { 82 | return Math.abs(m_rand.nextLong()); 83 | } 84 | 85 | public static Class getClass(Object obj) { 86 | return obj != null ? obj.getClass() : null; 87 | } 88 | 89 | public static Map fromJsonString(String strJson) throws JsonParseException, IOException { 90 | Map mapOut = null; 91 | if (strJson != null) { 92 | ObjectMapper mapper = new ObjectMapper(); 93 | mapOut = mapper.readValue(strJson, Map.class); 94 | } 95 | return mapOut; 96 | } 97 | 98 | public static long generateEventId(Map event) { 99 | long lEventId = 0; 100 | if (event != null) { 101 | Number numIpHost = (Number)event.get("host"); 102 | if (numIpHost != null) 103 | lEventId = JetstreamSideEventIdGenerator.generateEventId(numIpHost.intValue()); 104 | } 105 | if (lEventId == 0 && event.get("id") instanceof Number) 106 | lEventId = ((Number)event.get("id")).longValue(); 107 | return lEventId; 108 | } 109 | 110 | public static String getEventOrigin(Map mapInput) { 111 | String strOrigin = null; 112 | if (mapInput != null) 113 | strOrigin = (String)mapInput.get(JetstreamReservedKeys.EventOrigin.toString()); 114 | return strOrigin != null ? strOrigin : "unknown"; 115 | } 116 | 117 | public static String getEventType(Map mapInput) { 118 | String strType = null; 119 | if (mapInput != null) 120 | strType = (String)mapInput.get(JetstreamReservedKeys.EventType.toString()); 121 | return strType != null ? strType : "unknown"; 122 | } 123 | 124 | public static String getHostName() { 125 | String host = "unknownhost"; 126 | try { 127 | host = java.net.InetAddress.getLocalHost().getHostName(); 128 | } catch (java.net.UnknownHostException e) { 129 | 130 | } 131 | return host; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/epl/sampleEPL.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /* ---------- Routing for RTD events ----------- */ 5 | /* If ip is present, notify ip stream */ 6 | INSERT INTO RTD_IP_STREAM 7 | SELECT ip, rid, Math.round(cost/1000) as cost, EPLUtils.getEventOrigin(*) as res_type 8 | FROM RTD(ip is not null and uid is null and cguid is null); 9 | 10 | select field_0, count(field_0) as googlebotcount from RTD where field_0 = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)' group by field_0 output snapshot when terminated order by count(field_0) desc; 11 | 12 | /* If userid is present, notify user stream */ 13 | INSERT INTO RTD_USER_STREAM 14 | SELECT uid, rid, Math.round(cost/1000) as cost, EPLUtils.getEventOrigin(*) as res_type 15 | FROM RTD(ip is not null and uid is not null); 16 | 17 | /* If cguid is present, send to cguid stream */ 18 | INSERT INTO RTD_GUID_STREAM 19 | SELECT cguid, rid, Math.round(cost/1000) as cost, EPLUtils.getEventOrigin(*) as res_type 20 | FROM RTD(ip is not null and cguid is not null); 21 | 22 | /* ---------- Routing for RTDBLK events ----------- */ 23 | /* If ip is present, notify ip stream */ 24 | INSERT INTO RTDBLK_IP_STREAM 25 | SELECT ip, uid, cguid, rid, flag, host, EPLUtils.getEventOrigin(*) as res_type 26 | FROM RTDBLK(ip is not null and sid=1); 27 | 28 | /* If userid is present, notify user stream */ 29 | INSERT INTO RTDBLK_USER_STREAM 30 | SELECT ip, uid, cguid, rid, flag, host, EPLUtils.getEventOrigin(*) as res_type 31 | FROM RTDBLK(ip is not null and sid=2); 32 | 33 | /* If cguid is present, send to cguid stream */ 34 | INSERT INTO RTDBLK_GUID_STREAM 35 | SELECT ip, uid, cguid, rid, flag, host, EPLUtils.getEventOrigin(*) as res_type 36 | FROM RTDBLK(ip is not null and sid=3); 37 | 38 | /* the pattern fires every 2 minutes. i.e. to send an TABLE_CHECKER_STREAM event from each ER node 39 | With affinity key, one and only one RTD node will receive the envents{heartbeat=1} 40 | ***/ 41 | INSERT INTO TABLE_CHECKER_STREAM 42 | SELECT EPLUtils.getHostName() as sourcehost, 43 | 1 as heartbeat FROM pattern [every timer:interval(120 sec)]; 44 | @ClusterAffinityTag(colname="heartbeat") 45 | @OutputTo("outboundMessageChannel") 46 | SELECT * FROM TABLE_CHECKER_STREAM; 47 | 48 | /* ------- Affinity association for RTD -------*/ 49 | @ClusterAffinityTag(colname="ip") 50 | @OutputTo("outboundMessageChannel") 51 | SELECT * FROM RTD_IP_STREAM; 52 | 53 | @ClusterAffinityTag(colname="uid") 54 | @OutputTo("outboundMessageChannel") 55 | SELECT * FROM RTD_USER_STREAM; 56 | 57 | @ClusterAffinityTag(colname="cguid") 58 | @OutputTo("outboundMessageChannel") 59 | SELECT * FROM RTD_GUID_STREAM; 60 | 61 | /* ------- Affinity association for RTDBLK -------*/ 62 | @ClusterAffinityTag(colname="ip") 63 | @OutputTo("outboundMessageChannel") 64 | SELECT * FROM RTDBLK_IP_STREAM; 65 | 66 | @ClusterAffinityTag(colname="uid") 67 | @OutputTo("outboundMessageChannel") 68 | SELECT * FROM RTDBLK_USER_STREAM; 69 | 70 | @ClusterAffinityTag(colname="cguid") 71 | @OutputTo("outboundMessageChannel") 72 | SELECT * FROM RTDBLK_GUID_STREAM; 73 | 74 | /* --- Create RAW_EVENT_STREAM for RTD ---- */ 75 | INSERT INTO RAW_EVENT_LOG 76 | SELECT cguid, ip, id, 77 | EPLUtils.getEventType(*) as EVENT_TYPE, 78 | EPLUtils.getEventOrigin(*) AS EVENT_SOURCE, 79 | EPLUtils.toTimestamp(current_timestamp) AS EVENT_TIME, 80 | toJson(*) AS RAW_EVENT 81 | FROM RTD(ip is not null); 82 | 83 | /* --- Create RAW_EVENT_STREAM for RTDBLK ---- */ 84 | INSERT INTO RAW_EVENT_LOG 85 | SELECT cguid, ip, id, 86 | EPLUtils.getEventType(*) as EVENT_TYPE, 87 | EPLUtils.getEventOrigin(*) AS EVENT_SOURCE, 88 | EPLUtils.toTimestamp(current_timestamp) AS EVENT_TIME, 89 | toJson(*) AS RAW_EVENT 90 | FROM RTDBLK(ip is not null and sid is not null); 91 | 92 | /* Write the raw event log */ 93 | @OutputTo("DbWriter") 94 | @DBInfo(mode=WriteMode.INSERT_ONLY) 95 | SELECT coalesce(cguid, ip) as UUID, 96 | id AS EVENT_ID, 97 | ip AS CLIENT_IP, 98 | EVENT_TYPE, 99 | EVENT_SOURCE, 100 | EVENT_TIME, 101 | RAW_EVENT, 102 | EPLUtils.toTimestamp(current_timestamp + 86400000) as EXPIRATION_TIME 103 | FROM RAW_EVENT_LOG; 104 | 105 | 106 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/advice/EsperProcessorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper.advice; 7 | 8 | import static org.junit.Assert.assertNotNull; 9 | import static org.junit.Assert.assertTrue; 10 | import static org.mockito.Mockito.atLeastOnce; 11 | import static org.mockito.Mockito.mock; 12 | import static org.mockito.Mockito.verify; 13 | 14 | import org.junit.BeforeClass; 15 | import org.junit.Test; 16 | 17 | import com.ebay.jetstream.config.ApplicationInformation; 18 | import com.ebay.jetstream.config.Configuration; 19 | import com.ebay.jetstream.config.RootConfiguration; 20 | import com.ebay.jetstream.event.JetstreamEvent; 21 | import com.ebay.jetstream.event.RetryEventCode; 22 | import com.ebay.jetstream.event.advice.Advice; 23 | import com.ebay.jetstream.event.processor.esper.EsperProcessor; 24 | import com.ebay.jetstream.event.support.AbstractEventProcessor; 25 | 26 | public class EsperProcessorTest { 27 | 28 | 29 | private static final Configuration s_springConfiguration = new RootConfiguration(new ApplicationInformation( 30 | "EsperProcessorTest", "0.0"), new String[] { "src/test/java/com/ebay/jetstream/event/processor/esper/advice/EsperConfig.xml" }); 31 | 32 | private static EsperProcessor processor = (EsperProcessor) s_springConfiguration.getBean("EsperProcessor1"); 33 | 34 | private static AbstractEventProcessor espersink = (AbstractEventProcessor) s_springConfiguration.getBean("espersink"); 35 | 36 | 37 | @BeforeClass 38 | public static void setUp(){ 39 | 40 | } 41 | 42 | 43 | public void sendEvent() { 44 | processor.sendEvent(createEvent()); 45 | try { 46 | Thread.sleep(200); 47 | } catch (InterruptedException e) { 48 | e.printStackTrace(); 49 | } 50 | } 51 | 52 | public void sendBadEvent() { 53 | processor.sendEvent(createBadEvent()); 54 | try { 55 | Thread.sleep(200); 56 | } catch (InterruptedException e) { 57 | e.printStackTrace(); 58 | } 59 | } 60 | 61 | public void sendBadEvent(JetstreamEvent event) { 62 | processor.sendEvent(event); 63 | try { 64 | Thread.sleep(200); 65 | } catch (InterruptedException e) { 66 | e.printStackTrace(); 67 | } 68 | } 69 | 70 | @Test 71 | public void testEPLDynamicUpdate(){ 72 | 73 | 74 | } 75 | 76 | @Test 77 | public void testExceptionHandlerWithoutListner(){ 78 | assertNotNull(processor.getEsperExceptionHandler()); 79 | 80 | processor.setAdviceListener(null); // removing adviceListener 81 | 82 | sendBadEvent(); 83 | JetstreamEvent event = createBadEvent(); 84 | sendBadEvent(event); 85 | try { 86 | Thread.sleep(10000); // sleep for exception wait time window 87 | } catch (InterruptedException e) { 88 | e.printStackTrace(); 89 | } 90 | sendBadEvent(event); 91 | sendBadEvent(event); 92 | 93 | assertNotNull(processor.getEsperExceptionHandler().getLastException()); 94 | assertTrue(processor.getTotalEventsDropped() != 0); 95 | 96 | 97 | } 98 | 99 | 100 | @Test 101 | public void testExceptionHandlerWithAdviceLr(){ 102 | 103 | processor.getEsperExceptionHandler().clearAlertStatus(); // This is needed , since previous testcases might have set the alert status. 104 | 105 | Advice mockListner = mock(Advice.class); 106 | processor.setAdviceListener(mockListner); 107 | 108 | assertNotNull(processor.getEsperExceptionHandler()); 109 | assertNotNull(processor.getAdviceListener()); 110 | 111 | JetstreamEvent event = createBadEvent(); 112 | sendBadEvent(event); 113 | try { 114 | Thread.sleep(10000); // sleep for exception wait time window 115 | } catch (InterruptedException e) { 116 | e.printStackTrace(); 117 | } 118 | sendBadEvent(event); 119 | sendBadEvent(event); 120 | 121 | assertNotNull(processor.getEsperExceptionHandler().getLastException()); 122 | 123 | //verify AdviseListener is called atleast once to post the msg 124 | verify(mockListner, atLeastOnce()).retry(event, RetryEventCode.MSG_RETRY, processor.getEsperExceptionHandler().getLastException()); 125 | verify(mockListner, atLeastOnce()).stopReplay(); 126 | 127 | } 128 | 129 | 130 | 131 | private JetstreamEvent createEvent(){ 132 | JetstreamEvent event = new JetstreamEvent(); 133 | event.setEventType("TestEvent"); 134 | event.put("id", 11212); 135 | event.put("cost", 11212); 136 | return event; 137 | } 138 | 139 | 140 | private JetstreamEvent createBadEvent(){ 141 | JetstreamEvent event = new JetstreamEvent(); 142 | event.setEventType("BadEvent"); 143 | event.put("cost", 1234); 144 | event.put("id", 12345678900644L); 145 | return event; 146 | } 147 | 148 | 149 | 150 | } 151 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/EPL.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.io.Serializable; 9 | import java.util.ArrayList; 10 | import java.util.Collections; 11 | import java.util.List; 12 | import java.util.regex.Matcher; 13 | import java.util.regex.Pattern; 14 | 15 | import org.springframework.beans.factory.InitializingBean; 16 | import org.springframework.jmx.export.annotation.ManagedResource; 17 | 18 | import com.ebay.jetstream.config.AbstractNamedBean; 19 | import com.ebay.jetstream.event.JetstreamEvent; 20 | import com.ebay.jetstream.management.Management; 21 | import com.ebay.jetstream.xmlser.Hidden; 22 | import com.ebay.jetstream.xmlser.XSerializable; 23 | import com.espertech.esper.client.EPRuntime; 24 | 25 | @ManagedResource(objectName = "Event/Processor", description = "Esper Event Processor Langguage") 26 | public class EPL extends AbstractNamedBean implements Serializable, XSerializable, InitializingBean { 27 | 28 | private static final long serialVersionUID = 1L; 29 | 30 | private final List m_statements = new ArrayList(); 31 | 32 | private final List m_statementsWithComments = new ArrayList(); 33 | 34 | private List m_onDemandQueries; 35 | 36 | public void afterPropertiesSet() throws Exception { 37 | Management.removeBeanOrFolder(getBeanName(), this); 38 | Management.addBean(getBeanName(), this); 39 | } 40 | 41 | public List getOnDemandQueries() { 42 | return m_onDemandQueries; 43 | } 44 | 45 | /** 46 | * Gets the EPL statements as a single string block, such as a CDATA block. 47 | * 48 | * @return the set of all statements, in order, each separated by semicolon and new-line. 49 | */ 50 | 51 | public List getEPLStatements() { 52 | return Collections.unmodifiableList(m_statements); 53 | } 54 | 55 | /** 56 | * Gets the list of EPL statements. 57 | * 58 | * @return the list of all statements. 59 | */ 60 | @Hidden 61 | public List getStatements() { 62 | return Collections.unmodifiableList(m_statements); 63 | } 64 | 65 | public void processOnDemandQueries(EPRuntime runtime, int workerId, JetstreamEvent originalEvent) { 66 | List onDemandQueries = getOnDemandQueries(); 67 | if (onDemandQueries != null) { 68 | for (OnDemandQuery onDemandQuery : onDemandQueries) { 69 | onDemandQuery.process(runtime, workerId, originalEvent); 70 | } 71 | } 72 | } 73 | 74 | public void setOnDemandQueries(List onDemandQueries) { 75 | m_onDemandQueries = onDemandQueries; 76 | if (onDemandQueries != null) { 77 | for (OnDemandQuery onDemandQuery : onDemandQueries) { 78 | if (onDemandQuery == null) { 79 | throw new IllegalArgumentException("onDemandQueries bean must contain at least one 'OnDemandQuery' bean"); 80 | } 81 | onDemandQuery.validate(); 82 | } 83 | } 84 | } 85 | 86 | /** 87 | * Parses a single String containing a semicolon delimited list of EPL statements. 88 | * 89 | * @param statementBlock 90 | * the String containing all statements, with each statement separated by a semicolon. 91 | */ 92 | 93 | public void setStatementBlock(String statementBlock) { 94 | // TODO: better stmt delimiter parsing (e.g. skip ';' in a string or comment) 95 | m_statements.clear(); 96 | m_statementsWithComments.clear(); 97 | String noncomment = ""; 98 | 99 | /** 100 | * This pattern removes statements with comments. 101 | */ 102 | if(statementBlock.contains("/*")){ 103 | Pattern p = Pattern.compile("/\\*(.*?)\\*/", Pattern.MULTILINE | Pattern.DOTALL); 104 | Matcher m = p.matcher(statementBlock); 105 | while (m.find()) { 106 | noncomment = m.replaceAll(""); 107 | } 108 | }else { 109 | noncomment = statementBlock ; 110 | } 111 | 112 | 113 | String pattern = "(?s);(?=(?:(?:.*?(? 0) { 118 | m_statements.add(trimmed); 119 | } 120 | } 121 | } 122 | 123 | /** 124 | * Sets the list of EPL statements. 125 | * 126 | * @param statements 127 | * the list of EPL statements. 128 | */ 129 | public void setStatements(List statements) { 130 | m_statements.clear(); 131 | m_statements.addAll(statements); 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/advice/EsperConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 39 | 40 | 41 | 42 | 43 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 127 | 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /jetstream-esper/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | com.ebay.jetstream 11 | jetstream-parent 12 | 4.1.1-SNAPSHOT 13 | 14 | 15 | 4.0.0 16 | jetstream-esper 17 | 4.1.1-SNAPSHOT 18 | jar 19 | jetstream esper 20 | 21 | 22 | 4.1.1-SNAPSHOT 23 | 24 | 25 | 26 | 27 | GNU GENERAL PUBLIC LICENSE Version 2, June 1991 28 | http://www.gnu.org/licenses/gpl-2.0.txt 29 | repo 30 | 31 | 32 | 33 | 34 | scm:git:git@github.com:pulsarIO/jetstream-esper.git 35 | scm:git:git@github.com:pulsarIO/jetstream-esper.git 36 | scm:git:git@github.com:pulsarIO/jetstream-esper.git 37 | HEAD 38 | 39 | 40 | 41 | 42 | com.espertech 43 | esper 44 | 5.2.0 45 | 46 | 47 | 48 | com.espertech 49 | esper 50 | 5.2.0 51 | sources 52 | 53 | 54 | 55 | joda-time 56 | joda-time 57 | 2.7 58 | 59 | 60 | 61 | commons-jxpath 62 | commons-jxpath 63 | 1.3 64 | 65 | 66 | 67 | com.ebay.jetstream 68 | jetstreamframework 69 | ${jetstream.version} 70 | 71 | 72 | org.reflections 73 | reflections 74 | 0.9.9-RC2 75 | 76 | 77 | junit 78 | junit 79 | 4.12 80 | test 81 | 82 | 83 | org.mockito 84 | mockito-all 85 | 1.9.5 86 | test 87 | 88 | 89 | 90 | 91 | ${project.artifactId}.${project.version}.r${buildNumber} 92 | 93 | 94 | org.apache.maven.plugins 95 | maven-jar-plugin 96 | 97 | 98 | ${project.build.outputDirectory}/META-INF/MANIFEST.MF 99 | 100 | 101 | 102 | 103 | org.apache.maven.plugins 104 | maven-source-plugin 105 | 106 | 107 | ${project.build.outputDirectory}/META-INF/MANIFEST.MF 108 | 109 | 110 | 111 | 112 | attach-sources 113 | 114 | jar 115 | 116 | 117 | 118 | 119 | 120 | 121 | org.apache.maven.plugins 122 | maven-release-plugin 123 | 2.4.1 124 | 125 | 126 | org.apache.maven.scm 127 | maven-scm-provider-gitexe 128 | 1.8.1 129 | 130 | 131 | 132 | true 133 | false 134 | release 135 | deploy 136 | 137 | 138 | 140 | 141 | org.apache.felix 142 | maven-bundle-plugin 143 | 144 | 145 | 146 | 147 | 148 | 149 | <_nouses>true 150 | * 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/esper/annotations/EsperAnnotationTest.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | java.lang.Math 68 | com.ebay.jetstream.epl.EPLUtilities 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/esper/customaggregate/ESPAggrTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.esper.customaggregate; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import java.lang.Thread.UncaughtExceptionHandler; 11 | import java.util.ArrayList; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.concurrent.ThreadFactory; 15 | import java.util.concurrent.TimeUnit; 16 | 17 | import org.junit.Test; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | import org.springframework.beans.factory.InitializingBean; 21 | import org.springframework.context.ApplicationEvent; 22 | import org.springframework.context.ApplicationEventPublisher; 23 | import org.springframework.context.ApplicationEventPublisherAware; 24 | import org.springframework.context.ApplicationListener; 25 | import org.springframework.jmx.export.annotation.ManagedResource; 26 | 27 | import com.ebay.jetstream.config.ApplicationInformation; 28 | import com.ebay.jetstream.config.Configuration; 29 | import com.ebay.jetstream.config.RootConfiguration; 30 | import com.ebay.jetstream.event.EventException; 31 | import com.ebay.jetstream.event.EventSink; 32 | import com.ebay.jetstream.event.JetstreamEvent; 33 | import com.ebay.jetstream.event.processor.esper.EPL; 34 | import com.ebay.jetstream.event.processor.esper.EsperConfiguration; 35 | import com.ebay.jetstream.event.processor.esper.EsperProcessor; 36 | import com.ebay.jetstream.event.support.AbstractEventSource; 37 | import com.ebay.jetstream.event.support.channel.PipelineFlowControl; 38 | 39 | /** 40 | * This test needs 'JETSTREAM_HOME' variable to be set to something like D:\path\EventProcessor 41 | * 42 | * @author snikolaev 43 | * 44 | */ 45 | @ManagedResource(objectName = "ESPTest", description = "EsperEventProcessor Unit Test") 46 | public class ESPAggrTest extends AbstractEventSource implements ApplicationListener, InitializingBean, ApplicationEventPublisherAware { 47 | 48 | static class ESPTestThreadFactory implements ThreadFactory { 49 | 50 | public Thread newThread(Runnable r) { 51 | Thread t = new Thread(r); 52 | t.setUncaughtExceptionHandler(new UncaughtExceptionHandler() { 53 | public void uncaughtException(Thread t, Throwable e) { 54 | final Logger logger = LoggerFactory.getLogger("com.ebay.rdbd.queueworker"); 55 | logger.error( "Exception \'" + e.getMessage() + "\' in thread " + t.getId()); 56 | } 57 | }); 58 | return t; 59 | } 60 | } 61 | 62 | private static final Logger testLogger = LoggerFactory.getLogger("com.ebay.jetstream.event.esper.customaggregate.ESPAggrTest"); 63 | private static final int THREADS_NUM = 2; 64 | private static final int THREADS_NUM_AGGREGATION = 30; 65 | private static final Configuration s_springConfiguration = new RootConfiguration(new ApplicationInformation( 66 | "ESPTest", "0.0"), new String[] { "src/test/java/com/ebay/jetstream/event/esper/customaggregate/ESPAggrTest.xml" }); 67 | 68 | static { 69 | org.apache.log4j.xml.DOMConfigurator.configure("src/test/java/com/ebay/jetstream/event/processor/esper/log4j.xml"); 70 | } 71 | 72 | private final PipelineFlowControl m_flowHandler = new PipelineFlowControl(this); 73 | 74 | public ESPAggrTest() { 75 | } 76 | 77 | public void afterPropertiesSet() throws Exception { 78 | 79 | } 80 | 81 | 82 | private JetstreamEvent generateJetstreamEvent(int id) { 83 | HashMap map = new HashMap(); 84 | String field1 = "123.0"; 85 | map.put("id", id); 86 | map.put("field1", field1); 87 | return new JetstreamEvent("ESPTestEvent1", null, map); 88 | } 89 | 90 | private EPL getEpl(String name) { 91 | return (EPL) s_springConfiguration.getBean(name); 92 | } 93 | 94 | private EsperConfiguration getEsperConfiguration() { 95 | return (EsperConfiguration) s_springConfiguration.getBean("EsperConfiguration"); 96 | } 97 | 98 | private EsperProcessor getProcessor(String name) { 99 | return (EsperProcessor) s_springConfiguration.getBean(name); 100 | } 101 | 102 | @Override 103 | public void onApplicationEvent(ApplicationEvent event) { 104 | testLogger.warn( "ESPTest received " + event); 105 | } 106 | 107 | @Override 108 | public void pause() { 109 | // TODO Auto-generated method stub 110 | 111 | } 112 | 113 | @Override 114 | protected void processApplicationEvent(ApplicationEvent event) { 115 | // TODO Auto-generated method stub 116 | 117 | } 118 | 119 | @Override 120 | public void resume() { 121 | // TODO Auto-generated method stub 122 | 123 | } 124 | 125 | public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { 126 | m_flowHandler.setApplicationEventPublisher(applicationEventPublisher); 127 | // Management.addBean(this.getClass().getSimpleName(), this); 128 | } 129 | 130 | 131 | @Test 132 | public void testFireAndForget() throws InterruptedException { 133 | EsperProcessor processor = getProcessor("ESPTestProcessor1"); 134 | 135 | processor.addEventSink(new EventSink() { 136 | public String getBeanName() { 137 | // TODO Auto-generated method stub 138 | return null; 139 | } 140 | 141 | public void sendEvent(JetstreamEvent event) throws EventException { 142 | System.out.println("Receive event " + event); //KEEPME 143 | } 144 | }); 145 | 146 | List events = new ArrayList(); 147 | for (int i = 0; i < THREADS_NUM; i++) { 148 | events.add(generateJetstreamEvent(i)); 149 | 150 | } 151 | for (JetstreamEvent event : events) { 152 | TimeUnit.MILLISECONDS.sleep(1); 153 | processor.sendEvent(event); 154 | } 155 | Thread.sleep(2000); 156 | processor.stop(); 157 | 158 | //System.out.println(sink.getCount()); 159 | System.out.println(processor.getEsperEventListener().getEventReceived()); 160 | assertEquals(4, Integer.parseInt(processor.getEsperEventListener().getEventReceived().toString())); 161 | } 162 | 163 | 164 | } 165 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/epl/ClipBoard.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.epl; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | /** 12 | * @author shmurthy 13 | * 14 | * This class is intended for use in EPL as a temporary storage for building maps. The post() and get() methods 15 | * are intended for use with in EPL. The erase method must be called by the component sending an event to Esper. 16 | * it must be called immediately after the sendEvent() method of Esper returns. This class manages one or more 17 | * named clip boards. Each named ClipBoard is a hash map that contains key value pairs. The Named ClipBoards are 18 | * stored in a HashMap and the HashMap is erased upon every event is processed. 19 | * 20 | */ 21 | 22 | public class ClipBoard { 23 | 24 | private static ThreadLocal> m_clipBoardMap = new ThreadLocal>(); 25 | 26 | /** 27 | * erase() - is called to make sure that all the named clip boards are removed subsequent to the event being 28 | * processed. This way ownership of the named clip boards get passed to a downstream listener. This method is intended 29 | * to be called after the sendEvent() method for Esper runtime returns for every event submitted to the Esper runtime. 30 | */ 31 | 32 | public static void erase() { 33 | 34 | HashMap walls = (HashMap) m_clipBoardMap.get(); 35 | 36 | if (walls != null) { 37 | walls.clear(); 38 | } 39 | } 40 | 41 | /** 42 | * get() method is used to get a key value pair to a named ClipBoard. The workspace key is a dummy argument and must 43 | * be a field in the incoming event to Esper. Without this Esper will not execute the post method for more than one 44 | * event. 45 | * 46 | * @param eventKey 47 | * @param clipBoardName 48 | * @return a HashMap containing the key value pairs posted to the specified clip board 49 | */ 50 | 51 | @SuppressWarnings("unchecked") 52 | public static Map get(Object wallid, String clipBoardName) { 53 | 54 | String wall = null; 55 | if (wallid instanceof Long) { 56 | wall = wallid.toString(); 57 | } 58 | else if (wallid instanceof String) 59 | wall = wallid.toString(); 60 | 61 | HashMap walls = (HashMap) m_clipBoardMap.get(); 62 | 63 | if (walls == null) { 64 | walls = new HashMap(); 65 | m_clipBoardMap.set(walls); 66 | } 67 | 68 | HashMap wallClipBoards = (HashMap) walls.get(wall); 69 | 70 | if (wallClipBoards == null) { 71 | wallClipBoards = new HashMap(); 72 | walls.put(wall, wallClipBoards); 73 | } 74 | 75 | HashMap namedClipBoard = (HashMap) wallClipBoards.get(clipBoardName); 76 | 77 | if (namedClipBoard == null) { 78 | namedClipBoard = new HashMap(); 79 | wallClipBoards.put(clipBoardName, namedClipBoard); 80 | } 81 | 82 | return namedClipBoard; 83 | 84 | } 85 | 86 | /** 87 | * get() method is used to get a entry of a named ClipBoard entries by providing a keyName. The workspace key is a 88 | * dummy argument and must be a field in the incoming event to Esper. Without this Esper will not execute the post 89 | * method for more than one event. 90 | * 91 | * @param eventKey 92 | * @param clipBoardName 93 | * @param keyname 94 | * @return a HashMap containing the key value pairs posted to the specified clip board 95 | */ 96 | 97 | @SuppressWarnings("unchecked") 98 | public static Map get(Object wallid, String clipBoardName, String keyName) { 99 | return (Map) get(wallid, clipBoardName).get(keyName); 100 | } 101 | 102 | public static Object getAndRemovekey(Object wallid, String clipBoardName, String keyName) { 103 | HashMap namedClipBoard = (HashMap) get(wallid, clipBoardName); 104 | Object value = namedClipBoard.get(keyName); 105 | namedClipBoard.remove(keyName); 106 | return value; 107 | 108 | } 109 | 110 | /** 111 | * Post() method is used to post a key value pair to a named ClipBoard. The workspace key is a dummy argument and must 112 | * be a field in the incoming event to Esper. Without this Esper will not execute the post method for more than one 113 | * event. 114 | * 115 | * @param eventKey 116 | * @param clipBoardName 117 | * @param key 118 | * @param value 119 | * @return 120 | */ 121 | 122 | @SuppressWarnings("unchecked") 123 | public static Map post(Object wallid, String clipBoardName, String key, Object value) { 124 | 125 | // concept is we will have one or more walls containing one or more clipboards 126 | // first we will get the outer most HashMap which will hold one or more walls 127 | 128 | HashMap walls = (HashMap) m_clipBoardMap.get(); 129 | 130 | String wall = null; 131 | if (wallid instanceof Long) { 132 | wall = wallid.toString(); 133 | } 134 | else if (wallid instanceof String) 135 | wall = wallid.toString(); 136 | 137 | if (walls == null) { 138 | walls = new HashMap(); 139 | m_clipBoardMap.set(walls); 140 | } 141 | 142 | HashMap wallClipBoards = (HashMap) walls.get(wall); 143 | 144 | if (wallClipBoards == null) { 145 | wallClipBoards = new HashMap(); 146 | walls.put(wall, wallClipBoards); 147 | } 148 | 149 | // Now get the specified clipboard from the specified wall 150 | HashMap namedClipBoard = (HashMap) wallClipBoards.get(clipBoardName); 151 | 152 | if (namedClipBoard == null) { 153 | namedClipBoard = new HashMap(); 154 | wallClipBoards.put(clipBoardName, namedClipBoard); 155 | } 156 | 157 | namedClipBoard.put(key, value); 158 | return namedClipBoard; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /jetstream-esper/src/main/java/com/ebay/jetstream/event/processor/esper/OnDemandQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.jetstream.event.processor.esper; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import com.ebay.jetstream.event.JetstreamEvent; 15 | import com.ebay.jetstream.event.JetstreamReservedKeys; 16 | import com.ebay.jetstream.xmlser.XSerializable; 17 | import com.espertech.esper.client.EPOnDemandQueryResult; 18 | import com.espertech.esper.client.EPRuntime; 19 | import com.espertech.esper.client.EventBean; 20 | 21 | /** 22 | * On Demand query handler. See Esper docs for more details 23 | * 24 | * @author snikolaev 25 | * 26 | */ 27 | public class OnDemandQuery implements XSerializable { 28 | 29 | private static final String LOGGING_COMPONENT_NAME = "EsperEventListener"; 30 | private static final Logger logger = LoggerFactory.getLogger("com.ebay.jetstream.event.processor.esper.OnDemandQuery"); 31 | private static final EsperEventConverter DEFAULT_CONVERTER = new EsperWrappedEventConverter(); 32 | 33 | public static EsperEventConverter getEventConverter() { 34 | return DEFAULT_CONVERTER; 35 | } 36 | 37 | private String m_query; 38 | private String m_cleanUpEvent; 39 | private String m_outputEvent; 40 | private String m_workerIdentifier; 41 | 42 | public String getCleanUpEvent() { 43 | return m_cleanUpEvent; 44 | } 45 | 46 | public String getOutputEvent() { 47 | return m_outputEvent; 48 | } 49 | 50 | public String getQuery() { 51 | return m_query; 52 | } 53 | 54 | public String getWorkerIdentifier() { 55 | return m_workerIdentifier; 56 | } 57 | 58 | /** 59 | * Query processor. It does 3 things: 1) It executes the query set by 'query' property 2) It takes a result and sends 60 | * it as a NEW event set by 'outputEvent' property 3) It routes a NEW cleanup event set by 'cleanUpEvent' property 61 | * 62 | * @param runtime 63 | * @param workerId 64 | * @param originalEvent 65 | */ 66 | public void process(EPRuntime runtime, int workerId, JetstreamEvent originalEvent) { 67 | 68 | 69 | try { 70 | Object eventId = originalEvent.get(JetstreamReservedKeys.WorkerId.toString()); 71 | if (eventId == null || !(eventId instanceof Integer)) { 72 | logger.error( "EPL error: OnDemandQuery processor failed to retrieve " 73 | + JetstreamReservedKeys.WorkerId.toString() + " field from original event " + originalEvent 74 | + " because it's either null or not of Integer type", LOGGING_COMPONENT_NAME); 75 | } 76 | else { 77 | final Integer eventWorkerId = (Integer) eventId; 78 | 79 | EPOnDemandQueryResult result = runtime.executeQuery(getQuery()); 80 | for (EventBean row : result.getArray()) { 81 | Object id = row.get(getWorkerIdentifier()); 82 | if (id == null || !(id instanceof Integer)) { 83 | logger.warn( "EPL error: OnDemandQuery processor failed to convert " + id + " into " 84 | + getWorkerIdentifier() + " for EventBean " + row, LOGGING_COMPONENT_NAME); 85 | } 86 | Integer receivedWorkerId = (Integer) id; 87 | if (receivedWorkerId != null) { 88 | if(receivedWorkerId != workerId) { 89 | 90 | continue; 91 | } 92 | } 93 | JetstreamEvent jetstreamEvent = getEventConverter().getJetstreamEvent(row); 94 | if (jetstreamEvent == null) { 95 | logger.warn( "EPL error: OnDemandQuery processor ignored event of type " 96 | + row.getEventType() + ": " + row, LOGGING_COMPONENT_NAME); 97 | } 98 | else { 99 | if (logger.isDebugEnabled()) { 100 | logger.debug( "Sending " + getOutputEvent() + ": " + jetstreamEvent, 101 | LOGGING_COMPONENT_NAME); 102 | } 103 | runtime.sendEvent(jetstreamEvent, getOutputEvent()); 104 | } 105 | } 106 | Map cleanupEvent = new HashMap(); 107 | cleanupEvent.put(getWorkerIdentifier(), eventWorkerId); 108 | if (logger.isDebugEnabled()) { 109 | logger.debug( "Routing " + getCleanUpEvent() + ": " + cleanupEvent, LOGGING_COMPONENT_NAME); 110 | } 111 | runtime.sendEvent(cleanupEvent, getCleanUpEvent()); 112 | } 113 | } 114 | catch (Throwable e) { 115 | } 116 | finally { 117 | 118 | } 119 | } 120 | 121 | public void setCleanUpEvent(String cleanUpEvent) { 122 | m_cleanUpEvent = cleanUpEvent; 123 | } 124 | 125 | public void setOutputEvent(String outputEvent) { 126 | m_outputEvent = outputEvent; 127 | } 128 | 129 | public void setQuery(String query) { 130 | m_query = query; 131 | } 132 | 133 | public void setWorkerIdentifier(String workerIdentifier) { 134 | m_workerIdentifier = workerIdentifier; 135 | } 136 | 137 | public void validate() { 138 | String query = getQuery(); 139 | if (query != null) { 140 | query = query.trim(); 141 | } 142 | if (query == null || query.length() <= 0) { 143 | throw new IllegalArgumentException("No 'query' property for OnDemandQuery bean provided"); 144 | } 145 | setQuery(query); 146 | 147 | String cleanUpEvent = getCleanUpEvent(); 148 | if (cleanUpEvent != null) { 149 | cleanUpEvent = cleanUpEvent.trim(); 150 | } 151 | if (cleanUpEvent == null || cleanUpEvent.length() <= 0) { 152 | throw new IllegalArgumentException("No 'cleanUpEvent' property for OnDemandQuery bean provided"); 153 | } 154 | setCleanUpEvent(cleanUpEvent); 155 | 156 | String outputEvent = getOutputEvent(); 157 | if (outputEvent != null) { 158 | outputEvent = outputEvent.trim(); 159 | } 160 | if (outputEvent == null || outputEvent.length() <= 0) { 161 | throw new IllegalArgumentException("No 'outputEvent' property for OnDemandQuery bean provided"); 162 | } 163 | setOutputEvent(outputEvent); 164 | 165 | String workerIdentifier = getWorkerIdentifier(); 166 | if (workerIdentifier != null) { 167 | workerIdentifier = workerIdentifier.trim(); 168 | } 169 | if (workerIdentifier == null || workerIdentifier.length() <= 0) { 170 | throw new IllegalArgumentException("No 'workerIdentifier' property for OnDemandQuery bean provided"); 171 | } 172 | setWorkerIdentifier(workerIdentifier); 173 | } 174 | 175 | } 176 | -------------------------------------------------------------------------------- /jetstream-esper/src/test/java/com/ebay/jetstream/event/processor/esper/EsperStreamTest.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 12 | 13 | 22 | 25 | 27 | 28 | 29 | 30 | 40 | 41 | java.util.logging.ConsoleHandler, 42 | com.ebay.jetstream.logging.jetstreamLogAdapter 43 | 44 | 52 | INFO 53 | 60 | 63 | 64 | INFO 65 | 66 | 68 | java.util.logging.SimpleFormatter 69 | 70 | 72 | INFO 73 | 74 | 76 | java.util.logging.SimpleFormatter 77 | 78 | 88 | 89 | 90 | 91 | 92 | 94 | 95 | 96 | 97 | 99 | 100 | 101 | 103 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 113 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 123 | 125 | 126 | 127 | 129 | 131 | 133 | 135 | 137 | 139 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 157 | 158 | 159 | 160 | 161 | 163 | 164 | 165 | 166 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 192 | 193 | 194 | 195 | 197 | 198 | 199 | 200 | --------------------------------------------------------------------------------