├── .gitattributes ├── .gitignore ├── 3rdparty.md ├── CHANGELOG.md ├── Demo ├── .gitattributes ├── .gitignore ├── README.md ├── metricUI │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── assembly.xml │ ├── buildsrc │ │ ├── JetstreamConf │ │ │ ├── EPL.xml │ │ │ ├── EsperSetup.xml │ │ │ ├── messagecontext.xml │ │ │ ├── metricuiwiring.xml │ │ │ └── zookeeper.xml │ │ ├── scripts │ │ │ └── start.sh │ │ └── webapp │ │ │ ├── WEB-INF │ │ │ ├── pulsar-dispatcher-servlet.xml │ │ │ └── web.xml │ │ │ └── resources │ │ │ ├── css │ │ │ ├── bootstrap-theme.css.map │ │ │ ├── bootstrap-theme.min.css │ │ │ ├── bootstrap.css.map │ │ │ ├── bootstrap.min.css │ │ │ ├── flags.css │ │ │ ├── index.css │ │ │ └── nv.d3.min.css │ │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ └── opensans │ │ │ │ ├── bold │ │ │ │ ├── OpenSans-Bold.eot │ │ │ │ ├── OpenSans-Bold.svg │ │ │ │ ├── OpenSans-Bold.ttf │ │ │ │ └── OpenSans-Bold.woff │ │ │ │ ├── italic │ │ │ │ ├── opensans-italic.eot │ │ │ │ ├── opensans-italic.svg │ │ │ │ ├── opensans-italic.ttf │ │ │ │ └── opensans-italic.woff │ │ │ │ ├── light │ │ │ │ ├── opensans-light.eot │ │ │ │ ├── opensans-light.svg │ │ │ │ ├── opensans-light.ttf │ │ │ │ └── opensans-light.woff │ │ │ │ └── regular │ │ │ │ ├── opensans-regular.eot │ │ │ │ ├── opensans-regular.svg │ │ │ │ ├── opensans-regular.ttf │ │ │ │ └── opensans-regular.woff │ │ │ ├── imgs │ │ │ ├── blank.gif │ │ │ ├── favicon.ico │ │ │ ├── flags.png │ │ │ └── logo.png │ │ │ ├── index.html │ │ │ ├── js │ │ │ ├── controller │ │ │ │ ├── batchCtrl.js │ │ │ │ ├── realtimeCtrl.js │ │ │ │ └── twitterFeedCtrl.js │ │ │ ├── index.js │ │ │ ├── plugin │ │ │ │ ├── angular-resource.min.js │ │ │ │ ├── angular-resource.min.js.map │ │ │ │ ├── angular.min.js │ │ │ │ ├── angular.min.js.map │ │ │ │ ├── bootstrap.min.js │ │ │ │ ├── crossfilter.js │ │ │ │ ├── d3.min.js │ │ │ │ ├── hashmap.js │ │ │ │ ├── jquery-1.11.2.min.js │ │ │ │ ├── moment.min.js │ │ │ │ ├── ng-nvd3.js │ │ │ │ └── nv.d3.min.js │ │ │ └── service │ │ │ │ └── metricservice.js │ │ │ └── views │ │ │ ├── batch.html │ │ │ ├── realtime.html │ │ │ └── twitterFeed.html │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── ebay │ │ │ └── pulsar │ │ │ ├── metric │ │ │ ├── controller │ │ │ │ └── PulsarReportingController.java │ │ │ ├── data │ │ │ │ └── DataPoint.java │ │ │ ├── esper │ │ │ │ └── aggregator │ │ │ │ │ ├── TopNumReformAggregator.java │ │ │ │ │ └── TopNumReformAggregatorFactory.java │ │ │ ├── processor │ │ │ │ └── MetricProcessor.java │ │ │ ├── server │ │ │ │ ├── MetricDispatcherServlet.java │ │ │ │ ├── MetricServer.java │ │ │ │ └── SpringWebContextHolder.java │ │ │ └── util │ │ │ │ └── JsonUtil.java │ │ │ └── websocket │ │ │ ├── MetricWebSocket.java │ │ │ ├── MetricWebSocketCreator.java │ │ │ ├── MetricWebSocketServlet.java │ │ │ ├── PulsarEventConstant.java │ │ │ └── WebSocketConnectionManager.java │ │ └── resources │ │ └── logback.xml ├── metricservice │ ├── Dockerfile │ ├── README.md │ ├── assembly.xml │ ├── buildsrc │ │ ├── JetstreamConf │ │ │ ├── appwiring.xml │ │ │ ├── messagecontext.xml │ │ │ └── zookeeper.xml │ │ └── scripts │ │ │ └── start.sh │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── ebay │ │ │ └── pulsar │ │ │ └── metric │ │ │ ├── core │ │ │ ├── Counter.java │ │ │ ├── DataType.java │ │ │ ├── MetricsService.java │ │ │ ├── MetricsThreadFactory.java │ │ │ ├── NumericMetric.java │ │ │ ├── RawMetricMapper.java │ │ │ └── RawNumericMetric.java │ │ │ ├── impl │ │ │ └── cassandra │ │ │ │ ├── DataAccess.java │ │ │ │ ├── MetricsServiceCassandra.java │ │ │ │ └── QueryParam.java │ │ │ └── servlet │ │ │ ├── MetricRestServlet.java │ │ │ └── ServletStats.java │ │ └── resources │ │ └── logback.xml ├── pom.xml ├── rundemo.sh └── twittersample │ ├── Dockerfile │ ├── assembly.xml │ ├── buildsrc │ ├── JetstreamConf │ │ ├── appwiring.xml │ │ ├── messagecontext.xml │ │ └── zookeeper.xml │ └── scripts │ │ └── start.sh │ ├── pom.xml │ └── src │ └── main │ ├── java │ └── com │ │ └── ebay │ │ └── pulsar │ │ └── twittersample │ │ └── channel │ │ └── TwitterSampleChannel.java │ └── resources │ └── logback.xml ├── LICENSE ├── README.md ├── collector ├── Dockerfile ├── assembly.xml ├── buildsrc │ ├── JetstreamConf │ │ ├── EPL.xml │ │ ├── EsperSetup.xml │ │ ├── appwiring.xml │ │ ├── messagecontext.xml │ │ ├── replayconfig.xml │ │ └── zookeeper.xml │ ├── data │ │ ├── Event.txt │ │ ├── device.txt │ │ ├── ip.txt │ │ ├── item.txt │ │ ├── site.txt │ │ ├── ts.txt │ │ └── userAgent.txt │ └── scripts │ │ └── start.sh ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── ebay │ │ │ └── pulsar │ │ │ └── collector │ │ │ ├── servlet │ │ │ ├── IngestServlet.java │ │ │ └── ServletStats.java │ │ │ ├── simulator │ │ │ └── Simulator.java │ │ │ ├── udf │ │ │ ├── DeviceEnrichmentUtil.java │ │ │ ├── DeviceInfo.java │ │ │ ├── GeoEnrichmentUtil.java │ │ │ └── GeoInfo.java │ │ │ └── validation │ │ │ ├── DefaultValidator.java │ │ │ ├── ValidationResult.java │ │ │ └── Validator.java │ └── resources │ │ └── logback.xml │ └── test │ └── java │ └── com │ └── ebay │ └── pulsar │ └── collector │ ├── EnrichmentSimulator.java │ └── Simulator.java ├── distributor ├── Dockerfile ├── assembly.xml ├── buildsrc │ ├── JetstreamConf │ │ ├── EPL.xml │ │ ├── EsperSetup.xml │ │ ├── distributorwiring.xml │ │ ├── messagecontext.xml │ │ ├── replayconfig.xml │ │ └── zookeeper.xml │ └── scripts │ │ └── start.sh ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── ebay │ │ │ └── pulsar │ │ │ └── distributor │ │ │ ├── epl │ │ │ └── func │ │ │ │ └── DefaultValue.java │ │ │ └── replay │ │ │ └── MultipleTopicsOBCAdviceProcessor.java │ └── resources │ │ └── logback.xml │ └── test │ └── resources │ └── placeholder.txt ├── metriccalculator ├── Dockerfile ├── assembly.xml ├── buildsrc │ ├── JetstreamConf │ │ ├── EPL.xml │ │ ├── EsperSetup.xml │ │ ├── messagecontext.xml │ │ ├── metricwiring.xml │ │ ├── replayconfig.xml │ │ └── zookeeper.xml │ └── scripts │ │ └── start.sh ├── pom.xml ├── pulsar.cql └── src │ └── main │ ├── java │ └── com │ │ └── ebay │ │ └── pulsar │ │ └── metriccalculator │ │ ├── cache │ │ ├── CacheManager.java │ │ ├── OffHeapCacheConfig.java │ │ └── OffHeapMemoryManagerRegistry.java │ │ ├── cassandra │ │ └── CassandraConfig.java │ │ ├── esper │ │ └── aggregator │ │ │ ├── TopKNestedAggregator.java │ │ │ └── TopKNestedAggregatorFactory.java │ │ ├── metric │ │ ├── MCMetricGroupDemension.java │ │ └── MetricFrequency.java │ │ ├── offheap │ │ └── serializer │ │ │ ├── AvgCounterSerializer.java │ │ │ ├── CounterSerializer.java │ │ │ └── GroupDemensionSerializer.java │ │ ├── processor │ │ ├── MCMetricsProvider.java │ │ ├── MCScheduler.java │ │ ├── MCSummingProcessor.java │ │ ├── MetricCassandraCollector.java │ │ ├── SOJPipelineStatisticsCollector.java │ │ └── configuration │ │ │ └── MCSummingConfiguration.java │ │ ├── replay │ │ └── MultipleTopicsOBCAdviceProcessor.java │ │ ├── statistics │ │ └── basic │ │ │ ├── AvgCounter.java │ │ │ └── Counter.java │ │ └── util │ │ ├── MCConstant.java │ │ ├── MCCounterHelper.java │ │ ├── MetricDef.java │ │ ├── NamedThreadFactory.java │ │ └── ThreadSafeDateParser.java │ └── resources │ └── logback.xml ├── pom.xml ├── replay ├── Dockerfile ├── assembly.xml ├── buildsrc │ ├── JetstreamConf │ │ ├── CollectorRawEvent.xml │ │ ├── DistributorSsnzEvent.xml │ │ ├── FirstMCfirstMcCounter.xml │ │ ├── MCSsnzEvent.xml │ │ ├── RR1Mobile.xml │ │ ├── SessionizerRawEvent.xml │ │ ├── appwiring.xml │ │ ├── messagecontext.xml │ │ └── zookeeper.xml │ └── scripts │ │ └── start.sh ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── ebay │ │ └── pulsar │ │ ├── metriccalculator │ │ └── metric │ │ │ └── MCMetricGroupDemension.java │ │ └── replay │ │ └── processor │ │ ├── Constants.java │ │ ├── ReplayAdviceProcessor.java │ │ ├── ReplayInboundKafkaProcessor.java │ │ └── ReplayKafkaProcessorConfig.java │ └── resources │ └── logback.xml └── sessionizer ├── Dockerfile ├── assembly.xml ├── buildsrc ├── JetstreamConf │ ├── messagecontext.xml │ ├── replayconfig.xml │ ├── sessionizerconfig.xml │ ├── sessionizerwiring.xml │ └── zookeeper.xml └── scripts │ └── start.sh ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── ebay │ │ └── pulsar │ │ └── sessionizer │ │ ├── cache │ │ ├── MemoryCache.java │ │ ├── MemoryCacheConfig.java │ │ └── impl │ │ │ ├── LocalSessionAttributeMapEncoder.java │ │ │ ├── SessionMemoryCache.java │ │ │ └── SessionSerializer.java │ │ ├── cluster │ │ ├── ClusterManager.java │ │ └── SessionizerLoopbackRingListener.java │ │ ├── config │ │ ├── CommonSessionProfile.java │ │ ├── SessionProfile.java │ │ ├── SessionizerConfig.java │ │ ├── SessionizerConfigValidator.java │ │ └── SubSessionProfile.java │ │ ├── esper │ │ ├── annotation │ │ │ ├── AppendState.java │ │ │ ├── DebugSession.java │ │ │ ├── DecorateEvent.java │ │ │ ├── Session.java │ │ │ ├── SubSession.java │ │ │ ├── UpdateCounter.java │ │ │ ├── UpdateDuration.java │ │ │ ├── UpdateMetadata.java │ │ │ └── UpdateState.java │ │ └── impl │ │ │ ├── AttributeMapVariable.java │ │ │ ├── EsperController.java │ │ │ ├── EsperSessionizer.java │ │ │ ├── SessionVariable.java │ │ │ └── SessionizerEsperExceptionHandlerFactory.java │ │ ├── impl │ │ ├── EsperSessionizerCounter.java │ │ ├── LoopbackEventProducer.java │ │ ├── SessionizationInfo.java │ │ ├── Sessionizer.java │ │ ├── SessionizerCounterManager.java │ │ ├── SessionizerErrorManager.java │ │ ├── SessionizerProcessor.java │ │ ├── SessionizerTimerShutdownHelper.java │ │ └── SsnzEventSender.java │ │ ├── model │ │ ├── AbstractSession.java │ │ ├── Session.java │ │ └── SubSession.java │ │ ├── spi │ │ ├── RemoteStoreCallback.java │ │ ├── RemoteStoreProvider.java │ │ ├── SessionizerContext.java │ │ └── SessionizerExtension.java │ │ └── util │ │ ├── BinaryFormatSerializer.java │ │ ├── ByteBufferUtil.java │ │ ├── Constants.java │ │ └── SessionAttributeMapEncoder.java └── resources │ └── logback.xml └── test └── java └── com └── ebay └── pulsar └── sessionizer └── test ├── cache ├── SessionCacheStoreTest.java └── SessionSerializerTest.java └── esper └── SessionVariableTest.java /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text eol=lf 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #back up file 2 | *~ 3 | *.bak 4 | 5 | # maven target 6 | target 7 | target/* 8 | */target/* 9 | bin 10 | 11 | # intellij files 12 | /.idea 13 | **.iml 14 | **.patch 15 | 16 | # eclipse files 17 | build 18 | .metadata 19 | .project 20 | .settings 21 | .prefs 22 | .classpath 23 | 24 | #MAC files 25 | .DS_Store 26 | -------------------------------------------------------------------------------- /3rdparty.md: -------------------------------------------------------------------------------- 1 | # External dependencies 2 | 3 | ## Apache License 4 | 5 | [License link](http://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | 1. Datastax cassandra driver - https://github.com/datastax/java-driver 8 | 1. UADetector - http://uadetector.sourceforge.net/ 9 | 1. Unitils - http://unitils.org/ 10 | 11 | ## Creative Commons Attribution-ShareAlike 3.0 Unported License 12 | 13 | [License link](http://creativecommons.org/licenses/by-sa/3.0/) 14 | 15 | 1. Maxmind geolite2 geo db - http://dev.maxmind.com/geoip/geoip2/geolite2/ 16 | 17 | ## GPLV2 18 | 19 | [License link](http://opensource.org/licenses/gpl-2.0.php) 20 | 21 | 1. Esper - http://esper.codehaus.org/about/license/license.html 22 | 23 | # Demo dependencies 24 | 25 | ## MIT License 26 | 27 | 1. AngularJS - https://angularjs.org/ 28 | 1. Bootstrap - http://getbootstrap.com/ 29 | 1. Jquery - http://jquery.com/ 30 | 1. MomentJS - http://momentjs.com/ 31 | 32 | ## BSD license 33 | 34 | [License link](http://opensource.org/licenses/BSD-3-Clause) 35 | 36 | 1. D3 - http://d3js.org/ 37 | 38 | ## Apache License 39 | 40 | [License link](http://www.apache.org/licenses/LICENSE-2.0) 41 | 42 | 1. NVD3 - http://nvd3.org/ 43 | 1. Crossfilter - http://square.github.io/crossfilter/ 44 | 1. twitter4j - http://twitter4j.org/en/index.html 45 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 1.0.2 (2015-11-05) 2 | 3 | * Add EPL to publish events and sessions to Kafka and ingest by Druid. 4 | * Fix metriccalculator replay configuration errors. 5 | 6 | # 1.0.1 (2015-08-14) 7 | 8 | * Publish the core project to maven central 9 | * Update scripts to allow user customize the port, this will allow multiple docker instances with --net=host run on one host without port conflicts 10 | 11 | # 1.0.0 (2015-02-23) 12 | 13 | eBay opensource Pulsar, first public release 14 | -------------------------------------------------------------------------------- /Demo/.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text eol=lf 2 | -------------------------------------------------------------------------------- /Demo/.gitignore: -------------------------------------------------------------------------------- 1 | #back up file 2 | *~ 3 | *.bak 4 | 5 | # ebay generated 6 | ebay.log 7 | ebay.log.* 8 | gen-src 9 | gen-meta-src 10 | metricUI/logs 11 | temp_persist_config_*.xml 12 | 13 | # maven target 14 | target 15 | target/* 16 | */target/* 17 | bin 18 | 19 | # intellij files 20 | /.idea 21 | **.iml 22 | **.patch 23 | 24 | # eclipse files 25 | build 26 | .metadata 27 | .project 28 | .settings 29 | .prefs 30 | .classpath 31 | 32 | #jetstream 33 | */buildsrc/JetstreamConf/META-INF/* 34 | */buildsrc/JetstreamConf/WEB-INF/* 35 | */buildsrc/JetstreamConf/css/* 36 | */buildsrc/JetstreamConf/images/* 37 | */buildsrc/JetstreamConf/js/* 38 | */buildsrc/JetstreamConf/build.properties 39 | */buildsrc/JetstreamConf/buildinfo.properties 40 | 41 | #MAC files 42 | .DS_Store 43 | -------------------------------------------------------------------------------- /Demo/README.md: -------------------------------------------------------------------------------- 1 | PulsarIODemo 2 | ============ 3 | 4 | PulsarIODemo 5 | -------------------------------------------------------------------------------- /Demo/metricUI/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /Demo/metricUI/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name metricUI 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # App config 13 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 14 | ENV JETSTREAM_APP_NAME ${project_name} 15 | ENV JETSTREAM_CONFIG_VERSION 1.0 16 | 17 | # Dependency 18 | ENV JETSTREAM_ZKSERVER_HOST zkserver 19 | ENV JETSTREAM_ZKSERVER_PORT 2181 20 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 21 | ENV METRIC_SERVER_HOST metricservice 22 | ENV METRIC_SERVER_PORT 8083 23 | ENV METRIC_CALCULATOR_HOST metriccalculator 24 | ENV METRIC_CALCULATOR_PORT 9999 25 | 26 | # One http port, One context port 27 | ENV JETSTREAM_REST_BASEPORT 8088 28 | ENV JETSTREAM_CONTEXT_BASEPORT 15590 29 | ENV JETSTREAM_APP_PORT 9999 30 | 31 | EXPOSE 9999 15590 8088 32 | ENTRYPOINT ./start.sh 33 | -------------------------------------------------------------------------------- /Demo/metricUI/README.md: -------------------------------------------------------------------------------- 1 | # MetricUI 2 | 3 | Demonstrates the visualization solution for metrics. It's a web application build on Spring MVC. Spring MVC is only used to render the index page. 4 | The real MVC is controlled in Javascript via AngularJS. 5 | 6 | 7 | -------------------------------------------------------------------------------- /Demo/metricUI/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 19 | false 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.basedir}/buildsrc/webapp 46 | /webapp 47 | 48 | ** 49 | 50 | 51 | 52 | 53 | ${project.build.directory} 54 | / 55 | 56 | *.jar 57 | 58 | 59 | *-javadoc.jar 60 | *-sources.jar 61 | 62 | 63 | 64 | 65 | ${project.basedir}/buildsrc/scripts 66 | / 67 | 68 | * 69 | 70 | 0755 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/JetstreamConf/metricuiwiring.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 13 | 16 | 17 | 18 | 19 | 20 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 32 | 33 | 34 | Pulsar.Report/metric 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 53 | 54 | 55 | 56 | 58 | 59 | -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Djetstream.rest.baseport="${JETSTREAM_REST_BASEPORT:-8088}" \ 37 | -Djetstream.context.baseport="${JETSTREAM_CONTEXT_BASEPORT:-15590}" \ 38 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 39 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 40 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 41 | -Dmetricserver.host="$METRIC_SERVER_HOST" \ 42 | -Dmetricserver.port="$METRIC_SERVER_PORT" \ 43 | -Dmetriccalculator.host="$METRIC_CALCULATOR_HOST" \ 44 | -Dmetriccalculator.port="$METRIC_CALCULATOR_PORT" \ 45 | -jar $JETSTREAM_APP_JAR_NAME \ 46 | -n $JETSTREAM_APP_NAME \ 47 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 48 | -p ${JETSTREAM_APP_PORT:-9999} 49 | -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/WEB-INF/pulsar-dispatcher-servlet.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 25 | 26 | 27 | 28 | GET 29 | HEAD 30 | POST 31 | 32 | 33 | 34 | 36 | 37 | /resources/ 38 | 39 | 40 | .jsp 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 61 | 62 | -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | metricUI 9 | 10 | index 11 | 12 | 13 | 14 | 15 | appServlet 16 | com.ebay.pulsar.metric.server.MetricDispatcherServlet 17 | 18 | contextConfigLocation 19 | /WEB-INF/pulsar-dispatcher-servlet.xml 20 | 21 | 1 22 | 23 | 24 | websocketServlet 25 | com.ebay.pulsar.websocket.MetricWebSocketServlet 26 | 1 27 | 28 | 29 | appServlet 30 | / 31 | 32 | 33 | websocketServlet 34 | /websocket 35 | 36 | -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/bold/OpenSans-Bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/bold/OpenSans-Bold.eot -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/bold/OpenSans-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/bold/OpenSans-Bold.ttf -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/bold/OpenSans-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/bold/OpenSans-Bold.woff -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/italic/opensans-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/italic/opensans-italic.eot -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/italic/opensans-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/italic/opensans-italic.ttf -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/italic/opensans-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/italic/opensans-italic.woff -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/light/opensans-light.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/light/opensans-light.eot -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/light/opensans-light.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/light/opensans-light.ttf -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/light/opensans-light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/light/opensans-light.woff -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/regular/opensans-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/regular/opensans-regular.eot -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/regular/opensans-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/regular/opensans-regular.ttf -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/regular/opensans-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/fonts/opensans/regular/opensans-regular.woff -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/imgs/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/imgs/blank.gif -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/imgs/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/imgs/favicon.ico -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/imgs/flags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/imgs/flags.png -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/imgs/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/Demo/metricUI/buildsrc/webapp/resources/imgs/logo.png -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Pulsar IO 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 |
18 | 19 |
20 | 21 |
22 | 23 | 37 |
38 |
39 | 40 | 41 |
42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/js/service/metricservice.js: -------------------------------------------------------------------------------- 1 | angular.module("indexApp").factory('MetricService', ['$resource', function ($resource) { 2 | "use strict"; 3 | return $resource('http://'+location.host+'/pulsar/metric?:params' + '', {params:null}, {query: {method: 'GET', cache:false, isArray:true}}); 4 | }]); -------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/views/realtime.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |
5 | RealTime 6 |
7 | 8 | Next refresh in {{realTimeCounter}}s 9 | 10 |
11 | 12 |
13 |
14 | 15 |
16 | 17 |
18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |
IndexCountryPage
{{$index+1}}  {{data.country}}{{data.value}}
34 |
35 | 36 |
37 | 38 | 39 |
40 | 41 |
42 | 43 | 44 |
45 |
46 |
-------------------------------------------------------------------------------- /Demo/metricUI/buildsrc/webapp/resources/views/twitterFeed.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |
5 | Twitter Feed 6 |
7 | 8 | Next refresh in {{twitterFeedCounter}}s 9 | 10 |
11 | 12 |
13 |
14 | 15 |
16 | 17 |
18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |
IndexHash TagValue
{{$index+1}}{{data.hashtag | limitTo:35}}...{{data.value}}
34 |
35 | 36 |
37 | 38 | 39 |
40 | 41 |
42 | 43 | 44 |
45 |
46 |
-------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/metric/data/DataPoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.data; 7 | 8 | import java.util.Queue; 9 | import java.util.concurrent.ConcurrentLinkedQueue; 10 | 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | 13 | public class DataPoint { 14 | private long timestamp; 15 | private Queue events; 16 | 17 | public DataPoint(Long timestamp) { 18 | this.timestamp = timestamp; 19 | this.events = new ConcurrentLinkedQueue(); 20 | } 21 | 22 | public long getTimestamp() { 23 | return timestamp; 24 | } 25 | 26 | public Queue getEvents() { 27 | return events; 28 | } 29 | 30 | public void addEvent(JetstreamEvent event) { 31 | this.events.add(event); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/metric/esper/aggregator/TopNumReformAggregator.java: -------------------------------------------------------------------------------- 1 | package com.ebay.pulsar.metric.esper.aggregator; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | import java.util.Set; 7 | 8 | import com.clearspring.analytics.stream.Counter; 9 | import com.clearspring.analytics.stream.StreamSummary; 10 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 11 | 12 | public class TopNumReformAggregator implements AggregationMethod { 13 | 14 | StreamSummary m_counter; 15 | Integer m_capacity; 16 | Integer m_topElementCnt = new Integer(10); 17 | 18 | @SuppressWarnings("unchecked") 19 | @Override 20 | public void enter(Object value) { 21 | Object[] params = (Object[]) value; 22 | 23 | if (m_counter == null) { 24 | m_capacity = (Integer) params[0]; 25 | m_topElementCnt = (Integer) params[1]; 26 | m_counter = new StreamSummary(m_capacity.intValue()); 27 | } 28 | 29 | if (params[2] instanceof Map) { 30 | Map topCounters = (Map) params[2]; 31 | Set topKeySet = topCounters.keySet(); 32 | for (Object key : topKeySet) { 33 | m_counter.offer(key, topCounters.get(key).intValue()); 34 | } 35 | } 36 | } 37 | 38 | @Override 39 | public void leave(Object value) { 40 | } 41 | 42 | @Override 43 | public Object getValue() { 44 | Map topN = new LinkedHashMap(); 45 | if ((m_counter == null)) 46 | return topN; 47 | 48 | List> topCounters = m_counter.topK(m_topElementCnt); 49 | 50 | for (Counter counter : topCounters) { 51 | topN.put(counter.getItem(), counter.getCount()); 52 | } 53 | return topN; 54 | } 55 | 56 | @Override 57 | public Class getValueType() { 58 | return Map.class; 59 | } 60 | 61 | @Override 62 | public void clear() { 63 | m_counter = null; 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/metric/esper/aggregator/TopNumReformAggregatorFactory.java: -------------------------------------------------------------------------------- 1 | package com.ebay.pulsar.metric.esper.aggregator; 2 | 3 | import java.util.Map; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 9 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 10 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 11 | 12 | public class TopNumReformAggregatorFactory implements 13 | AggregationFunctionFactory { 14 | 15 | private static final Logger LOGGER = LoggerFactory 16 | .getLogger("com.ebay.pulsar.metric.esper.aggregator.TopNumReformAggregatorFactory"); 17 | 18 | @Override 19 | public void setFunctionName(String functionName) { 20 | } 21 | 22 | @Override 23 | public void validate(AggregationValidationContext validationContext) { 24 | if (validationContext.getParameterTypes().length != 3) 25 | LOGGER.error("TopK Aggregation Function requires 3 parameters viz. max capacity, # top elements and element, - topNumReform(maxCapacit, topElements, element)"); 26 | } 27 | 28 | @Override 29 | public AggregationMethod newAggregator() { 30 | return new TopNumReformAggregator(); 31 | } 32 | 33 | @Override 34 | public Class getValueType() { 35 | return Map.class; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/metric/server/MetricDispatcherServlet.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.server; 7 | 8 | import org.springframework.beans.BeanUtils; 9 | import org.springframework.context.ApplicationContext; 10 | import org.springframework.context.ApplicationContextException; 11 | import org.springframework.web.context.ConfigurableWebApplicationContext; 12 | import org.springframework.web.context.WebApplicationContext; 13 | import org.springframework.web.servlet.DispatcherServlet; 14 | 15 | public class MetricDispatcherServlet extends DispatcherServlet { 16 | private static final long serialVersionUID = 7689614315500644056L; 17 | 18 | protected WebApplicationContext createWebApplicationContext(ApplicationContext parent) { 19 | Class contextClass = getContextClass(); 20 | if (this.logger.isDebugEnabled()) { 21 | this.logger.debug("Servlet with name '" + getServletName() + 22 | "' will try to create custom WebApplicationContext context of class '" + 23 | contextClass.getName() + "'" + ", using parent context [" + parent + "]"); 24 | } 25 | if (!ConfigurableWebApplicationContext.class.isAssignableFrom(contextClass)) { 26 | throw new ApplicationContextException( 27 | "Fatal initialization error in servlet with name '" + getServletName() + 28 | "': custom WebApplicationContext class [" + contextClass.getName() + 29 | "] is not of type ConfigurableWebApplicationContext"); 30 | } 31 | ConfigurableWebApplicationContext wac = 32 | (ConfigurableWebApplicationContext) BeanUtils.instantiateClass(contextClass); 33 | 34 | wac.setParent(parent); 35 | if (wac.getParent() == null) { 36 | ApplicationContext rootContext = (ApplicationContext) getServletContext().getAttribute("JetStreamRoot"); 37 | wac.setParent(rootContext); 38 | } 39 | wac.setConfigLocation(getContextConfigLocation()); 40 | configureAndRefreshWebApplicationContext(wac); 41 | return wac; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/metric/server/SpringWebContextHolder.java: -------------------------------------------------------------------------------- 1 | package com.ebay.pulsar.metric.server; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.context.ApplicationContext; 5 | import org.springframework.context.ApplicationContextAware; 6 | 7 | public class SpringWebContextHolder implements ApplicationContextAware{ 8 | public static ApplicationContext wac; 9 | 10 | @Override 11 | public void setApplicationContext(ApplicationContext applicationContext) 12 | throws BeansException { 13 | SpringWebContextHolder.wac = applicationContext; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/metric/util/JsonUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.util; 7 | 8 | import java.io.IOException; 9 | import org.codehaus.jackson.JsonGenerationException; 10 | import org.codehaus.jackson.map.JsonMappingException; 11 | import org.codehaus.jackson.map.ObjectMapper; 12 | import org.codehaus.jackson.map.ObjectWriter; 13 | 14 | public final class JsonUtil { 15 | 16 | private static ObjectWriter ow = new ObjectMapper().writer(); 17 | 18 | public static String toJson(T t) { 19 | String json = null; 20 | try { 21 | json = ow.writeValueAsString(t); 22 | } catch (JsonGenerationException e) { 23 | e.printStackTrace(); 24 | } catch (JsonMappingException e) { 25 | e.printStackTrace(); 26 | } catch (IOException e) { 27 | e.printStackTrace(); 28 | } 29 | return json; 30 | } 31 | 32 | /** Cannot instantiate. */ 33 | private JsonUtil() { 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/websocket/MetricWebSocketCreator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.websocket; 7 | 8 | import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest; 9 | import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; 10 | import org.eclipse.jetty.websocket.servlet.WebSocketCreator; 11 | 12 | public class MetricWebSocketCreator implements WebSocketCreator { 13 | 14 | private WebSocketConnectionManager wsConnectionManager; 15 | 16 | @Override 17 | public Object createWebSocket(ServletUpgradeRequest req, 18 | ServletUpgradeResponse resp) { 19 | // Borrow WebSocket sub-protocols as input for channels and filters, 20 | // which are at position 0 and 1 in the sub-protocols array 21 | MetricWebSocket socket = new MetricWebSocket(req.getSubProtocols().get(0), req.getSubProtocols().get(1), wsConnectionManager); 22 | resp.setAcceptedSubProtocol(req.getSubProtocols().get(0)); 23 | return socket; 24 | } 25 | 26 | public WebSocketConnectionManager getWsConnectionManager() { 27 | return wsConnectionManager; 28 | } 29 | 30 | public void setWsConnectionManager( 31 | WebSocketConnectionManager wsConnectionManager) { 32 | this.wsConnectionManager = wsConnectionManager; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/websocket/MetricWebSocketServlet.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.websocket; 7 | 8 | import org.eclipse.jetty.websocket.servlet.WebSocketServlet; 9 | import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; 10 | 11 | import com.ebay.pulsar.metric.server.SpringWebContextHolder; 12 | 13 | public class MetricWebSocketServlet extends WebSocketServlet { 14 | private static final long serialVersionUID = -4260093440316144872L; 15 | 16 | @Override 17 | public void configure(WebSocketServletFactory factory) { 18 | factory.register(MetricWebSocket.class); 19 | MetricWebSocketCreator webSocketCreator = SpringWebContextHolder.wac.getBean("MetricWebSocketCreator", MetricWebSocketCreator.class); 20 | factory.setCreator(webSocketCreator); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/java/com/ebay/pulsar/websocket/PulsarEventConstant.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.websocket; 7 | 8 | public class PulsarEventConstant { 9 | public static String AND = "&"; 10 | public static String EQUAL = "equal"; 11 | 12 | public static String SAMPLING = "sampling"; 13 | public static String CHANNEL = "channel"; 14 | public static String GUID = "g"; 15 | } 16 | -------------------------------------------------------------------------------- /Demo/metricUI/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /Demo/metricservice/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name metricservice 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # App config 13 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 14 | ENV JETSTREAM_APP_NAME ${project_name} 15 | ENV JETSTREAM_CONFIG_VERSION 1.0 16 | 17 | # Dependency 18 | ENV JETSTREAM_ZKSERVER_HOST zkserver 19 | ENV JETSTREAM_ZKSERVER_PORT 2181 20 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 21 | ENV PULSAR_CASSANDRA cassandraserver 22 | 23 | # One http port, no context port 24 | ENV JETSTREAM_REST_BASEPORT 8083 25 | ENV JETSTREAM_APP_PORT 9999 26 | 27 | EXPOSE 8083 9999 28 | ENTRYPOINT ./start.sh 29 | -------------------------------------------------------------------------------- /Demo/metricservice/README.md: -------------------------------------------------------------------------------- 1 | # Metricservice 2 | 3 | Metricservice provides a restful interface to query Cassandra table to visualize the metrics and draw some graphs. This is just for demo purpose. 4 | 5 | Sample urls: 6 | 7 | http://HOST:PORT/pulsar/metric?columnFamilyName=mc_groupmetric&metricname=visitorsperbrowser 8 | 9 | http://HOST:PORT/pulsar/counter?metricname=MCSessionCount&groupid=2015-01-08 -------------------------------------------------------------------------------- /Demo/metricservice/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 19 | false 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.build.directory} 46 | / 47 | 48 | *.jar 49 | 50 | 51 | *-javadoc.jar 52 | *-sources.jar 53 | 54 | 55 | 56 | 57 | ${project.basedir}/buildsrc/scripts 58 | / 59 | 60 | * 61 | 62 | 0755 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /Demo/metricservice/buildsrc/JetstreamConf/messagecontext.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | 14 | 15 | 16 | 17 | 18 | 19 | dev.geo-jetstream.com 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /Demo/metricservice/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /Demo/metricservice/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Dpulsar.runtime.cassandra="$PULSAR_CASSANDRA" \ 37 | -Djetstream.rest.baseport="${JETSTREAM_REST_BASEPORT:-8083}" \ 38 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 39 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 40 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 41 | -jar $JETSTREAM_APP_JAR_NAME \ 42 | -n $JETSTREAM_APP_NAME \ 43 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 44 | -p ${JETSTREAM_APP_PORT:-9999} 45 | -------------------------------------------------------------------------------- /Demo/metricservice/src/main/java/com/ebay/pulsar/metric/core/DataType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.core; 7 | 8 | public enum DataType { 9 | 10 | RAW, MAX, MIN, AVG; 11 | 12 | public static DataType valueOf(int type) { 13 | switch (type) { 14 | case 0: return RAW; 15 | case 1 : return MAX; 16 | case 2 : return MIN; 17 | case 3 : return AVG; 18 | default: throw new IllegalArgumentException(type + " is not a supported " + 19 | DataType.class.getSimpleName()); 20 | } 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /Demo/metricservice/src/main/java/com/ebay/pulsar/metric/core/MetricsService.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.core; 7 | 8 | import java.util.List; 9 | 10 | import com.ebay.pulsar.metric.impl.cassandra.QueryParam; 11 | import com.google.common.util.concurrent.ListenableFuture; 12 | 13 | public interface MetricsService { 14 | ListenableFuture> findCounters(String metric, String group); 15 | ListenableFuture> findData(QueryParam queryParam); 16 | } 17 | -------------------------------------------------------------------------------- /Demo/metricservice/src/main/java/com/ebay/pulsar/metric/core/MetricsThreadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.core; 7 | 8 | import java.util.concurrent.ThreadFactory; 9 | import java.util.concurrent.atomic.AtomicInteger; 10 | 11 | public class MetricsThreadFactory implements ThreadFactory, Thread.UncaughtExceptionHandler { 12 | 13 | private AtomicInteger threadNumber = new AtomicInteger(0); 14 | 15 | private String poolName = "MetricsThreadPool"; 16 | 17 | public Thread newThread(Runnable r) { 18 | Thread t = new Thread(r, poolName + "-" + threadNumber.getAndIncrement()); 19 | t.setDaemon(false); 20 | t.setUncaughtExceptionHandler(this); 21 | 22 | return t; 23 | } 24 | 25 | @Override 26 | public void uncaughtException(Thread t, Throwable e) { 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Demo/metricservice/src/main/java/com/ebay/pulsar/metric/core/NumericMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.core; 7 | 8 | public interface NumericMetric { 9 | String getMetricName(); 10 | String getGroupId(); 11 | long getValue(); 12 | long getTimestamp(); 13 | } 14 | -------------------------------------------------------------------------------- /Demo/metricservice/src/main/java/com/ebay/pulsar/metric/core/RawMetricMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.core; 7 | 8 | import java.util.ArrayList; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | import com.datastax.driver.core.ColumnDefinitions; 14 | import com.datastax.driver.core.ColumnDefinitions.Definition; 15 | import com.datastax.driver.core.ResultSet; 16 | import com.datastax.driver.core.Row; 17 | 18 | public class RawMetricMapper { 19 | 20 | public RawNumericMetric map(Row row) { 21 | RawNumericMetric metricRow = new RawNumericMetric(row.getString(0), row.getString(1), row.getDate(2).getTime(), row.getInt(3)); 22 | ColumnDefinitions columeDef = row.getColumnDefinitions(); 23 | List columeDefList = columeDef.asList(); 24 | Map tagMap = new HashMap(); 25 | for(Definition def: columeDefList){ 26 | if(def.getName().startsWith("tag_")){ 27 | tagMap.put(def.getName(), row.getString(def.getName())); 28 | } 29 | } 30 | 31 | if(tagMap.size()>0){ 32 | metricRow.setTagMap(tagMap); 33 | } 34 | return metricRow; 35 | } 36 | 37 | public List map(ResultSet resultSet) { 38 | 39 | List metrics = new ArrayList(); 40 | 41 | for (Row row : resultSet) { 42 | try{ 43 | metrics.add(map(row)); 44 | }catch(Exception ex){ 45 | ex.printStackTrace(); 46 | } 47 | } 48 | return metrics; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Demo/metricservice/src/main/java/com/ebay/pulsar/metric/servlet/ServletStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metric.servlet; 7 | 8 | import java.util.concurrent.atomic.AtomicLong; 9 | 10 | import org.springframework.jmx.export.annotation.ManagedAttribute; 11 | import org.springframework.jmx.export.annotation.ManagedOperation; 12 | import org.springframework.jmx.export.annotation.ManagedResource; 13 | 14 | import com.ebay.jetstream.event.support.ErrorManager; 15 | 16 | @ManagedResource(objectName = "Event/Channel", description = "MetricServlet") 17 | public class ServletStats { 18 | private String lastFailedRequest; 19 | private final ErrorManager errors = new ErrorManager(); 20 | private final AtomicLong errorCount = new AtomicLong(0); 21 | private final AtomicLong queryRequestCount = new AtomicLong(0); 22 | private final AtomicLong invalidRequestCount = new AtomicLong(0); 23 | 24 | @ManagedOperation 25 | public void cleanErrors() { 26 | errorCount.set(0); 27 | errors.clearErrorList(); 28 | lastFailedRequest = null; 29 | } 30 | 31 | public long getQueryRequestCount() { 32 | return queryRequestCount.get(); 33 | } 34 | 35 | public long getErrorCount() { 36 | return errorCount.get(); 37 | } 38 | 39 | @ManagedAttribute 40 | public ErrorManager getErrorManager() { 41 | return errors; 42 | } 43 | 44 | public String getLastFailedRequest() { 45 | return lastFailedRequest; 46 | } 47 | 48 | public void registerError(Throwable ex) { 49 | errorCount.incrementAndGet(); 50 | errors.registerError(ex); 51 | } 52 | 53 | public void setLastFailedRequest(String lastFailedRequest) { 54 | this.lastFailedRequest = lastFailedRequest; 55 | } 56 | 57 | public void incInvalidRequestCount() { 58 | invalidRequestCount.incrementAndGet(); 59 | } 60 | 61 | public long getInvalidRequestCount() { 62 | return invalidRequestCount.get(); 63 | } 64 | 65 | public void incQueryRequestCount() { 66 | queryRequestCount.incrementAndGet(); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /Demo/metricservice/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /Demo/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 9 | 10 | com.ebay.pulsar 11 | realtime-analytics 12 | 1.0.2 13 | .. 14 | 15 | 4.0.0 16 | com.ebay.pulsar 17 | realtime-analytics-demo 18 | pom 19 | realtime-analytics pipeline demo 20 | 21 | twittersample 22 | metricservice 23 | metricUI 24 | 25 | 26 | -------------------------------------------------------------------------------- /Demo/rundemo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo '>>> Start external dependency' 4 | sudo docker run -d --name zkserver -t "jetstream/zookeeper" 5 | sleep 5 6 | sudo docker run -d --name mongoserver -t "mongo" 7 | sleep 5 8 | sudo docker run -d --name kafkaserver -t "jetstream/kafka" 9 | sleep 5 10 | sudo docker run -d --name cassandraserver -t "poklet/cassandra" 11 | 12 | echo '>>> Sleep 30 seconds' 13 | sleep 30 14 | mkdir /tmp/pulsarcql 15 | wget https://raw.githubusercontent.com/pulsarIO/realtime-analytics/master/metriccalculator/pulsar.cql -O /tmp/pulsarcql/pulsar.cql 16 | sudo docker run -it --rm --link cassandraserver:cass1 -v /tmp/pulsarcql:/data poklet/cassandra bash -c 'cqlsh $CASS1_PORT_9160_TCP_ADDR -f /data/pulsar.cql' 17 | 18 | echo '>>> Sleep 10 seconds' 19 | sleep 10 20 | 21 | echo '>>> Start Pulsar pipeline' 22 | sudo docker run -d -p 0.0.0.0:8000:9999 -p 0.0.0.0:8081:8080 --link zkserver:zkserver --link mongoserver:mongoserver -t "jetstream/config" 23 | sleep 5 24 | sudo docker run -d -p 0.0.0.0:8001:9999 --link zkserver:zkserver --link mongoserver:mongoserver --link kafkaserver:kafkaserver -t "pulsar/replay" 25 | sleep 5 26 | sudo docker run -d -p 0.0.0.0:8003:9999 --link zkserver:zkserver --link mongoserver:mongoserver --link kafkaserver:kafkaserver -t "pulsar/sessionizer" 27 | sleep 5 28 | sudo docker run -d -p 0.0.0.0:8004:9999 --link zkserver:zkserver --link mongoserver:mongoserver --link kafkaserver:kafkaserver -t "pulsar/distributor" 29 | sleep 5 30 | sudo docker run -d -p 0.0.0.0:8005:9999 --name metriccalculator --link zkserver:zkserver --link mongoserver:mongoserver --link kafkaserver:kafkaserver --link cassandraserver:cassandraserver -t "pulsar/metriccalculator" 31 | sleep 5 32 | sudo docker run -d -p 0.0.0.0:8002:9999 -p 0.0.0.0:8080:8080 --link zkserver:zkserver --link mongoserver:mongoserver --link kafkaserver:kafkaserver -t "pulsar/collector" 33 | sleep 5 34 | 35 | echo '>>> Start demo' 36 | sudo docker run -d -p 0.0.0.0:8006:9999 -p 0.0.0.0:8083:8083 --name metricservice --link zkserver:zkserver --link mongoserver:mongoserver --link cassandraserver:cassandraserver -t "pulsar/metricservice" 37 | sleep 5 38 | sudo docker run -d -p 0.0.0.0:8007:9999 -p 0.0.0.0:8088:8088 --link zkserver:zkserver --link mongoserver:mongoserver --link metricservice:metricservice --link metriccalculator:metriccalculator -t "pulsar/metricui" 39 | 40 | # The twitter sample use the twitter4j api, to run the app, it requires twitter OAUTH token (https://dev.twitter.com/oauth/overview/application-owner-access-tokens) 41 | #sleep 5 42 | #sudo docker run -d -p 0.0.0.0:8008:9999 --link zkserver:zkserver --link mongoserver:mongoserver -e TWITTER4J_OAUTH_CONSUMERKEY= -e TWITTER4J_OAUTH_CONSUMERSECRET= -e TWITTER4J_OAUTH_ACCESSTOKEN= -e TWITTER4J_OAUTH_ACCESSTOKENSECRET= -t pulsar/twittersample 43 | #sleep 5 44 | -------------------------------------------------------------------------------- /Demo/twittersample/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name twittersample 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # App config 13 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 14 | ENV JETSTREAM_APP_NAME ${project_name} 15 | ENV JETSTREAM_CONFIG_VERSION 1.0 16 | 17 | # Dependency 18 | ENV JETSTREAM_ZKSERVER_HOST zkserver 19 | ENV JETSTREAM_ZKSERVER_PORT 2181 20 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 21 | 22 | #Suggested production JAVA_OPS for Open JDK. 23 | #ENV JETSTREAM_JAVA_OPTS -server -Xms6g -Xmx6g -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseCompressedOops -X:MaxTenuringThreshold=8 -XX:CMSInitiatingOccupancyFraction=75 -XX:MaxNewSize=3g -XX:NewSize=3g -XX:+CMSConcurrentMTEnabled -XX:+CMSScavengeBeforeRemark 24 | 25 | # One context port 26 | ENV JETSTREAM_CONTEXT_BASEPORT 15590 27 | ENV JETSTREAM_APP_PORT 9999 28 | 29 | EXPOSE 9999 30 | ENTRYPOINT ./start.sh 31 | -------------------------------------------------------------------------------- /Demo/twittersample/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | false 19 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.build.directory} 46 | / 47 | 48 | *.jar 49 | 50 | 51 | *-javadoc.jar 52 | *-sources.jar 53 | 54 | 55 | 56 | 57 | ${project.basedir}/buildsrc/scripts 58 | / 59 | 60 | *.* 61 | 62 | 0755 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /Demo/twittersample/buildsrc/JetstreamConf/appwiring.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 13 | 14 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 30 | 31 | 32 | 33 | 36 | 37 | 38 | 39 | 41 | 42 | 43 | Pulsar.MC/ssnzEvent 44 | 45 | 46 | 47 | 48 | 50 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /Demo/twittersample/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /Demo/twittersample/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Djetstream.context.baseport="${JETSTREAM_CONTEXT_BASEPORT:-15590}" \ 37 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 38 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 39 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 40 | -Dtwitter4j.oauth.consumerKey="$TWITTER4J_OAUTH_CONSUMERKEY" \ 41 | -Dtwitter4j.oauth.consumerSecret="$TWITTER4J_OAUTH_CONSUMERSECRET" \ 42 | -Dtwitter4j.oauth.accessToken="$TWITTER4J_OAUTH_ACCESSTOKEN" \ 43 | -Dtwitter4j.oauth.accessTokenSecret="$TWITTER4J_OAUTH_ACCESSTOKENSECRET" \ 44 | -jar $JETSTREAM_APP_JAR_NAME \ 45 | -n $JETSTREAM_APP_NAME \ 46 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 47 | -p ${JETSTREAM_APP_PORT:-9999} 48 | -------------------------------------------------------------------------------- /Demo/twittersample/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Pulsar is a highly scalable and reliable event-driven data pipeline for real-time analytics. It was primarily created for user behavior analytics but could be used for many other user cases. 2 | 3 | See [**wiki**](../../wiki) for more detail. 4 | -------------------------------------------------------------------------------- /collector/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name collector 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # Download and install mmdb. 13 | RUN wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz && gunzip GeoLite2-City.mmdb.gz && mkdir geodb && mv GeoLite2-City.mmdb geodb/ 14 | 15 | # App config 16 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 17 | ENV JETSTREAM_APP_NAME ${project_name} 18 | ENV JETSTREAM_CONFIG_VERSION 1.0 19 | 20 | # Dependency 21 | ENV JETSTREAM_ZKSERVER_HOST zkserver 22 | ENV JETSTREAM_ZKSERVER_PORT 2181 23 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 24 | ENV PULSAR_KAFKA_BROKERS kafkaserver:9092 25 | 26 | # One Http port, Two Context ports 27 | ENV JETSTREAM_REST_BASEPORT 8080 28 | ENV JETSTREAM_CONTEXT_BASEPORT 15590 29 | ENV JETSTREAM_APP_PORT 9999 30 | 31 | EXPOSE 8080 9999 15590 15591 32 | ENTRYPOINT ./start.sh 33 | -------------------------------------------------------------------------------- /collector/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | false 19 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.basedir}/buildsrc/data 46 | data 47 | 48 | ** 49 | 50 | 51 | 52 | 53 | ${project.build.directory} 54 | / 55 | 56 | *.jar 57 | 58 | 59 | *-javadoc.jar 60 | *-sources.jar 61 | 62 | 63 | 64 | 65 | ${project.basedir}/buildsrc/scripts 66 | / 67 | 68 | * 69 | 70 | 0755 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /collector/buildsrc/JetstreamConf/EPL.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | 12 | 13 | 14 | 15 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /collector/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /collector/buildsrc/data/Event.txt: -------------------------------------------------------------------------------- 1 | { 2 | "si": "${siValue}", 3 | "ac": "727354723", 4 | "tn": "TenantName", 5 | "or": "OrignalName", 6 | "ct": ${ctValue}, 7 | "ipv4": "${ipValue}", 8 | "ua": "${uaValue}", 9 | "et": "Mobile", 10 | "rf": "http://referrer.somewhere.com", 11 | "url": "http://LandingPage.com", 12 | "itmT":"${itmTitle}", 13 | "itmP":${itmPrice}, 14 | "cap":"${campaignName}", 15 | "capG":${cmapaignGMV}, 16 | "capQ":${cmapaignQuantity}, 17 | "ts":"${trafficsource}", 18 | "t":"${site}", 19 | "_dwell":${dwell}, 20 | "dd_df":"${devicefamily}" 21 | 22 | } 23 | -------------------------------------------------------------------------------- /collector/buildsrc/data/device.txt: -------------------------------------------------------------------------------- 1 | Desktop 2 | Desktop 3 | Desktop 4 | Desktop 5 | Desktop 6 | Desktop 7 | Mobile 8 | Mobile 9 | Mobile 10 | Mobile 11 | TouchScreen 12 | TouchScreen 13 | Tablet 14 | Other 15 | Other 16 | Other 17 | Other 18 | Other 19 | Other -------------------------------------------------------------------------------- /collector/buildsrc/data/ip.txt: -------------------------------------------------------------------------------- 1 | 71.190.6.172 2 | 99.44.126.240 3 | 76.180.58.217 4 | 172.56.17.73 5 | 50.149.232.110 6 | 166.182.83.70 7 | 50.132.111.181 8 | 66.249.67.40 9 | 76.168.10.123 10 | 173.241.26.203 11 | 68.6.207.85 12 | 99.117.41.116 13 | 107.222.91.12 14 | 162.237.73.188 15 | 84.179.167.171 16 | 84.179.67.171 17 | 84.179.123.171 18 | 84.179.139.171 19 | 84.178.67.171 20 | 84.177.67.171 21 | 84.179.116.171 22 | 84.179.102.171 23 | 84.179.89.171 24 | 84.179.76.171 25 | 84.179.63.171 26 | 94.14.123.37 27 | 94.14.15.37 28 | 94.14.167.37 29 | 94.14.198.37 30 | 94.14.34.37 31 | 94.14.64.37 32 | 94.14.84.37 33 | 94.14.104.37 34 | 94.14.194.37 35 | 77.97.68.236 36 | 86.134.144.100 37 | 120.21.55.211 38 | 120.21.155.211 39 | 120.21.123.211 40 | 120.21.128.211 41 | 120.21.127.211 42 | 120.21.90.211 43 | 120.21.109.196 44 | 123.2.143.46 45 | 212.54.195.162 -------------------------------------------------------------------------------- /collector/buildsrc/data/item.txt: -------------------------------------------------------------------------------- 1 | Apple iPad-1:120.33 2 | Apple iPad-2:121.33 3 | Apple iPad-3:122.33 4 | Apple iPad-4:123.33 5 | Apple iPad-5:124.33 6 | Apple iPad-6:125.33 7 | Apple iPad-7:126.33 8 | Apple iPad-8:127.33 9 | Apple iPad-9:128.33 10 | Apple iPad-10:129.33 11 | Apple iPad-11:130.33 12 | Apple iPad-12:131.33 13 | Apple iPad-13:132.33 14 | Apple iPad-14:133.33 15 | Apple iPad-15:134.33 16 | Apple iPad-16:135.33 17 | Apple iPad-17:136.33 18 | Apple iPad-18:137.33 19 | Apple iPad-19:138.33 20 | Apple iPad-20:139.33 21 | Apple iPad-21:140.33 22 | Apple iPad-22:141.33 23 | Apple iPad-23:142.33 24 | Apple iPad-24:143.33 25 | Apple iPad-25:144.33 26 | Apple iPad-26:145.33 27 | Apple iPad-27:146.33 28 | Apple iPad-28:147.33 29 | Apple iPad-29:148.33 30 | Apple iPad-30:149.33 31 | -------------------------------------------------------------------------------- /collector/buildsrc/data/site.txt: -------------------------------------------------------------------------------- 1 | 0 2 | 0 3 | 0 4 | 0 5 | 0 6 | 0 7 | 0 8 | 0 9 | 0 10 | 0 11 | 3 12 | 3 13 | 3 14 | 3 15 | 3 16 | 16 17 | 16 18 | 16 19 | 16 20 | 77 21 | 77 -------------------------------------------------------------------------------- /collector/buildsrc/data/ts.txt: -------------------------------------------------------------------------------- 1 | Direct 2 | Direct 3 | Direct 4 | Direct 5 | Direct 6 | Direct 7 | Direct 8 | Direct 9 | Direct 10 | Direct 11 | Referral 12 | Referral 13 | Referral 14 | Referral 15 | Referral 16 | Referral 17 | Other 18 | Organic Search 19 | Social Media 20 | Social Media 21 | Social Media 22 | Natural Search 23 | Affiliates 24 | Paid Campaign 25 | Email Marketing 26 | Email -------------------------------------------------------------------------------- /collector/buildsrc/data/userAgent.txt: -------------------------------------------------------------------------------- 1 | Mozilla/5.0 (Windows NT 6.2; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0 2 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36 3 | eBayNioHttpClient 1.0 4 | eBayAndroid/3.0.0.19 5 | Mozilla/5.0 (Linux; Android 4.4.2; Ultimax Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Safari/537.36 6 | Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) 7 | Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0 8 | Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko 9 | Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; LGMS323 Build/KOT49I.MS32310c) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.1599.103 Mobile Safari/537.36 10 | Mozilla/5.0 (iPhone; CPU iPhone OS 8_1_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B440 Safari/600.1.4 11 | Mozilla/5.0 (Linux; Android 4.4.2; es-us; SAMSUNG SGH-T399N Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Version/1.5 Chrome/28.0.1500.94 Mobile Safari/537.36 12 | Mozilla/5.0 (Linux; Android 4.4.4; SAMSUNG-SGH-I337 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.93 Mobile Safari/537.36 13 | Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) 14 | Mozilla/5.0 (iPod; CPU iPhone OS 6_0_1 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A523 Safari/8536.25 -------------------------------------------------------------------------------- /collector/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Dpulsar.runtime.kafka.brokers="$PULSAR_KAFKA_BROKERS" \ 37 | -Djetstream.rest.baseport="${JETSTREAM_REST_BASEPORT:-8080}" \ 38 | -Djetstream.context.baseport="${JETSTREAM_CONTEXT_BASEPORT:-15590}" \ 39 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 40 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 41 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 42 | -jar $JETSTREAM_APP_JAR_NAME \ 43 | -n $JETSTREAM_APP_NAME \ 44 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 45 | -p ${JETSTREAM_APP_PORT:-9999} 46 | -------------------------------------------------------------------------------- /collector/src/main/java/com/ebay/pulsar/collector/servlet/ServletStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector.servlet; 7 | 8 | import java.util.concurrent.atomic.AtomicLong; 9 | 10 | import org.springframework.jmx.export.annotation.ManagedAttribute; 11 | import org.springframework.jmx.export.annotation.ManagedOperation; 12 | import org.springframework.jmx.export.annotation.ManagedResource; 13 | 14 | import com.ebay.jetstream.event.support.ErrorManager; 15 | 16 | @ManagedResource(objectName = "Event/Channel", description = "IngestServlet") 17 | public class ServletStats { 18 | private String lastFailedRequest; 19 | private final ErrorManager errors = new ErrorManager(); 20 | private final AtomicLong errorCount = new AtomicLong(0); 21 | private final AtomicLong ingestRequestCount = new AtomicLong(0); 22 | private final AtomicLong ingestBatchRequestCount = new AtomicLong(0); 23 | private final AtomicLong invalidRequestCount = new AtomicLong(0); 24 | 25 | @ManagedOperation 26 | public void cleanErrors() { 27 | errorCount.set(0); 28 | errors.clearErrorList(); 29 | lastFailedRequest = null; 30 | } 31 | 32 | public long getIngestRequestCount() { 33 | return ingestRequestCount.get(); 34 | } 35 | 36 | public long getBatchIngestRequestCount() { 37 | return ingestBatchRequestCount.get(); 38 | } 39 | 40 | public long getErrorCount() { 41 | return errorCount.get(); 42 | } 43 | 44 | @ManagedAttribute 45 | public ErrorManager getErrorManager() { 46 | return errors; 47 | } 48 | 49 | public String getLastFailedRequest() { 50 | return lastFailedRequest; 51 | } 52 | 53 | public void incIngestRequestCount() { 54 | ingestRequestCount.incrementAndGet(); 55 | } 56 | 57 | public void incBatchIngestRequestCount() { 58 | ingestBatchRequestCount.incrementAndGet(); 59 | } 60 | 61 | public void registerError(Throwable ex) { 62 | errorCount.incrementAndGet(); 63 | errors.registerError(ex); 64 | } 65 | 66 | public void setLastFailedRequest(String lastFailedRequest) { 67 | this.lastFailedRequest = lastFailedRequest; 68 | } 69 | 70 | public void incInvalidRequestCount() { 71 | invalidRequestCount.incrementAndGet(); 72 | } 73 | 74 | public long getInvalidRequestCount() { 75 | return invalidRequestCount.get(); 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /collector/src/main/java/com/ebay/pulsar/collector/udf/DeviceEnrichmentUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector.udf; 7 | 8 | import net.sf.uadetector.OperatingSystem; 9 | import net.sf.uadetector.ReadableUserAgent; 10 | import net.sf.uadetector.UserAgentStringParser; 11 | import net.sf.uadetector.service.UADetectorServiceFactory; 12 | 13 | import org.springframework.beans.factory.InitializingBean; 14 | 15 | public class DeviceEnrichmentUtil implements InitializingBean { 16 | 17 | private UserAgentStringParser parser; 18 | 19 | private static final DeviceEnrichmentUtil INSTANCE = new DeviceEnrichmentUtil(); 20 | 21 | public static DeviceEnrichmentUtil getInstance() { 22 | return INSTANCE; 23 | } 24 | 25 | private DeviceEnrichmentUtil() { 26 | //singleton 27 | } 28 | 29 | @Override 30 | public void afterPropertiesSet() throws Exception { 31 | parser = UADetectorServiceFactory.getCachingAndUpdatingParser(); 32 | } 33 | 34 | public static DeviceInfo getDeviceInfo(String userAgent) { 35 | return getInstance()._getDeviceInfo(userAgent); 36 | } 37 | 38 | private DeviceInfo _getDeviceInfo(String userAgent) { 39 | DeviceInfo deviceInfo = new DeviceInfo(); 40 | if (userAgent == null) { 41 | return deviceInfo; 42 | } 43 | 44 | ReadableUserAgent agent = parser.parse(userAgent); 45 | 46 | String deviceCategory = agent.getDeviceCategory().getName(); 47 | 48 | OperatingSystem operatingSystem = agent.getOperatingSystem(); 49 | String osFamily = operatingSystem.getFamilyName(); 50 | String osVersion = operatingSystem.getVersionNumber().toVersionString(); 51 | 52 | 53 | String userAgentName = agent.getName(); 54 | String userAgentType = agent.getTypeName(); 55 | String userAgentFamily = agent.getFamily().getName(); 56 | String userAgentVersion = agent.getVersionNumber().toVersionString(); 57 | 58 | deviceInfo.setDeviceCategory(deviceCategory); 59 | deviceInfo.setOsFamily(osFamily); 60 | deviceInfo.setOsVersion(osVersion); 61 | deviceInfo.setUserAgent(userAgentName); 62 | deviceInfo.setUserAgentType(userAgentType); 63 | deviceInfo.setUserAgentFamily(userAgentFamily); 64 | deviceInfo.setUserAgentVersion(userAgentVersion); 65 | 66 | return deviceInfo; 67 | } 68 | } -------------------------------------------------------------------------------- /collector/src/main/java/com/ebay/pulsar/collector/udf/DeviceInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector.udf; 7 | 8 | public class DeviceInfo { 9 | 10 | private String deviceCategory; 11 | private String osFamily; 12 | private String osVersion; 13 | private String userAgent; 14 | private String userAgentType; 15 | private String userAgentFamily; 16 | private String userAgentVersion; 17 | 18 | public String getDeviceCategory() { 19 | return deviceCategory; 20 | } 21 | public void setDeviceCategory(String deviceCategory) { 22 | this.deviceCategory = deviceCategory; 23 | } 24 | public String getOsFamily() { 25 | return osFamily; 26 | } 27 | public void setOsFamily(String osFamily) { 28 | this.osFamily = osFamily; 29 | } 30 | public String getOsVersion() { 31 | return osVersion; 32 | } 33 | public void setOsVersion(String osVersion) { 34 | this.osVersion = osVersion; 35 | } 36 | public String getUserAgent() { 37 | return userAgent; 38 | } 39 | public void setUserAgent(String userAgent) { 40 | this.userAgent = userAgent; 41 | } 42 | public String getUserAgentType() { 43 | return userAgentType; 44 | } 45 | public void setUserAgentType(String userAgentType) { 46 | this.userAgentType = userAgentType; 47 | } 48 | public String getUserAgentFamily() { 49 | return userAgentFamily; 50 | } 51 | public void setUserAgentFamily(String userAgentFamily) { 52 | this.userAgentFamily = userAgentFamily; 53 | } 54 | public String getUserAgentVersion() { 55 | return userAgentVersion; 56 | } 57 | public void setUserAgentVersion(String userAgentVersion) { 58 | this.userAgentVersion = userAgentVersion; 59 | } 60 | } -------------------------------------------------------------------------------- /collector/src/main/java/com/ebay/pulsar/collector/udf/GeoInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector.udf; 7 | 8 | public class GeoInfo { 9 | private String countryIsoCode; 10 | private String country; 11 | private String region; 12 | private String city; 13 | private String continent; 14 | private String postalCode; 15 | private Double latitude; 16 | private Double longitude; 17 | 18 | public String getCountryIsoCode() { 19 | return countryIsoCode; 20 | } 21 | public void setCountryIsoCode(String countryIsoCode) { 22 | this.countryIsoCode = countryIsoCode; 23 | } 24 | public String getCountry() { 25 | return country; 26 | } 27 | public void setCountry(String country) { 28 | this.country = country; 29 | } 30 | public String getRegion() { 31 | return region; 32 | } 33 | public void setRegion(String region) { 34 | this.region = region; 35 | } 36 | public String getCity() { 37 | return city; 38 | } 39 | public void setCity(String city) { 40 | this.city = city; 41 | } 42 | public String getContinent() { 43 | return continent; 44 | } 45 | public void setContinent(String continent) { 46 | this.continent = continent; 47 | } 48 | public String getPostalCode() { 49 | return postalCode; 50 | } 51 | public void setPostalCode(String postalCode) { 52 | this.postalCode = postalCode; 53 | } 54 | public Double getLatitude() { 55 | return latitude; 56 | } 57 | public void setLatitude(Double latitude) { 58 | this.latitude = latitude; 59 | } 60 | public Double getLongitude() { 61 | return longitude; 62 | } 63 | public void setLongitude(Double longitude) { 64 | this.longitude = longitude; 65 | } 66 | } -------------------------------------------------------------------------------- /collector/src/main/java/com/ebay/pulsar/collector/validation/DefaultValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector.validation; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | public class DefaultValidator implements Validator { 12 | private List mandatoryTags; 13 | 14 | 15 | public List getMandatoryTags() { 16 | return mandatoryTags; 17 | } 18 | 19 | 20 | public void setMandatoryTags(List mandatoryTags) { 21 | this.mandatoryTags = mandatoryTags; 22 | } 23 | 24 | 25 | @Override 26 | public ValidationResult validate(Map event, String eventType) { 27 | if (mandatoryTags != null) { 28 | for (int i = 0, t = mandatoryTags.size(); i < t; i++) { 29 | if (!event.containsKey(mandatoryTags.get(i))) { 30 | ValidationResult r = new ValidationResult(); 31 | r.setSuccess(false); 32 | r.setDetail(mandatoryTags.get(i) + " is mandatory"); 33 | return r; 34 | } 35 | } 36 | } 37 | return null; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /collector/src/main/java/com/ebay/pulsar/collector/validation/ValidationResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector.validation; 7 | 8 | public class ValidationResult { 9 | private boolean success; 10 | 11 | private String detail; 12 | 13 | 14 | public boolean isSuccess() { 15 | return success; 16 | } 17 | 18 | public void setSuccess(boolean success) { 19 | this.success = success; 20 | } 21 | 22 | public String getDetail() { 23 | return detail; 24 | } 25 | 26 | public void setDetail(String detail) { 27 | this.detail = detail; 28 | } 29 | } -------------------------------------------------------------------------------- /collector/src/main/java/com/ebay/pulsar/collector/validation/Validator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector.validation; 7 | 8 | import java.util.Map; 9 | 10 | public interface Validator { 11 | ValidationResult validate(Map event, String eventType); 12 | } 13 | -------------------------------------------------------------------------------- /collector/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /collector/src/test/java/com/ebay/pulsar/collector/EnrichmentSimulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector; 7 | 8 | import com.ebay.pulsar.collector.udf.DeviceEnrichmentUtil; 9 | import com.ebay.pulsar.collector.udf.DeviceInfo; 10 | import com.ebay.pulsar.collector.udf.GeoEnrichmentUtil; 11 | import com.ebay.pulsar.collector.udf.GeoInfo; 12 | 13 | public class EnrichmentSimulator { 14 | public static void main(String[] args) throws Exception { 15 | GeoEnrichmentUtil geoEnrichmentUtil = GeoEnrichmentUtil.getInstance(); 16 | geoEnrichmentUtil.setGeoDBFilePath("buildsrc/geodb/GeoLite2-City.mmdb"); 17 | geoEnrichmentUtil.afterPropertiesSet(); 18 | @SuppressWarnings("static-access") 19 | GeoInfo geoInfo = geoEnrichmentUtil.getGeoInfo("192.128.10.5"); 20 | 21 | System.out.println("countryIsoCode: " + geoInfo.getCountryIsoCode()); 22 | System.out.println("country: " + geoInfo.getCountry()); 23 | System.out.println("region: " + geoInfo.getRegion()); 24 | System.out.println("city: " + geoInfo.getCity()); 25 | System.out.println("continent: " + geoInfo.getContinent()); 26 | System.out.println("postalCode: " + geoInfo.getPostalCode()); 27 | System.out.println("latitude: " + geoInfo.getLatitude()); 28 | System.out.println("longitude: " + geoInfo.getLongitude()); 29 | 30 | DeviceEnrichmentUtil deviceEnrichmentUtil = DeviceEnrichmentUtil.getInstance(); 31 | deviceEnrichmentUtil.afterPropertiesSet(); 32 | @SuppressWarnings("static-access") 33 | DeviceInfo deviceInfo = deviceEnrichmentUtil.getDeviceInfo("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D201 Safari/9537.53"); 34 | 35 | System.out.println("deviceCategory: " + deviceInfo.getDeviceCategory()); 36 | System.out.println("osFamily: " + deviceInfo.getOsFamily()); 37 | System.out.println("osVersion: " + deviceInfo.getOsVersion()); 38 | System.out.println("userAgent: " + deviceInfo.getUserAgent()); 39 | System.out.println("userAgentType: " + deviceInfo.getUserAgentType()); 40 | System.out.println("userAgentFamily: " + deviceInfo.getUserAgentFamily()); 41 | System.out.println("userAgentVersion: " + deviceInfo.getUserAgentVersion()); 42 | } 43 | } -------------------------------------------------------------------------------- /collector/src/test/java/com/ebay/pulsar/collector/Simulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.collector; 7 | 8 | import java.io.IOException; 9 | import java.io.UnsupportedEncodingException; 10 | import java.util.Arrays; 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | 14 | import org.apache.commons.httpclient.HttpClient; 15 | import org.apache.commons.httpclient.HttpException; 16 | import org.apache.commons.httpclient.methods.PostMethod; 17 | import org.apache.commons.httpclient.methods.StringRequestEntity; 18 | import org.codehaus.jackson.JsonGenerationException; 19 | import org.codehaus.jackson.JsonProcessingException; 20 | import org.codehaus.jackson.map.JsonMappingException; 21 | import org.codehaus.jackson.map.ObjectMapper; 22 | 23 | public class Simulator { 24 | 25 | public static void main(String[] args) throws Exception { 26 | for (int i = 0; i < 1; i++) { 27 | sendMessage(); 28 | } 29 | } 30 | 31 | private static void sendMessage() throws IOException, JsonProcessingException, JsonGenerationException, 32 | JsonMappingException, UnsupportedEncodingException, HttpException { 33 | ObjectMapper mapper = new ObjectMapper(); 34 | 35 | Map m = new HashMap(); 36 | 37 | m.put("si", "12345"); 38 | m.put("ct", System.currentTimeMillis()); 39 | 40 | String payload = mapper.writeValueAsString(m); 41 | HttpClient client = new HttpClient(); 42 | PostMethod method = new PostMethod("http://localhost:8080/tracking/ingest/PulsarRawEvent"); 43 | // method.addRequestHeader("Accept-Encoding", "gzip,deflate,sdch"); 44 | method.setRequestEntity(new StringRequestEntity(payload, "application/json", "UTF-8")); 45 | int status = client.executeMethod(method); 46 | System.out.println(Arrays.toString(method.getResponseHeaders())); 47 | System.out.println("Status code: " + status + ", Body: " + method.getResponseBodyAsString()); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /distributor/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name distributor 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # App config 13 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 14 | ENV JETSTREAM_APP_NAME ${project_name} 15 | ENV JETSTREAM_CONFIG_VERSION 1.0 16 | 17 | # Dependency 18 | ENV JETSTREAM_ZKSERVER_HOST zkserver 19 | ENV JETSTREAM_ZKSERVER_PORT 2181 20 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 21 | ENV PULSAR_KAFKA_BROKERS kafkaserver:9092 22 | 23 | # Four context ports 24 | ENV JETSTREAM_CONTEXT_BASEPORT 15590 25 | ENV JETSTREAM_APP_PORT 9999 26 | 27 | EXPOSE 9999 15590 15591 15592 15593 28 | ENTRYPOINT ./start.sh 29 | -------------------------------------------------------------------------------- /distributor/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 19 | false 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.build.directory} 46 | / 47 | 48 | *.jar 49 | 50 | 51 | *-javadoc.jar 52 | *-sources.jar 53 | 54 | 55 | 56 | 57 | ${project.basedir}/buildsrc/scripts 58 | / 59 | 60 | * 61 | 62 | 0755 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /distributor/buildsrc/JetstreamConf/EPL.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 13 | 14 | 15 | 16 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /distributor/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 13 | 14 | 16 | 18 | 19 | 20 | 21 | 22 | 24 | 25 | 26 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 36 | 38 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /distributor/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Dpulsar.runtime.kafka.brokers="$PULSAR_KAFKA_BROKERS" \ 37 | -Djetstream.context.baseport="${JETSTREAM_CONTEXT_BASEPORT:-15590}" \ 38 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 39 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 40 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 41 | -jar $JETSTREAM_APP_JAR_NAME \ 42 | -n $JETSTREAM_APP_NAME \ 43 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 44 | -p ${JETSTREAM_APP_PORT:-9999} 45 | -------------------------------------------------------------------------------- /distributor/src/main/java/com/ebay/pulsar/distributor/epl/func/DefaultValue.java: -------------------------------------------------------------------------------- 1 | package com.ebay.pulsar.distributor.epl.func; 2 | 3 | public class DefaultValue { 4 | public static Number NumberNullDefaultZero(Number v){ 5 | return v==null?0:v; 6 | } 7 | public static Number NumberNullDefault(Number v, Number defaultValue){ 8 | return v==null?defaultValue:v; 9 | } 10 | public static Integer IntegerNullDefaultZero(Integer v){ 11 | return v==null?0:v; 12 | } 13 | public static Integer IntegerNullDefault(Integer v, Integer defaultValue){ 14 | return v==null?defaultValue:v; 15 | } 16 | public static Long LongNullDefaultZero(Long v){ 17 | return v==null?0L:v; 18 | } 19 | public static Long LongNullDefault(Long v, Long defaultValue){ 20 | return v==null?defaultValue:v; 21 | } 22 | public static String StringNullDefaultEmpty(String v){ 23 | return v==null?"":v; 24 | } 25 | public static String StringNullDefault(String v, String defaultValue){ 26 | return v==null?defaultValue:v; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /distributor/src/main/java/com/ebay/pulsar/distributor/replay/MultipleTopicsOBCAdviceProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.distributor.replay; 7 | 8 | import org.springframework.jmx.export.annotation.ManagedResource; 9 | 10 | import com.ebay.jetstream.event.EventException; 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | import com.ebay.jetstream.event.JetstreamReservedKeys; 13 | import com.ebay.jetstream.event.support.AdviceProcessor; 14 | 15 | @ManagedResource(objectName = "Event/MultipleTopicsOBMAdviceProcessor", description = "Multiple topics outbound messaging channel Advice processor") 16 | public class MultipleTopicsOBCAdviceProcessor extends AdviceProcessor { 17 | @Override 18 | public void sendEvent(JetstreamEvent event) throws EventException { 19 | String key = JetstreamReservedKeys.EventReplayTopic.toString(); 20 | 21 | if (event.containsKey(key)) { 22 | String retryTopic = (String) event.get(key); 23 | if (retryTopic != null && retryTopic.length() != 0) { 24 | retryTopic = "Replay-" + retryTopic.replaceAll("/", "-"); 25 | event.setForwardingTopics(new String[] { retryTopic }); 26 | } 27 | } 28 | 29 | super.sendEvent(event); 30 | } 31 | } -------------------------------------------------------------------------------- /distributor/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /distributor/src/test/resources/placeholder.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulsarIO/realtime-analytics/3b1a82736fcf209978c07e95d01577d567346fa4/distributor/src/test/resources/placeholder.txt -------------------------------------------------------------------------------- /metriccalculator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name metriccalculator 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # App config 13 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 14 | ENV JETSTREAM_APP_NAME ${project_name} 15 | ENV JETSTREAM_CONFIG_VERSION 1.0 16 | 17 | # Dependency 18 | ENV JETSTREAM_ZKSERVER_HOST zkserver 19 | ENV JETSTREAM_ZKSERVER_PORT 2181 20 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 21 | ENV PULSAR_KAFKA_BROKERS kafkaserver:9092 22 | ENV PULSAR_CASSANDRA cassandraserver 23 | 24 | # Three context ports 25 | ENV JETSTREAM_CONTEXT_BASEPORT 15590 26 | ENV JETSTREAM_APP_PORT 9999 27 | 28 | EXPOSE 9999 15590 15591 15592 29 | ENTRYPOINT ./start.sh 30 | -------------------------------------------------------------------------------- /metriccalculator/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 19 | false 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.build.directory} 46 | / 47 | 48 | *.jar 49 | 50 | 51 | *-javadoc.jar 52 | *-sources.jar 53 | 54 | 55 | 56 | 57 | ${project.basedir}/buildsrc/scripts 58 | / 59 | 60 | * 61 | 62 | 0755 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /metriccalculator/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 13 | 14 | 16 | 18 | 19 | 20 | 21 | 22 | 24 | 25 | 26 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 36 | 38 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /metriccalculator/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Dpulsar.runtime.kafka.brokers="$PULSAR_KAFKA_BROKERS" \ 37 | -Dpulsar.runtime.cassandra="$PULSAR_CASSANDRA" \ 38 | -Djetstream.context.baseport="${JETSTREAM_CONTEXT_BASEPORT:-15590}" \ 39 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 40 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 41 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 42 | -jar $JETSTREAM_APP_JAR_NAME \ 43 | -n $JETSTREAM_APP_NAME \ 44 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 45 | -p ${JETSTREAM_APP_PORT:-9999} 46 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/cache/CacheManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.cache; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | 11 | import com.ebay.jetstream.util.offheap.MapBuilder; 12 | import com.ebay.jetstream.util.offheap.MapBuilder.Unit; 13 | import com.ebay.jetstream.util.offheap.OffHeapMemoryManager; 14 | import com.ebay.pulsar.metriccalculator.metric.MCMetricGroupDemension; 15 | import com.ebay.pulsar.metriccalculator.offheap.serializer.CounterSerializer; 16 | import com.ebay.pulsar.metriccalculator.offheap.serializer.GroupDemensionSerializer; 17 | import com.ebay.pulsar.metriccalculator.statistics.basic.Counter; 18 | 19 | public class CacheManager { 20 | 21 | public static final int GROUPCOUNTER_INIT_SIZE = 20000; 22 | 23 | public static Map getCounterCache() { 24 | return new ConcurrentHashMap( 25 | GROUPCOUNTER_INIT_SIZE); 26 | } 27 | 28 | public static Map getCounterOffHeapCache( 29 | String memoryManagerName, OffHeapCacheConfig config) { 30 | MapBuilder builder = MapBuilder.newBuilder(); 31 | GroupDemensionSerializer keySerializer = new GroupDemensionSerializer(); 32 | CounterSerializer valueSerialize = new CounterSerializer(); 33 | 34 | builder.withKeySerializer(keySerializer) 35 | .withValueSerializer(valueSerialize) 36 | .withBlockSize(config.getBlockSize()) 37 | .withHashCapacity(config.getHashCapacity()) 38 | .withPageSize(config.getMemoryPageSize()) 39 | .withUnit(Unit.B) 40 | .withMaxMemory( 41 | ((long) config.getPageNumer()) 42 | * config.getMemoryPageSize()); 43 | OffHeapMemoryManager memoryManager = builder.getOffHeapMemoryManager(); 44 | OffHeapMemoryManagerRegistry.getInstance().addMemoryManager( 45 | memoryManagerName, memoryManager); 46 | return builder.buildHashMap(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/cache/OffHeapCacheConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.cache; 7 | 8 | public class OffHeapCacheConfig { 9 | private int nativeMemoryInGB = 1; 10 | private int memoryPageSize = 8192; 11 | private int blockSize = 256; // Better to align with CPU cache size. In 12 | // existed prod env, it is 64 bytes. 13 | private int hashCapacity = 1 << 22; // 4M 14 | 15 | public int getBlockSize() { 16 | return blockSize; 17 | } 18 | 19 | public void setBlockSize(int blockSize) { 20 | this.blockSize = blockSize; 21 | } 22 | 23 | public int getHashCapacity() { 24 | return hashCapacity; 25 | } 26 | 27 | public void setHashCapacity(int hashCapacity) { 28 | this.hashCapacity = hashCapacity; 29 | } 30 | 31 | public int getMemoryPageSize() { 32 | return memoryPageSize; 33 | } 34 | 35 | public void setMemoryPageSize(int memoryPageSize) { 36 | this.memoryPageSize = memoryPageSize; 37 | } 38 | 39 | public int getNativeMemoryInGB() { 40 | return nativeMemoryInGB; 41 | } 42 | 43 | public void setNativeMemoryInGB(int nativeMemoryInGB) { 44 | this.nativeMemoryInGB = nativeMemoryInGB; 45 | } 46 | 47 | public int getPageNumer() { 48 | return (int) ((((long) nativeMemoryInGB) << 30) / memoryPageSize); 49 | } 50 | } -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/cache/OffHeapMemoryManagerRegistry.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.cache; 7 | 8 | import java.util.Collections; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | import com.ebay.jetstream.util.offheap.OffHeapMemoryManager; 13 | 14 | public class OffHeapMemoryManagerRegistry { 15 | 16 | private static OffHeapMemoryManagerRegistry m_instance = new OffHeapMemoryManagerRegistry(); 17 | 18 | private Map memoryManagers = new HashMap(); 19 | 20 | private OffHeapMemoryManagerRegistry() { 21 | } 22 | 23 | public static OffHeapMemoryManagerRegistry getInstance() { 24 | return m_instance; 25 | } 26 | 27 | public void addMemoryManager(String name, 28 | OffHeapMemoryManager fffHeapMemoryManager) { 29 | memoryManagers.put(name, fffHeapMemoryManager); 30 | } 31 | 32 | public Map getMemoryManagers() { 33 | return Collections.unmodifiableMap(memoryManagers); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/esper/aggregator/TopKNestedAggregator.java: -------------------------------------------------------------------------------- 1 | package com.ebay.pulsar.metriccalculator.esper.aggregator; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import com.clearspring.analytics.stream.Counter; 8 | import com.clearspring.analytics.stream.StreamSummary; 9 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 10 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 11 | import com.google.common.base.Splitter; 12 | 13 | public class TopKNestedAggregator implements AggregationMethod { 14 | 15 | StreamSummary m_counter; 16 | Integer m_capacity; 17 | Integer m_topElementCnt = new Integer(10); 18 | Splitter splitter; 19 | 20 | @Override 21 | public void enter(Object value) { 22 | 23 | Object[] params = (Object[]) value; 24 | 25 | if (m_counter == null) { 26 | m_capacity = (Integer) params[0]; 27 | m_topElementCnt = (Integer) params[1]; 28 | m_counter = new StreamSummary(m_capacity.intValue()); 29 | } 30 | 31 | if (splitter == null) { 32 | if (params[3] instanceof Character) { 33 | splitter = Splitter.on((Character) params[3]); 34 | } else if (params[3] instanceof String) { 35 | splitter = Splitter.on((String) params[3]); 36 | } 37 | } 38 | if (params[2] instanceof String && splitter != null) { 39 | Iterable it = splitter.split((String) params[2]); 40 | for (String nested : it) { 41 | m_counter.offer(nested); 42 | } 43 | } else { 44 | m_counter.offer(params[2]); 45 | } 46 | } 47 | 48 | @Override 49 | public void leave(Object value) { 50 | 51 | } 52 | 53 | @Override 54 | public Object getValue() { 55 | Map topN = new LinkedHashMap(); 56 | if ((m_counter == null)) 57 | return topN; 58 | 59 | List> topCounters = m_counter.topK(m_topElementCnt); 60 | 61 | for (Counter counter : topCounters) { 62 | topN.put(counter.getItem(), counter.getCount()); 63 | } 64 | return topN; 65 | } 66 | 67 | @Override 68 | public Class getValueType() { 69 | 70 | return Map.class; 71 | } 72 | 73 | @Override 74 | public void clear() { 75 | m_counter = null; 76 | 77 | } 78 | 79 | // @Override 80 | public void validate(AggregationValidationContext validationContext) { 81 | 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/esper/aggregator/TopKNestedAggregatorFactory.java: -------------------------------------------------------------------------------- 1 | package com.ebay.pulsar.metriccalculator.esper.aggregator; 2 | 3 | import java.util.Map; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import com.espertech.esper.client.hook.AggregationFunctionFactory; 9 | import com.espertech.esper.epl.agg.aggregator.AggregationMethod; 10 | import com.espertech.esper.epl.agg.service.AggregationValidationContext; 11 | 12 | public class TopKNestedAggregatorFactory implements AggregationFunctionFactory { 13 | 14 | private static final Logger LOGGER = LoggerFactory.getLogger("com.ebay.pulsar.metriccalculator.esper.aggregator"); 15 | 16 | @Override 17 | public void setFunctionName(String functionName) { 18 | // nothing needed here 19 | } 20 | 21 | // @Override 22 | public void validate(AggregationValidationContext validationContext) { 23 | if (validationContext.getParameterTypes().length != 4) 24 | LOGGER.error( "TopK Aggregation Function requires 4 parameters viz. max capacity, # top elements and element and split string, - topKNested(maxCapacit, topElements, element, split)"); 25 | } 26 | 27 | @Override 28 | public AggregationMethod newAggregator() { 29 | return new TopKNestedAggregator(); 30 | } 31 | 32 | @Override 33 | public Class getValueType() { 34 | return Map.class; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/metric/MetricFrequency.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.metric; 7 | 8 | public enum MetricFrequency { 9 | ONE_MINUTE(60000), FIVE_MINUTES(300000), TEN_MINUTES(600000), THIRTY_MINUTES( 10 | 1800000); 11 | 12 | private long period; 13 | 14 | private MetricFrequency(long period) { 15 | this.period = period; 16 | } 17 | 18 | public long getValue() { 19 | return period; 20 | } 21 | 22 | public static MetricFrequency valueOf(int freq) { 23 | for (MetricFrequency frequency : values()) { 24 | if (frequency.getValue() == freq) { 25 | return frequency; 26 | } 27 | } 28 | 29 | return null; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/offheap/serializer/AvgCounterSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.offheap.serializer; 7 | 8 | 9 | import java.nio.BufferOverflowException; 10 | import java.nio.ByteBuffer; 11 | import java.util.concurrent.atomic.AtomicLong; 12 | 13 | import com.ebay.jetstream.util.offheap.OffHeapSerializer; 14 | import com.ebay.jetstream.util.offheap.serializer.util.UnsignedLongEncoder; 15 | import com.ebay.pulsar.metriccalculator.statistics.basic.AvgCounter; 16 | 17 | public class AvgCounterSerializer implements OffHeapSerializer { 18 | private static ThreadLocal keyBuffer = new ThreadLocal() { 19 | @Override 20 | protected ByteBuffer initialValue() { 21 | return ByteBuffer.allocate(1024); 22 | } 23 | }; 24 | 25 | private final UnsignedLongEncoder encoder = new UnsignedLongEncoder(); 26 | 27 | @Override 28 | public AvgCounter deserialize(ByteBuffer data, int pos, int length) { 29 | AvgCounter counter = new AvgCounter(); 30 | counter.setTotalTime(new AtomicLong(encoder.decode(data))); 31 | counter.setSnapshotTimeValue(encoder.decode(data)); 32 | counter.setOldsnapshotTimeValue(encoder.decode(data)); 33 | counter.setTotalValue(new AtomicLong(encoder.decode(data))); 34 | counter.setSnapshotValue(encoder.decode(data)); 35 | counter.setOldsnapshotValue(encoder.decode(data)); 36 | return counter; 37 | } 38 | 39 | @Override 40 | public ByteBuffer serialize(AvgCounter v) { 41 | ByteBuffer buffer = keyBuffer.get(); 42 | buffer.clear(); 43 | while (true) { 44 | try { 45 | encoder.encode(v.getTotalTime().get(), buffer); 46 | encoder.encode(v.getSnapshotTimeValue(), buffer); 47 | encoder.encode(v.getOldsnapshotTimeValue(), buffer); 48 | 49 | encoder.encode(v.getTotalValue().get(), buffer); 50 | encoder.encode(v.getSnapshotValue(), buffer); 51 | encoder.encode(v.getOldsnapshotValue(), buffer); 52 | buffer.flip(); 53 | break; 54 | } catch (BufferOverflowException ex) { 55 | buffer = ByteBuffer.allocate(buffer.capacity() * 2); 56 | keyBuffer.set(buffer); 57 | } 58 | } 59 | return buffer; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/offheap/serializer/CounterSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.offheap.serializer; 7 | 8 | import java.nio.BufferOverflowException; 9 | import java.nio.ByteBuffer; 10 | import java.util.concurrent.atomic.AtomicLong; 11 | 12 | import com.ebay.jetstream.util.offheap.OffHeapSerializer; 13 | import com.ebay.jetstream.util.offheap.serializer.util.AsciiFirstStringEncoder; 14 | import com.ebay.jetstream.util.offheap.serializer.util.UnsignedLongEncoder; 15 | import com.ebay.pulsar.metriccalculator.statistics.basic.Counter; 16 | 17 | public class CounterSerializer implements OffHeapSerializer { 18 | 19 | private static ThreadLocal keyBuffer = new ThreadLocal() { 20 | @Override 21 | protected ByteBuffer initialValue() { 22 | return ByteBuffer.allocate(1024); 23 | } 24 | }; 25 | 26 | private final UnsignedLongEncoder encoder = new UnsignedLongEncoder(); 27 | private final AsciiFirstStringEncoder stringEncoder = new AsciiFirstStringEncoder(); 28 | 29 | @Override 30 | public Counter deserialize(ByteBuffer data, int pos, int length) { 31 | Counter counter = new Counter(); 32 | counter.setTotalValue(new AtomicLong(encoder.decode(data))); 33 | counter.setSnapshotValue(encoder.decode(data)); 34 | counter.setOldsnapshotValue(encoder.decode(data)); 35 | counter.setLastCounterTime(stringEncoder.decode(data)); 36 | return counter; 37 | } 38 | 39 | @Override 40 | public ByteBuffer serialize(Counter v) { 41 | ByteBuffer buffer = keyBuffer.get(); 42 | buffer.clear(); 43 | while (true) { 44 | try { 45 | encoder.encode(v.getTotalValue().get(), buffer); 46 | encoder.encode(v.getSnapshotValue(), buffer); 47 | encoder.encode(v.getOldsnapshotValue(), buffer); 48 | stringEncoder.encode(v.getLastCounterTime(), buffer); 49 | buffer.flip(); 50 | break; 51 | } catch (BufferOverflowException ex) { 52 | buffer = ByteBuffer.allocate(buffer.capacity() * 2); 53 | keyBuffer.set(buffer); 54 | } 55 | } 56 | return buffer; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/processor/MCMetricsProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.processor; 7 | 8 | import java.util.Set; 9 | 10 | public interface MCMetricsProvider { 11 | public void collectMetrics(Set counterNames, boolean flushNormal); 12 | } 13 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/processor/MCScheduler.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.processor; 7 | 8 | import java.util.concurrent.Executors; 9 | import java.util.concurrent.ScheduledExecutorService; 10 | 11 | import com.ebay.pulsar.metriccalculator.util.NamedThreadFactory; 12 | 13 | public class MCScheduler { 14 | 15 | private static ScheduledExecutorService timer = Executors 16 | .newScheduledThreadPool(1, 17 | new NamedThreadFactory("MCSchedulerTick")); 18 | 19 | public static ScheduledExecutorService getMCScheduler() { 20 | return timer; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/processor/configuration/MCSummingConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.processor.configuration; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.springframework.beans.factory.InitializingBean; 12 | 13 | import com.ebay.jetstream.config.AbstractNamedBean; 14 | import com.ebay.jetstream.xmlser.XSerializable; 15 | import com.ebay.pulsar.metriccalculator.cache.OffHeapCacheConfig; 16 | 17 | public class MCSummingConfiguration extends AbstractNamedBean implements 18 | XSerializable, InitializingBean { 19 | 20 | private boolean enableGroupByCounter = true; 21 | private Map metricsThreshold = new HashMap(); 22 | private Map offheapMetricConf = new HashMap(); 23 | private int groupCounterMax = 1000000; 24 | 25 | public Map getMetricsThreshold() { 26 | return metricsThreshold; 27 | } 28 | 29 | public void setMetricsThreshold(Map metricsThreshold) { 30 | this.metricsThreshold = metricsThreshold; 31 | } 32 | 33 | public Map getOffheapMetricConf() { 34 | return offheapMetricConf; 35 | } 36 | 37 | public void setOffheapMetricConf( 38 | Map offheapMetricConf) { 39 | this.offheapMetricConf = offheapMetricConf; 40 | } 41 | 42 | public boolean isEnableGroupByCounter() { 43 | return enableGroupByCounter; 44 | } 45 | 46 | public void setEnableGroupByCounter(boolean enableGroupByCounter) { 47 | this.enableGroupByCounter = enableGroupByCounter; 48 | } 49 | 50 | public int getGroupCounterMax() { 51 | return groupCounterMax; 52 | } 53 | 54 | public void setGroupCounterMax(int groupCounterMax) { 55 | this.groupCounterMax = groupCounterMax; 56 | } 57 | 58 | @Override 59 | public void afterPropertiesSet() throws Exception { 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/replay/MultipleTopicsOBCAdviceProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.replay; 7 | 8 | import org.springframework.jmx.export.annotation.ManagedResource; 9 | 10 | import com.ebay.jetstream.event.EventException; 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | import com.ebay.jetstream.event.JetstreamReservedKeys; 13 | import com.ebay.jetstream.event.support.AdviceProcessor; 14 | 15 | @ManagedResource(objectName = "Event/MultipleTopicsOBMAdviceProcessor", description = "Multiple topics outbound messaging channel Advice processor") 16 | public class MultipleTopicsOBCAdviceProcessor extends AdviceProcessor { 17 | @Override 18 | public void sendEvent(JetstreamEvent event) throws EventException { 19 | String key = JetstreamReservedKeys.EventReplayTopic.toString(); 20 | 21 | if (event.containsKey(key)) { 22 | String retryTopic = (String) event.get(key); 23 | if (retryTopic != null && retryTopic.length() != 0) { 24 | retryTopic = "Replay-" + retryTopic.replaceAll("/", "-"); 25 | event.setForwardingTopics(new String[] { retryTopic }); 26 | } 27 | } 28 | 29 | super.sendEvent(event); 30 | } 31 | } -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/statistics/basic/AvgCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.statistics.basic; 7 | 8 | import java.io.Externalizable; 9 | import java.io.IOException; 10 | import java.io.ObjectInput; 11 | import java.io.ObjectOutput; 12 | import java.util.concurrent.atomic.AtomicLong; 13 | 14 | public final class AvgCounter extends Counter implements Externalizable { 15 | private AtomicLong totalTime = new AtomicLong(0); 16 | private long oldsnapshotTimeValue; 17 | private long snapshotTimeValue; 18 | 19 | public AvgCounter() { 20 | } 21 | 22 | public void inc(long v, long time) { 23 | super.inc(v); 24 | totalTime.addAndGet(time); 25 | } 26 | 27 | public void mark() { 28 | super.mark(); 29 | oldsnapshotTimeValue = snapshotTimeValue; 30 | snapshotTimeValue = totalTime.get(); 31 | } 32 | 33 | public long getAvgValue() { 34 | long c = super.getTotalValue().get(); 35 | if (c > 0) { 36 | return totalTime.get() / c; 37 | } else { 38 | return 0L; 39 | } 40 | } 41 | 42 | public AtomicLong getTotalTime() { 43 | return totalTime; 44 | } 45 | 46 | public void setTotalTime(AtomicLong totalTime) { 47 | this.totalTime = totalTime; 48 | } 49 | 50 | public long getOldsnapshotTimeValue() { 51 | return oldsnapshotTimeValue; 52 | } 53 | 54 | public void setOldsnapshotTimeValue(long oldsnapshotTimeValue) { 55 | this.oldsnapshotTimeValue = oldsnapshotTimeValue; 56 | } 57 | 58 | public long getSnapshotTimeValue() { 59 | return snapshotTimeValue; 60 | } 61 | 62 | public void setSnapshotTimeValue(long snapshotTimeValue) { 63 | this.snapshotTimeValue = snapshotTimeValue; 64 | } 65 | 66 | public long getLatestAvgValue() { 67 | if (super.getLastDeltaValue() > 0) { 68 | return (snapshotTimeValue - oldsnapshotTimeValue) 69 | / super.getLastDeltaValue(); 70 | } else { 71 | return 0L; 72 | } 73 | } 74 | 75 | @Override 76 | public void writeExternal(ObjectOutput out) throws IOException { 77 | super.writeExternal(out); 78 | out.writeLong(totalTime.get()); 79 | out.writeLong(oldsnapshotTimeValue); 80 | out.writeLong(snapshotTimeValue); 81 | } 82 | 83 | @Override 84 | public void readExternal(ObjectInput in) throws IOException, 85 | ClassNotFoundException { 86 | super.readExternal(in); 87 | totalTime = new AtomicLong(in.readLong()); 88 | oldsnapshotTimeValue = in.readLong(); 89 | snapshotTimeValue = in.readLong(); 90 | } 91 | } -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/statistics/basic/Counter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.statistics.basic; 7 | 8 | import java.io.Externalizable; 9 | import java.io.IOException; 10 | import java.io.ObjectInput; 11 | import java.io.ObjectOutput; 12 | import java.util.concurrent.atomic.AtomicLong; 13 | 14 | public class Counter implements Externalizable { 15 | private AtomicLong totalValue = new AtomicLong(0); 16 | private long oldsnapshotValue; 17 | private long snapshotValue; 18 | private String lastCounterTime; 19 | 20 | public Counter() { 21 | } 22 | 23 | public long getOldsnapshotValue() { 24 | return oldsnapshotValue; 25 | } 26 | 27 | public void setOldsnapshotValue(long oldsnapshotValue) { 28 | this.oldsnapshotValue = oldsnapshotValue; 29 | } 30 | 31 | public long getSnapshotValue() { 32 | return snapshotValue; 33 | } 34 | 35 | public void setSnapshotValue(long snapshotValue) { 36 | this.snapshotValue = snapshotValue; 37 | } 38 | 39 | public void setTotalValue(AtomicLong totalValue) { 40 | this.totalValue = totalValue; 41 | } 42 | 43 | public void inc() { 44 | totalValue.incrementAndGet(); 45 | } 46 | 47 | public long getAndIncrement() { 48 | return totalValue.getAndIncrement(); 49 | } 50 | 51 | public void inc(long v) { 52 | totalValue.addAndGet(v); 53 | } 54 | 55 | public long getLastDeltaValue() { 56 | return (snapshotValue - oldsnapshotValue); 57 | } 58 | 59 | public void mark() { 60 | oldsnapshotValue = snapshotValue; 61 | snapshotValue = totalValue.get(); 62 | } 63 | 64 | public AtomicLong getTotalValue() { 65 | return totalValue; 66 | } 67 | 68 | public String getLastCounterTime() { 69 | return lastCounterTime; 70 | } 71 | 72 | public void setLastCounterTime(String lastCounterTime) { 73 | this.lastCounterTime = lastCounterTime; 74 | } 75 | 76 | @Override 77 | public void writeExternal(ObjectOutput out) throws IOException { 78 | out.writeLong(totalValue.get()); 79 | out.writeLong(oldsnapshotValue); 80 | out.writeLong(snapshotValue); 81 | out.writeObject(lastCounterTime); 82 | } 83 | 84 | @Override 85 | public void readExternal(ObjectInput in) throws IOException, 86 | ClassNotFoundException { 87 | totalValue = new AtomicLong(in.readLong()); 88 | oldsnapshotValue = in.readLong(); 89 | snapshotValue = in.readLong(); 90 | Object obj = in.readObject(); 91 | if (obj != null) 92 | lastCounterTime = (String) obj; 93 | } 94 | } -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/util/MCConstant.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.util; 7 | 8 | public class MCConstant { 9 | public static final String METRIC_NAME = "metricName"; 10 | public static final String METRIC_FREQUENCY = "metricFrequency"; 11 | public static final String METRIC_COUNT = "count"; 12 | public static final String GROUP_ID = "groupId"; 13 | public static final String AGGREGATED_COUNT = "value"; 14 | public static final String METRIC_TIME = "metricTime"; 15 | public static final String METRIC_DEMENSION = "metricDemension"; 16 | public static final String FREQUENCY_IN_MIN = "frequencyInMin"; 17 | public static final String SHUTDOWN_FLUSH = "ShutDown"; 18 | public static final String FIRST_FLUSH = "FirstFlush"; 19 | public static String TAG_TIME_IGNORE = "tag_time"; 20 | } 21 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/util/MCCounterHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.util; 7 | 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | import com.ebay.jetstream.event.JetstreamEvent; 12 | 13 | public class MCCounterHelper { 14 | public static String COUNTEREVENT = "MC_Metric"; 15 | public static String MultiCOUNTEREVENT = "MC_MultiMetric"; 16 | public static String COUNTERGROUPBYEVENT = "group"; 17 | public static String COUNTERGROUPBYEVENTTAG = "tag"; 18 | public static String COUNTERGROUPBYMETICTIMEEVENT = "mctimegroup"; 19 | 20 | public static String TAG_METRICTIME = "tag_" + MCConstant.METRIC_TIME; 21 | 22 | public static String AVG = "avg"; 23 | 24 | public static boolean isMCCounterEvent(JetstreamEvent event) { 25 | if ((event != null) && event.getEventType().contains(COUNTEREVENT)) { 26 | return true; 27 | } 28 | return false; 29 | } 30 | 31 | public static boolean isMCMultiCounterEvent(JetstreamEvent event) { 32 | if ((event != null) && event.getEventType().contains(MultiCOUNTEREVENT)) { 33 | return true; 34 | } 35 | return false; 36 | } 37 | 38 | public static boolean isGroupByCounterEvent(JetstreamEvent event) { 39 | if ((event != null) 40 | && event.getEventType().contains(COUNTERGROUPBYEVENT)) { 41 | return true; 42 | } 43 | return false; 44 | } 45 | 46 | public static boolean isGroupByCounterEventWithTag(JetstreamEvent event, 47 | Map tags) { 48 | boolean result = false; 49 | if ((event != null) 50 | && event.getEventType().contains(COUNTERGROUPBYEVENT)) { 51 | Set keys = event.keySet(); 52 | String[] keyArray = keys.toArray(new String[0]); 53 | for (int i = 0; i < keyArray.length; i++) { 54 | if (keyArray[i].contains(COUNTERGROUPBYEVENTTAG)) { 55 | result = true; 56 | if (tags != null) { 57 | tags.put(keyArray[i], 58 | String.valueOf(event.get(keyArray[i]))); 59 | } 60 | } 61 | } 62 | 63 | if (event.getEventType().contains(COUNTERGROUPBYMETICTIMEEVENT)) { 64 | tags.put(TAG_METRICTIME, 65 | String.valueOf(event.get(MCConstant.METRIC_TIME))); 66 | } 67 | } 68 | return result; 69 | } 70 | 71 | public static boolean isAvgEvent(String metricName) { 72 | if (metricName.toLowerCase().contains(AVG)) { 73 | return true; 74 | } 75 | return false; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/util/NamedThreadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.util; 7 | 8 | import java.util.concurrent.ThreadFactory; 9 | import java.util.concurrent.atomic.AtomicInteger; 10 | 11 | public class NamedThreadFactory implements ThreadFactory { 12 | protected final String id; 13 | private final int priority; 14 | protected final AtomicInteger n = new AtomicInteger(1); 15 | 16 | public NamedThreadFactory(String id) { 17 | this(id, Thread.NORM_PRIORITY); 18 | } 19 | 20 | public NamedThreadFactory(String id, int priority) { 21 | 22 | this.id = id; 23 | this.priority = priority; 24 | } 25 | 26 | public Thread newThread(Runnable runnable) { 27 | String name = id + ":" + n.getAndIncrement(); 28 | Thread thread = new Thread(runnable, name); 29 | thread.setPriority(priority); 30 | thread.setDaemon(true); 31 | return thread; 32 | } 33 | } -------------------------------------------------------------------------------- /metriccalculator/src/main/java/com/ebay/pulsar/metriccalculator/util/ThreadSafeDateParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.metriccalculator.util; 7 | 8 | import java.text.DateFormat; 9 | import java.text.ParseException; 10 | import java.text.SimpleDateFormat; 11 | import java.util.Date; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | import java.util.TimeZone; 15 | 16 | public class ThreadSafeDateParser { 17 | private static final ThreadLocal> PARSERS = new ThreadLocal>() { 18 | protected Map initialValue() { 19 | return new HashMap(); 20 | } 21 | }; 22 | 23 | static private final DateFormat getParser(final String Pattern, 24 | final TimeZone zone) { 25 | Map parserMap = PARSERS.get(); 26 | DateFormat df = parserMap.get(Pattern); 27 | if (null == df) { 28 | // if parser for the same pattern does not exist yet, create one and 29 | // save it into map 30 | 31 | df = new SimpleDateFormat(Pattern); 32 | df.setTimeZone(zone); 33 | parserMap.put(Pattern, df); 34 | } 35 | return df; 36 | } 37 | 38 | static public Date parse(final String StrDate, final String Pattern, 39 | final TimeZone zone) throws ParseException { 40 | return getParser(Pattern, zone).parse(StrDate); 41 | } 42 | 43 | static public String format(final Date date, final String Pattern, 44 | final TimeZone zone) throws ParseException { 45 | return getParser(Pattern, zone).format(date); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /metriccalculator/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /replay/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name replay 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # App config 13 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 14 | ENV JETSTREAM_APP_NAME ${project_name} 15 | ENV JETSTREAM_CONFIG_VERSION 1.0 16 | 17 | 18 | # Dependency 19 | ENV JETSTREAM_ZKSERVER_HOST zkserver 20 | ENV JETSTREAM_ZKSERVER_PORT 2181 21 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 22 | ENV PULSAR_KAFKA_ZK kafkaserver:2181 23 | ENV PULSAR_KAFKA_BROKERS kafkaserver:9092 24 | 25 | # Three Context ports, actually not used 26 | ENV JETSTREAM_CONTEXT_BASEPORT 15590 27 | ENV JETSTREAM_APP_PORT 9999 28 | 29 | EXPOSE 9999 15590 15591 15592 30 | ENTRYPOINT ./start.sh 31 | -------------------------------------------------------------------------------- /replay/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | false 19 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.build.directory} 46 | / 47 | 48 | *.jar 49 | 50 | 51 | *-javadoc.jar 52 | *-sources.jar 53 | 54 | 55 | 56 | 57 | ${project.basedir}/buildsrc/scripts 58 | / 59 | 60 | *.* 61 | 62 | 0755 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /replay/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /replay/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Dpulsar.runtime.kafka.brokers="$PULSAR_KAFKA_BROKERS" \ 37 | -Dpulsar.runtime.kafka.zk="$PULSAR_KAFKA_ZK" \ 38 | -Djetstream.context.baseport="${JETSTREAM_CONTEXT_BASEPORT:-15590}" \ 39 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 40 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 41 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 42 | -jar $JETSTREAM_APP_JAR_NAME \ 43 | -n $JETSTREAM_APP_NAME \ 44 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 45 | -p ${JETSTREAM_APP_PORT:-9999} 46 | -------------------------------------------------------------------------------- /replay/src/main/java/com/ebay/pulsar/replay/processor/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.replay.processor; 7 | 8 | public class Constants { 9 | public static final String REPLAY_INITIAL_TS = "rply_init_ts"; 10 | public static final String REPLAY_LAST_TS = "rply_last_ts"; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /replay/src/main/java/com/ebay/pulsar/replay/processor/ReplayKafkaProcessorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.replay.processor; 7 | 8 | import com.ebay.jetstream.event.processor.kafka.SimpleKafkaProcessorConfig; 9 | 10 | /** 11 | * @author weifang 12 | * 13 | */ 14 | public class ReplayKafkaProcessorConfig extends SimpleKafkaProcessorConfig { 15 | public static final String KAFKA_TIMESTAMP_KEY = "js_evt_kafka_produce_ts"; 16 | public static final int DFLT_MAX_READ_RATE = 5000; 17 | 18 | private long delayInMs = 60 * 1000; 19 | 20 | private long retentionInMs = 24 * 3600 * 1000; // default 6 hours. 21 | 22 | private String timestampKey = KAFKA_TIMESTAMP_KEY; 23 | 24 | public void setDelayInMs(long delayInMs) { 25 | this.delayInMs = delayInMs; 26 | } 27 | 28 | public void setTimestampKey(String timestampKey) { 29 | this.timestampKey = timestampKey; 30 | } 31 | 32 | public long getRetentionInMs() { 33 | return retentionInMs; 34 | } 35 | 36 | public void setRetentionInMs(long retentionInMs) { 37 | this.retentionInMs = retentionInMs; 38 | } 39 | 40 | public long getDelayInMs() { 41 | return delayInMs; 42 | } 43 | 44 | public String getTimestampKey() { 45 | return timestampKey; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /replay/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /sessionizer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:openjdk-7-jdk 2 | 3 | ENV project_version 1.0.2 4 | ENV project_name sessionizer 5 | 6 | COPY target/${project_name}-${project_version}-bin.tar.gz /opt/app/${project_name}-${project_version}-bin.tar.gz 7 | WORKDIR /opt/app 8 | RUN tar -zxvf ${project_name}-${project_version}-bin.tar.gz 9 | RUN ln -s /opt/app/${project_name}-${project_version} jetstreamapp 10 | WORKDIR /opt/app/jetstreamapp 11 | 12 | # App config 13 | ENV JETSTREAM_APP_JAR_NAME ${project_name}.jar 14 | ENV JETSTREAM_APP_NAME ${project_name} 15 | ENV JETSTREAM_CONFIG_VERSION 1.0 16 | 17 | # Dependency 18 | ENV JETSTREAM_ZKSERVER_HOST zkserver 19 | ENV JETSTREAM_ZKSERVER_PORT 2181 20 | ENV JETSTREAM_MONGOURL mongo://mongoserver:27017/config 21 | ENV PULSAR_KAFKA_BROKERS kafkaserver:9092 22 | 23 | # Two Context ports 24 | ENV JETSTREAM_CONTEXT_BASEPORT 15590 25 | ENV JETSTREAM_APP_PORT 9999 26 | 27 | EXPOSE 9999 15590 15591 28 | ENTRYPOINT ./start.sh 29 | -------------------------------------------------------------------------------- /sessionizer/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 11 | bin 12 | 13 | tar.gz 14 | 15 | 16 | 17 | /lib 18 | false 19 | ${artifact.artifactId}-${artifact.baseVersion}.${artifact.extension} 20 | false 21 | runtime 22 | 23 | 24 | 25 | 26 | 27 | ${project.basedir} 28 | / 29 | 30 | README* 31 | LICENSE* 32 | NOTICE* 33 | 34 | 35 | 36 | 37 | ${project.basedir}/buildsrc/JetstreamConf 38 | JetstreamConf 39 | 40 | ** 41 | 42 | 43 | 44 | 45 | ${project.build.directory} 46 | / 47 | 48 | *.jar 49 | 50 | 51 | *-javadoc.jar 52 | *-sources.jar 53 | 54 | 55 | 56 | 57 | ${project.basedir}/buildsrc/scripts 58 | / 59 | 60 | * 61 | 62 | 0755 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /sessionizer/buildsrc/JetstreamConf/zookeeper.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 13 | 14 | 15 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /sessionizer/buildsrc/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Default NIC use eth0 3 | export JETSTREAM_NETMASK=`ip -f inet addr show eth0 | grep inet | awk "{print \\$2}"` 4 | 5 | # JETSTREAM_MONGOURL must be in this form mongo://:/Jetstream 6 | if [ ! -z "$JETSTREAM_MONGOURL" ]; then 7 | export MONGO_HOME=$JETSTREAM_MONGOURL 8 | fi 9 | 10 | export JETSTREAM_HOME=$(pwd) 11 | 12 | if [ -z "$JETSTREAM_APP_JAR_NAME" ]; then 13 | list=$(echo *.jar) 14 | count=0 15 | 16 | for str in $list 17 | do 18 | JETSTREAM_APP_JAR_NAME=$str 19 | if [ -z "$JETSTREAM_APP_NAME" ]; then 20 | JETSTREAM_APP_NAME=`echo "$str" | cut -d"." -f1` 21 | fi 22 | count=`expr $count + 1` 23 | if [ $count -gt 1 ] 24 | then 25 | echo "Aborting! more than 1 app in current folder" 26 | exit 1 27 | else 28 | echo "found app $JETSTREAM_APP_NAME" 29 | fi 30 | done 31 | fi 32 | 33 | 34 | java \ 35 | $JETSTREAM_JAVA_OPTS \ 36 | -Dpulsar.runtime.kafka.brokers="$PULSAR_KAFKA_BROKERS" \ 37 | -Djetstream.context.baseport="${JETSTREAM_CONTEXT_BASEPORT:-15590}" \ 38 | -Djetstream.runtime.zkserver.host="${JETSTREAM_ZKSERVER_HOST:-127.0.0.1}" \ 39 | -Djetstream.runtime.zkserver.port="${JETSTREAM_ZKSERVER_PORT:-2181}" \ 40 | -Djetstream.runtime.netmask="$JETSTREAM_NETMASK" \ 41 | -jar $JETSTREAM_APP_JAR_NAME \ 42 | -n $JETSTREAM_APP_NAME \ 43 | -cv ${JETSTREAM_CONFIG_VERSION:-1.0} \ 44 | -p ${JETSTREAM_APP_PORT:-9999} 45 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/cache/MemoryCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.cache; 7 | 8 | import com.ebay.pulsar.sessionizer.model.Session; 9 | 10 | /** 11 | * Local off-heap cache. 12 | * 13 | * It is a linked-hash cache with expiration support. It includes 14 | * a linked hash map and an address to blob map. 15 | * 16 | * @author xingwang 17 | */ 18 | public interface MemoryCache { 19 | Session get(String key); 20 | 21 | long getReservedMemory(); 22 | 23 | long getUsedMemory(); 24 | 25 | long getFreeMemory(); 26 | 27 | long getMaxMemory(); 28 | 29 | long getSize(); 30 | 31 | boolean put(String key, Session value); 32 | 33 | boolean remove(String key, Session value); 34 | 35 | Session removeExpired(long timestamp); 36 | 37 | Session removeFirst(); 38 | 39 | long getOOMErrorCount(); 40 | 41 | int getMaxItemSize(); 42 | 43 | void resetMaxItemSize(); 44 | } 45 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/cache/MemoryCacheConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.cache; 7 | 8 | /** 9 | * Sessionizer off-heap cache config. 10 | * 11 | * @author xingwang 12 | * 13 | */ 14 | public interface MemoryCacheConfig { 15 | /** 16 | * Block size. 17 | * 18 | * @return 19 | */ 20 | int getBlockSize(); 21 | 22 | /** 23 | * Hash capacity of the cache. 24 | * 25 | * @return 26 | */ 27 | int getHashCapacity(); 28 | 29 | /** 30 | * Max time slots. 31 | * 32 | * @return 33 | */ 34 | int getMaxTimeSlots(); 35 | 36 | /** 37 | * Memory page size. 38 | * 39 | * @return 40 | */ 41 | int getMemoryPageSize(); 42 | 43 | /** 44 | * Total page numbers. 45 | * 46 | * @return 47 | */ 48 | int getPageNumer(); 49 | 50 | /** 51 | * Thread number. 52 | * 53 | * @return 54 | */ 55 | int getThreadNum(); 56 | } 57 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/cluster/ClusterManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.cluster; 7 | 8 | import com.ebay.pulsar.sessionizer.impl.SessionizerProcessor; 9 | 10 | /** 11 | * Sessionizer cluster manager. 12 | * 13 | * It use a loopback consistent hash ring to determine the whole cluster info. 14 | * 15 | * @author xingwang 16 | * 17 | */ 18 | public interface ClusterManager { 19 | /** 20 | * Max idle time of the session. 21 | * 22 | * @return 23 | */ 24 | long getMaxIdleTime(); 25 | 26 | /** 27 | * The current host Id. 28 | * 29 | * @return 30 | */ 31 | long getHostId(); 32 | 33 | /** 34 | * Check whether the ownership changed recently on the key. 35 | * 36 | * @param affinityKey 37 | * @return 38 | */ 39 | boolean isOwnershipChangedRecently(T affinityKey); 40 | 41 | /** 42 | * Check whether current node has the ownership of the key. 43 | * 44 | * @param affinityKey 45 | * @return 46 | */ 47 | boolean hasOwnership(T affinityKey); 48 | 49 | /** 50 | * Is current node the leader. 51 | * 52 | * @return 53 | */ 54 | boolean isLeader(); 55 | 56 | /** 57 | * Check whether the host specified by the hostId live or not 58 | * @param hostId 59 | * @return 60 | */ 61 | boolean isHostLive(long hostId); 62 | 63 | /** 64 | * Inject the sessionizer processor. 65 | * 66 | * @param processor 67 | */ 68 | void setSessionizer(SessionizerProcessor processor); 69 | 70 | /** 71 | * Get the host failure time. 72 | * 73 | * @param hostId 74 | * @return 75 | */ 76 | long getHostFailureTime(long hostId); 77 | } 78 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/config/SubSessionProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.config; 7 | 8 | /** 9 | * Sub Session profile. 10 | * 11 | * An marker class for future extension. 12 | * 13 | * @author xingwang 14 | */ 15 | public class SubSessionProfile extends CommonSessionProfile { 16 | 17 | } 18 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/AppendState.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * This annotation to be used to maintain a session variable with a list type. 10 | * 11 | * It will append the selected column value into the list, and if the list reach 12 | * the max length, it will evict the oldest one. 13 | * 14 | * @author xingwang 15 | * 16 | */ 17 | public @interface AppendState { 18 | /** 19 | * Variable name. 20 | * 21 | * @return 22 | */ 23 | String name(); 24 | 25 | /** 26 | * Column name to select value. 27 | * 28 | * @return 29 | */ 30 | String colname(); 31 | 32 | /** 33 | * Indicate whether the value must be unique. 34 | * 35 | * @return 36 | */ 37 | boolean unique() default true; 38 | 39 | /** 40 | * Max length of the list. Any value less or equals 0 means no limitation. 41 | * @return 42 | */ 43 | int maxlength() default 0; 44 | } 45 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/DebugSession.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * This only used for debugging. 10 | * 11 | * There is a counter to count the statement be executed. And if the colname 12 | * is available, it will create a counter for each value and use the counter name 13 | * as prefix. The audit will be used to log the event detail. 14 | * 15 | * 16 | * @author xingwang 17 | * 18 | */ 19 | public @interface DebugSession { 20 | /** 21 | * Counter name. 22 | * 23 | * @return 24 | */ 25 | String counter(); 26 | String colname() default ""; 27 | boolean audit() default false; 28 | } 29 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/DecorateEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * Insert the selected event into the raw event. 10 | * 11 | * @author xingwang 12 | * 13 | */ 14 | public @interface DecorateEvent { 15 | } 16 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/Session.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * An annotation to provide hint for the sessionizer. 10 | * 11 | * The value of the annotation is a session profile name, and the EPL statement 12 | * should return the session identifier (_pk_) of the event, and it must be a String. 13 | * And it can also return two optional tags: _timestamp_ to indicate the Event timestamp, 14 | * the _timestamp_ type must be Long, if it is not available, it will use system current time; 15 | * _duration_ to indicate max inactivity time of the session, if it is not available, it will 16 | * use default ttl from the session profile. 17 | * 18 | * @author xingwang 19 | * 20 | */ 21 | public @interface Session { 22 | /** 23 | * The name of the session profile. 24 | * 25 | * @return 26 | */ 27 | String value(); 28 | } 29 | 30 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/SubSession.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * An annotation to provide hint to run sessionizer on sub session. 10 | * 11 | * The value of the annotation is a sub session profile name, and the EPL statement 12 | * should return the sub session identifier (_pk_) of the event, and it must be a String. 13 | * And it can also return a optional tag _duration_ with Long type to indicate max inactivity 14 | * time of the sub session, if it is not available, it will use default ttl from the sub session profile. 15 | * 16 | * @author xingwang 17 | * 18 | */ 19 | public @interface SubSession { 20 | /** 21 | * The name of the sub session profile. 22 | * 23 | * @return 24 | */ 25 | String value(); 26 | } 27 | 28 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/UpdateCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * Increment a session counter with the selected event count. 10 | * 11 | * The sessionizer will auto create a counter if it does not exist. 12 | * 13 | * @author xingwang 14 | * 15 | */ 16 | public @interface UpdateCounter { 17 | /** 18 | * Counter name. 19 | * 20 | * @return 21 | */ 22 | String value(); 23 | } 24 | 25 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/UpdateDuration.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * Change session duration. 10 | * 11 | * @author xingwang 12 | * 13 | */ 14 | public @interface UpdateDuration { 15 | } 16 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/UpdateMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * Save the new event and store it on the session metadata. 10 | * 11 | * The new event must be flat map and value must be primitive value. 12 | * 13 | * @author xingwang 14 | */ 15 | public @interface UpdateMetadata { 16 | 17 | } 18 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/annotation/UpdateState.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.annotation; 7 | 8 | /** 9 | * Save the new event and store it on the session variables. 10 | * 11 | * The new event must be a flat map and value must be primitive value. 12 | * 13 | * @author xingwang 14 | * 15 | */ 16 | public @interface UpdateState { 17 | 18 | } 19 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/impl/SessionVariable.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.impl; 7 | 8 | import com.ebay.pulsar.sessionizer.model.AbstractSession; 9 | import com.ebay.pulsar.sessionizer.model.Session; 10 | 11 | /** 12 | * Session variable for the Esper. 13 | * 14 | * @author xingwang 15 | * 16 | */ 17 | public class SessionVariable extends AttributeMapVariable { 18 | private AbstractSession data; 19 | private Session mainSession; 20 | 21 | public String getSessionId() { 22 | return data.getSessionId(); 23 | } 24 | 25 | public long getDuration() { 26 | return data.getDuration(); 27 | } 28 | public int getBotEventCount() { 29 | return mainSession.getBotEventCount(); 30 | } 31 | 32 | public int getBotType() { 33 | return mainSession.getBotType(); 34 | } 35 | 36 | public long getCreationTime() { 37 | return data.getCreationTime(); 38 | } 39 | 40 | public int getEventCount() { 41 | return data.getEventCount(); 42 | } 43 | 44 | public long getExpirationTime() { 45 | return data.getExpirationTime(); 46 | } 47 | 48 | public long getFirstEventTimestamp() { 49 | return data.getFirstEventTimestamp(); 50 | } 51 | 52 | public String getIdentifier() { 53 | return data.getIdentifier(); 54 | } 55 | 56 | 57 | public long getLastModifiedTime() { 58 | return data.getLastModifiedTime(); 59 | } 60 | 61 | 62 | public void resetSessionData(AbstractSession data, Session mainSession) { 63 | this.data = data; 64 | this.mainSession = mainSession; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/esper/impl/SessionizerEsperExceptionHandlerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.esper.impl; 7 | 8 | import com.ebay.pulsar.sessionizer.impl.SessionizerErrorManager; 9 | import com.espertech.esper.client.hook.ExceptionHandler; 10 | import com.espertech.esper.client.hook.ExceptionHandlerContext; 11 | import com.espertech.esper.client.hook.ExceptionHandlerFactory; 12 | import com.espertech.esper.client.hook.ExceptionHandlerFactoryContext; 13 | 14 | /** 15 | * Register Esper errors to unified error manager. 16 | * 17 | * @author xingwang 18 | * 19 | */ 20 | public class SessionizerEsperExceptionHandlerFactory implements ExceptionHandlerFactory { 21 | private static class SessionizerExceptionHandler implements ExceptionHandler { 22 | 23 | @Override 24 | public void handle(ExceptionHandlerContext context) { 25 | if (errorManager != null) { 26 | errorManager.registerError(context.getThrowable(), SessionizerErrorManager.ErrorType.Esper); 27 | errorManager.registerError(context.getEngineURI(), SessionizerErrorManager.ErrorType.Esper, context.getEpl()); 28 | } 29 | } 30 | 31 | } 32 | 33 | private static SessionizerErrorManager errorManager; 34 | 35 | public static void setErrorManager(SessionizerErrorManager errorManager) { 36 | SessionizerEsperExceptionHandlerFactory.errorManager = errorManager; 37 | } 38 | 39 | @Override 40 | public ExceptionHandler getHandler(ExceptionHandlerFactoryContext context) { 41 | return new SessionizerExceptionHandler(); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/impl/EsperSessionizerCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.impl; 7 | 8 | import java.util.Date; 9 | import java.util.Map; 10 | import java.util.concurrent.ConcurrentHashMap; 11 | 12 | import org.springframework.jmx.export.annotation.ManagedAttribute; 13 | import org.springframework.jmx.export.annotation.ManagedOperation; 14 | import org.springframework.jmx.export.annotation.ManagedResource; 15 | 16 | import com.ebay.jetstream.counter.LongCounter; 17 | import com.ebay.jetstream.messaging.transport.netty.eventconsumer.EventConsumer; 18 | 19 | /** 20 | * A counter manager for troubleshooting purpose. 21 | * 22 | * @author xingwang 23 | * 24 | */ 25 | @ManagedResource(objectName = "EsperSessionizerCounter", description = "Counter for debugging") 26 | public class EsperSessionizerCounter { 27 | private final Map counterMap = new ConcurrentHashMap(); 28 | private Date startTime = new Date(); 29 | 30 | private final SessionizerProcessor processor; 31 | 32 | public EsperSessionizerCounter(SessionizerProcessor processor) { 33 | this.processor = processor; 34 | } 35 | 36 | 37 | public void addCount(String name, int count) { 38 | LongCounter counter = counterMap.get(name); 39 | if (counter == null) { 40 | counter = new LongCounter(); 41 | counterMap.put(name, counter); 42 | } 43 | counter.addAndGet(count); 44 | } 45 | 46 | @ManagedAttribute 47 | public Map getCustomizedCounters() { 48 | return counterMap; 49 | } 50 | 51 | @ManagedAttribute 52 | public Map getBuiltInCounters() { 53 | return processor.collectDetailStatistics(); 54 | } 55 | 56 | 57 | @ManagedAttribute 58 | public long getHostd() { 59 | return EventConsumer.getConsumerId(); 60 | } 61 | 62 | @ManagedAttribute 63 | public long getServerCurrentTime() { 64 | return System.currentTimeMillis(); 65 | } 66 | 67 | public String getResetTime() { 68 | return startTime.toString(); 69 | } 70 | 71 | @ManagedOperation 72 | public void reset() { 73 | counterMap.clear(); 74 | startTime = new Date(); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/impl/SessionizationInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.impl; 7 | 8 | /** 9 | * Hints for sessionizer processor. 10 | * 11 | * @author xingwang 12 | * 13 | */ 14 | public class SessionizationInfo { 15 | private int ttl = 30 * 60 * 1000; 16 | private long timestamp; 17 | private String identifier; 18 | private String name; 19 | public String getIdentifier() { 20 | return identifier; 21 | } 22 | 23 | public String getName() { 24 | return name; 25 | } 26 | 27 | public long getTimestamp() { 28 | return timestamp; 29 | } 30 | 31 | public int getTtl() { 32 | return ttl; 33 | } 34 | public void setIdentifier(String identifier) { 35 | this.identifier = identifier; 36 | } 37 | public void setName(String name) { 38 | this.name = name; 39 | } 40 | public void setTimestamp(long timestamp) { 41 | this.timestamp = timestamp; 42 | } 43 | public void setTtl(int ttl) { 44 | this.ttl = ttl; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/impl/SessionizerTimerShutdownHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.impl; 7 | 8 | import com.ebay.jetstream.common.ShutDownable; 9 | 10 | /** 11 | * Help bean for sessionizer to shutdown timer first. 12 | * 13 | * @author xingwang 14 | * 15 | */ 16 | public class SessionizerTimerShutdownHelper implements ShutDownable { 17 | private SessionizerProcessor processor; 18 | 19 | 20 | @Override 21 | public int getPendingEvents() { 22 | return 0; 23 | } 24 | 25 | public void setProcessor(SessionizerProcessor processor) { 26 | this.processor = processor; 27 | } 28 | 29 | @Override 30 | public void shutDown() { 31 | processor.shutdownTimer(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/impl/SsnzEventSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.impl; 7 | 8 | import com.ebay.jetstream.event.JetstreamEvent; 9 | import com.ebay.pulsar.sessionizer.model.Session; 10 | import com.ebay.pulsar.sessionizer.model.SubSession; 11 | 12 | /** 13 | * An internal interface between SessionizerProcessor and Sessionizer. 14 | * 15 | * @author xingwang 16 | * 17 | */ 18 | public interface SsnzEventSender { 19 | void sendSessionBeginEvent(int sessionType, Session session, JetstreamEvent event); 20 | void sendSessionEndEvent(int sessionType, Session session, JetstreamEvent event); 21 | void sendSubSessionBeginEvent(int sessionType, SubSession subSession, JetstreamEvent event); 22 | void sendSubSessionEndEvent(int sessionType, SubSession subSession, JetstreamEvent event); 23 | } 24 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/model/SubSession.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.model; 7 | 8 | /** 9 | * AppSession POJO. 10 | * 11 | * It is a sub session of the Session. 12 | * 13 | * @author xingwang 14 | */ 15 | public class SubSession extends AbstractSession { 16 | 17 | private String name; 18 | 19 | /** 20 | * Sub session name. 21 | * 22 | * @return 23 | */ 24 | public String getName() { 25 | return name; 26 | } 27 | 28 | public void setName(String name) { 29 | this.name = name; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/spi/RemoteStoreCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.spi; 7 | 8 | import com.ebay.jetstream.notification.AlertListener.AlertStrength; 9 | import com.ebay.pulsar.sessionizer.impl.SessionizerErrorManager.ErrorType; 10 | import com.ebay.pulsar.sessionizer.model.Session; 11 | 12 | /** 13 | * Callback for remote store provider. 14 | * 15 | * @author xingwang 16 | */ 17 | public interface RemoteStoreCallback { 18 | 19 | /** 20 | * Register error into sessionizer error manager. 21 | * 22 | * @param ex 23 | * @param type 24 | */ 25 | void registerError(Throwable ex, ErrorType type); 26 | 27 | /** 28 | * Register error into sessionizer error manager. 29 | * 30 | * @param category 31 | * @param type 32 | * @param message 33 | */ 34 | void registerError(String category, ErrorType type, String message); 35 | 36 | /** 37 | * Send alert for remote store. Utilized the sessionizer alert listener. 38 | * 39 | * @param message 40 | * @param severity 41 | */ 42 | void sendRemoteStoreAlert(String message, AlertStrength severity); 43 | 44 | /** 45 | * Notify a remote session loaded. 46 | * 47 | * @param uniqueSessionId 48 | * @param sojSession 49 | * @param affinityKey 50 | */ 51 | void remoteSessionLoaded(String uniqueSessionId, Session sojSession, String affinityKey); 52 | 53 | /** 54 | * Send a remote session expired check event across the cluster. 55 | * 56 | * The node which received the event should load from remote store and check whether it expired or not. 57 | * 58 | * @param uniqueSessionId 59 | * @param affinityKey 60 | */ 61 | void sendExpirationCheckEvent(String uniqueSessionId, String affinityKey); 62 | 63 | /** 64 | * Notify the session is expired and the session should be handled by current node. 65 | * 66 | * @param uid 67 | * @param payload 68 | * @param metadata 69 | * @param ak 70 | */ 71 | void remoteSessionExpired(String uid, String affinityKey, byte[] payload, byte[] metadata); 72 | } 73 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/spi/RemoteStoreProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.spi; 7 | 8 | import com.ebay.pulsar.sessionizer.model.Session; 9 | 10 | 11 | /** 12 | * Provide remote session store. 13 | * 14 | * The integration point between sessionizer and external remote store. 15 | * The provider provide a layer to decouple external store with sessionizer. 16 | * 17 | * @author xingwang 18 | */ 19 | public interface RemoteStoreProvider { 20 | /** 21 | * Init the remote store. 22 | * 23 | * Invoked by sessionizer processor when sessionizer processor initialized. 24 | * 25 | * @param callback 26 | */ 27 | void init(RemoteStoreCallback callback); 28 | 29 | /** 30 | * Start the provider. 31 | * 32 | * Invoked by sessionizer processor after the init method. 33 | */ 34 | void start(); 35 | 36 | /** 37 | * Stop the provider. 38 | * 39 | * Invoked by sessionizer processor when shutdown. 40 | */ 41 | void stop(); 42 | 43 | 44 | /** 45 | * Send heart beat check of the current client. 46 | * 47 | * @param timestamp 48 | */ 49 | void updateHeartbeat(long timestamp); 50 | 51 | 52 | /** 53 | * Check leaked expired sessions on remote store. 54 | */ 55 | void checkExpiredSession(); 56 | 57 | /** 58 | * Async load the session by the key. 59 | * 60 | * affinityKey is used for send response back to sessionizer processor. 61 | * 62 | * @param key 63 | * @param affinityKey 64 | * @return 65 | */ 66 | boolean asyncLoad(String key, String affinityKey); 67 | 68 | /** 69 | * Whether support async load. 70 | * 71 | * @return 72 | */ 73 | boolean asyncLoadSupport(); 74 | 75 | /** 76 | * Insert a new session. 77 | * 78 | * @param session 79 | * @param key 80 | */ 81 | void insert(Session session, String key); 82 | 83 | /** 84 | * Load a session by key. 85 | * 86 | * @param key 87 | * @return 88 | */ 89 | Session load(String key); 90 | 91 | /** 92 | * Delete the session. 93 | * 94 | * @param session 95 | * @param key 96 | */ 97 | void delete(Session session, String key); 98 | 99 | /** 100 | * Update the session. 101 | * 102 | * @param session 103 | * @param key 104 | */ 105 | void update(Session session, String key); 106 | 107 | } 108 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/spi/SessionizerExtension.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.spi; 7 | 8 | import com.espertech.esper.client.UpdateListener; 9 | import com.espertech.esper.client.soda.AnnotationPart; 10 | import com.espertech.esper.client.soda.EPStatementObjectModel; 11 | 12 | /** 13 | * Extension for sessionizer Esper engine. 14 | * 15 | * @author xingwang 16 | */ 17 | public interface SessionizerExtension { 18 | /** 19 | * The annotation class. 20 | * 21 | * @return 22 | */ 23 | Class getAnnotation(); 24 | 25 | /** 26 | * The listener associated with the annotation. 27 | * 28 | * @param context 29 | * @param annoation 30 | * @param model 31 | * @return 32 | */ 33 | UpdateListener createUpdateListener(SessionizerContext context, AnnotationPart annoation, EPStatementObjectModel model); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/util/ByteBufferUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.util; 7 | 8 | import java.nio.ByteBuffer; 9 | 10 | /** 11 | * Use thread local bytebuffer to reduce memory footprint. 12 | * 13 | * @author xingwang 14 | * 15 | */ 16 | public final class ByteBufferUtil { 17 | private ByteBufferUtil() { 18 | //Utility class 19 | } 20 | private static final ThreadLocal TMP_BUFFER = new ThreadLocal() { 21 | @Override 22 | protected ByteBuffer initialValue() { 23 | return ByteBuffer.allocate(4096); 24 | } 25 | }; 26 | 27 | /** 28 | * Return the thread local byte buffer. 29 | * 30 | * @return 31 | */ 32 | public static ByteBuffer getThreadLocalByteBuffer() { 33 | return TMP_BUFFER.get(); 34 | } 35 | 36 | /** 37 | * Double the thread local buffer capacity. 38 | * 39 | * @return 40 | */ 41 | public static ByteBuffer enlargeThreadLocalByteBuffer() { 42 | TMP_BUFFER.set(ByteBuffer 43 | .allocate(TMP_BUFFER.get().capacity() * 2)); 44 | return TMP_BUFFER.get(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/util/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.util; 7 | 8 | /** 9 | * Constants used by sessionizer. 10 | * 11 | * @author xingwang 12 | */ 13 | public interface Constants { 14 | //Tags used by internal events 15 | /** 16 | * Session instance key. 17 | */ 18 | String EVENT_PAYLOAD_SESSION_OBJ = "SessionObj"; 19 | /** 20 | * Session type key. 21 | */ 22 | String EVENT_PAYLOAD_SESSION_TYPE = "SessionType"; 23 | /** 24 | * Session unique id key. 25 | */ 26 | String EVENT_PAYLOAD_SESSION_UNIQUEID = "SessionUniqueId"; 27 | /** 28 | * Session payload key. 29 | */ 30 | String EVENT_PAYLOAD_SESSION_PAYLOAD = "SessionPayloadInBytes"; 31 | /** 32 | * Session metadata key. 33 | */ 34 | String EVENT_PAYLOAD_SESSION_METADATA = "SessionMetadataInBytes"; 35 | 36 | // Internal event type 37 | /** 38 | * Transferred session between sessionizer nodes. 39 | */ 40 | String EVENT_TYPE_SESSION_TRANSFERED_EVENT = "_IntraSessionExpiredEvent"; 41 | /** 42 | * Remote session expired event. 43 | */ 44 | String EVENT_TYPE_SESSION_EXPIRED_EVENT = "_RemoteSessionExpired"; 45 | /** 46 | * Session loaded from remote store event. 47 | */ 48 | String EVENT_TYPE_SESSION_LOAD_EVENT = "_SessionLoaded"; 49 | } 50 | -------------------------------------------------------------------------------- /sessionizer/src/main/java/com/ebay/pulsar/sessionizer/util/SessionAttributeMapEncoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | Pulsar 3 | Copyright (C) 2013-2015 eBay Software Foundation 4 | Licensed under the GPL v2 license. See LICENSE for full terms. 5 | */ 6 | package com.ebay.pulsar.sessionizer.util; 7 | 8 | import java.nio.ByteBuffer; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | import com.ebay.jetstream.util.offheap.serializer.util.AsciiFirstStringEncoder; 13 | import com.ebay.jetstream.util.offheap.serializer.util.PrimitiveTLVEncoder; 14 | 15 | /** 16 | * Encoder for session attributes. 17 | * 18 | * It is thread-safe. 19 | * 20 | * @author xingwang 21 | * 22 | */ 23 | public class SessionAttributeMapEncoder { 24 | private final PrimitiveTLVEncoder encoder = new PrimitiveTLVEncoder(); 25 | private final AsciiFirstStringEncoder stringEncoder = new AsciiFirstStringEncoder(); 26 | public Map decode(ByteBuffer buffer) { 27 | int length = buffer.getInt(); 28 | if (length == 0) { 29 | return null; 30 | } 31 | Map attributes = new HashMap(length, 2.0f); //avoid rehash 32 | for (int i = 0, t = length; i < t; i++) { 33 | String id = stringEncoder.decode(buffer); 34 | attributes.put(id, encoder.decode(buffer)); 35 | } 36 | return attributes; 37 | } 38 | 39 | public void encode(Map dynamicAttributes, ByteBuffer buffer) { 40 | if (dynamicAttributes == null || dynamicAttributes.isEmpty()) { 41 | buffer.putInt(0); // EMPTY 42 | } else { 43 | int position = buffer.position(); 44 | buffer.putInt(0); //place holder. 45 | int count = 0; 46 | for (Map.Entry e : dynamicAttributes.entrySet()) { 47 | Object value = e.getValue(); 48 | if (value != null) { 49 | stringEncoder.encode(e.getKey(), buffer); 50 | encoder.encode(value, buffer); 51 | count ++; 52 | } 53 | } 54 | buffer.putInt(position, count); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /sessionizer/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | logs/jetstream-%d{yyyy-MM-dd}.%i.log 10 | 11 | 10 MB 12 | 13 | 10 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 23 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 24 | 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | --------------------------------------------------------------------------------