├── .gitignore ├── BriskBenchmarks ├── BriskBenchmarks.iml ├── pom.xml └── src │ ├── main │ ├── java │ │ ├── applications │ │ │ ├── AppDriver.java │ │ │ ├── BriskRunner.java │ │ │ ├── bolts │ │ │ │ └── udf │ │ │ │ │ ├── README.md │ │ │ │ │ ├── comm │ │ │ │ │ ├── AbstractFilterBolt.java │ │ │ │ │ ├── AbstractScoreBolt.java │ │ │ │ │ ├── ForwardBolt.java │ │ │ │ │ ├── FraudDetectionParserBolt.java │ │ │ │ │ ├── GeneralParserBolt.java │ │ │ │ │ ├── GeneralParserBolt_FT.java │ │ │ │ │ ├── SensorParserBolt.java │ │ │ │ │ ├── StringParserBolt.java │ │ │ │ │ ├── StringParserBoltInorder.java │ │ │ │ │ └── StringParserBolt_latency.java │ │ │ │ │ ├── fd │ │ │ │ │ └── FraudPredictorBolt.java │ │ │ │ │ ├── lg │ │ │ │ │ ├── GeoStatsBolt.java │ │ │ │ │ ├── GeographyBolt.java │ │ │ │ │ ├── StatusCountBolt.java │ │ │ │ │ └── VolumeCountBolt.java │ │ │ │ │ ├── lr │ │ │ │ │ ├── AccidentDetectionBolt.java │ │ │ │ │ ├── AccidentNotificationBolt.java │ │ │ │ │ ├── AccountBalanceBolt.java │ │ │ │ │ ├── AverageVehicleSpeedBolt.java │ │ │ │ │ ├── AverageVehicleSpeedBolt_latency.java │ │ │ │ │ ├── CountVehiclesBolt.java │ │ │ │ │ ├── CountVehiclesBolt_latency.java │ │ │ │ │ ├── DailyExpenditureBolt.java │ │ │ │ │ ├── DispatcherBolt.java │ │ │ │ │ ├── DispatcherBolt_latency.java │ │ │ │ │ ├── LatestAverageVelocityBolt.java │ │ │ │ │ ├── LatestAverageVelocityBolt_latency.java │ │ │ │ │ ├── StreamMerger.java │ │ │ │ │ ├── TimestampMerger.java │ │ │ │ │ ├── TollNotificationBolt.java │ │ │ │ │ ├── TollNotificationBolt_cv.java │ │ │ │ │ ├── TollNotificationBolt_cv_latency.java │ │ │ │ │ ├── TollNotificationBolt_las.java │ │ │ │ │ ├── TollNotificationBolt_las_latency.java │ │ │ │ │ ├── TollNotificationBolt_pos.java │ │ │ │ │ ├── TollNotificationBolt_pos_latency.java │ │ │ │ │ └── model │ │ │ │ │ │ ├── Accident.java │ │ │ │ │ │ ├── AccidentImmutable.java │ │ │ │ │ │ ├── AccountBalance.java │ │ │ │ │ │ ├── MinuteStatistics.java │ │ │ │ │ │ ├── NovLav.java │ │ │ │ │ │ ├── TollEntry.java │ │ │ │ │ │ ├── VehicleAccount.java │ │ │ │ │ │ └── VehicleInfo.java │ │ │ │ │ ├── sa │ │ │ │ │ ├── FilterBySum.java │ │ │ │ │ ├── WindowMedian.java │ │ │ │ │ └── WindowRank.java │ │ │ │ │ ├── sd │ │ │ │ │ ├── MovingAverageBolt.java │ │ │ │ │ └── SpikeDetectionBolt.java │ │ │ │ │ ├── tm │ │ │ │ │ ├── MapMatchingBolt.java │ │ │ │ │ └── SpeedCalculatorBolt.java │ │ │ │ │ ├── vs │ │ │ │ │ ├── ACDBolt.java │ │ │ │ │ ├── CTBolt.java │ │ │ │ │ ├── ECRBolt.java │ │ │ │ │ ├── ENCRBolt.java │ │ │ │ │ ├── FoFiRBolt.java │ │ │ │ │ ├── GlobalACDBolt.java │ │ │ │ │ ├── RCRBolt.java │ │ │ │ │ ├── ScoreBolt.java │ │ │ │ │ ├── URLBolt.java │ │ │ │ │ └── VoiceDispatcherBolt.java │ │ │ │ │ └── wc │ │ │ │ │ ├── SplitSentenceBolt.java │ │ │ │ │ ├── SplitSentenceBolt_FT.java │ │ │ │ │ ├── SplitSentenceBolt_latency.java │ │ │ │ │ ├── WordCountBolt.java │ │ │ │ │ ├── WordCountBolt_FT.java │ │ │ │ │ └── WordCountBolt_latency.java │ │ │ ├── datatype │ │ │ │ ├── AbstractInputTuple.java │ │ │ │ ├── AbstractLRBTuple.java │ │ │ │ ├── AbstractOutputTuple.java │ │ │ │ ├── AccidentNotification.java │ │ │ │ ├── AccountBalanceRequest.java │ │ │ │ ├── DailyExpenditureRequest.java │ │ │ │ ├── Pair.java │ │ │ │ ├── PositionReport.java │ │ │ │ ├── TextInt.java │ │ │ │ ├── TollNotification.java │ │ │ │ ├── TravelTimeRequest.java │ │ │ │ ├── internal │ │ │ │ │ ├── AccidentTuple.java │ │ │ │ │ ├── AvgSpeedTuple.java │ │ │ │ │ ├── AvgVehicleSpeedTuple.java │ │ │ │ │ ├── CountTuple.java │ │ │ │ │ └── LavTuple.java │ │ │ │ ├── toll │ │ │ │ │ ├── FileTollDataStore.java │ │ │ │ │ ├── MemoryTollDataStore.java │ │ │ │ │ └── TollDataStore.java │ │ │ │ └── util │ │ │ │ │ ├── AvgValue.java │ │ │ │ │ ├── CarCount.java │ │ │ │ │ ├── Constants.java │ │ │ │ │ ├── IPositionIdentifier.java │ │ │ │ │ ├── ISegmentIdentifier.java │ │ │ │ │ ├── LRTopologyControl.java │ │ │ │ │ ├── PositionIdentifier.java │ │ │ │ │ ├── SegmentIdentifier.java │ │ │ │ │ ├── TimeStampExtractor.java │ │ │ │ │ └── VSTopologyControl.java │ │ │ ├── sink │ │ │ │ ├── BaseSink.java │ │ │ │ ├── ConsoleSink.java │ │ │ │ ├── EmptySink.java │ │ │ │ ├── ForwardSink.java │ │ │ │ ├── MeasureSink.java │ │ │ │ ├── MeasureSink_FT.java │ │ │ │ ├── MeasureSink_Txn.java │ │ │ │ ├── MeasureSink_Txn_latency.java │ │ │ │ ├── MeasureSink_latency.java │ │ │ │ ├── NullSink.java │ │ │ │ ├── NullSink_LR.java │ │ │ │ ├── PKSink.java │ │ │ │ ├── ReadSink.java │ │ │ │ ├── formatter │ │ │ │ │ ├── BasicFormatter.java │ │ │ │ │ ├── Formatter.java │ │ │ │ │ └── FullInfoFormatter.java │ │ │ │ └── verbose │ │ │ │ │ └── ConsoleSink.java │ │ │ ├── spout │ │ │ │ ├── DataSource.java │ │ │ │ ├── FileSpout.java │ │ │ │ ├── LocalStateSpout.java │ │ │ │ ├── MemFileSpout.java │ │ │ │ ├── MemFileSpout_latency.java │ │ │ │ ├── PhoneCallGeneratorSpout.java │ │ │ │ ├── WCGeneratorSpout.java │ │ │ │ └── generator │ │ │ │ │ ├── Generator.java │ │ │ │ │ ├── PhoneCallGenerator.java │ │ │ │ │ └── WCGenerator.java │ │ │ └── topology │ │ │ │ ├── CompositeTopology.java │ │ │ │ ├── FraudDetection.java │ │ │ │ ├── LinearRoad.java │ │ │ │ ├── LogProcessing.java │ │ │ │ ├── SpikeDetection.java │ │ │ │ ├── StreamingAnalysis.java │ │ │ │ ├── TrafficMonitoring.java │ │ │ │ ├── VoIPSTREAM.java │ │ │ │ ├── WordCount.java │ │ │ │ └── latency │ │ │ │ ├── LinearRoad_latency.java │ │ │ │ └── WordCount_latency.java │ │ └── brisk │ │ │ ├── components │ │ │ ├── MultiStreamComponent.java │ │ │ ├── Topology.java │ │ │ ├── TopologyComponent.java │ │ │ ├── context │ │ │ │ └── TopologyContext.java │ │ │ ├── exception │ │ │ │ ├── InvalidIDException.java │ │ │ │ └── UnhandledCaseException.java │ │ │ ├── grouping │ │ │ │ ├── AllGrouping.java │ │ │ │ ├── FieldsGrouping.java │ │ │ │ ├── GlobalGrouping.java │ │ │ │ ├── Grouping.java │ │ │ │ ├── MarkerShuffleGrouping.java │ │ │ │ ├── PartialKeyGrouping.java │ │ │ │ └── ShuffleGrouping.java │ │ │ ├── operators │ │ │ │ ├── api │ │ │ │ │ ├── AbstractBolt.java │ │ │ │ │ ├── AbstractSpout.java │ │ │ │ │ ├── AbstractWindowedBolt.java │ │ │ │ │ ├── BaseOperator.java │ │ │ │ │ ├── BaseWindowedBolt.java │ │ │ │ │ ├── Checkpointable.java │ │ │ │ │ ├── IOperator.java │ │ │ │ │ ├── Operator.java │ │ │ │ │ └── WjoinBolt.java │ │ │ │ ├── base │ │ │ │ │ ├── MapBolt.java │ │ │ │ │ ├── filterBolt.java │ │ │ │ │ ├── splitBolt.java │ │ │ │ │ ├── unionBolt.java │ │ │ │ │ ├── w_applyBolt.java │ │ │ │ │ └── w_joinBolt.java │ │ │ │ └── executor │ │ │ │ │ ├── BasicBoltBatchExecutor.java │ │ │ │ │ ├── BasicSpoutBatchExecutor.java │ │ │ │ │ ├── BasicWindowBoltBatchExecutor.java │ │ │ │ │ ├── BoltExecutor.java │ │ │ │ │ ├── IExecutor.java │ │ │ │ │ ├── SpoutExecutor.java │ │ │ │ │ └── VirtualExecutor.java │ │ │ ├── streaminfo.java │ │ │ └── windowing │ │ │ │ ├── BasicEvent.java │ │ │ │ ├── CountEvictionPolicy.java │ │ │ │ ├── CountTriggerPolicy.java │ │ │ │ ├── DefaultEvictionContext.java │ │ │ │ ├── Event.java │ │ │ │ ├── EvictionContext.java │ │ │ │ ├── EvictionPolicy.java │ │ │ │ ├── TimeEvictionPolicy.java │ │ │ │ ├── TimeTriggerPolicy.java │ │ │ │ ├── TimestampExtractor.java │ │ │ │ ├── TriggerHandler.java │ │ │ │ ├── TriggerPolicy.java │ │ │ │ ├── TupleFieldTimestampExtractor.java │ │ │ │ ├── TupleWindow.java │ │ │ │ ├── TupleWindowImpl.java │ │ │ │ ├── WaterMarkEvent.java │ │ │ │ ├── Window.java │ │ │ │ ├── WindowLifecycleListener.java │ │ │ │ └── WindowManager.java │ │ │ ├── controller │ │ │ ├── affinity │ │ │ │ ├── AffinityController.java │ │ │ │ └── SequentialBinding.java │ │ │ ├── input │ │ │ │ ├── IISC.java │ │ │ │ ├── InputStreamController.java │ │ │ │ └── scheduler │ │ │ │ │ ├── SequentialScheduler.java │ │ │ │ │ └── UniformedScheduler.java │ │ │ └── output │ │ │ │ ├── IPartitionController.java │ │ │ │ ├── MultiStreamOutputContoller.java │ │ │ │ ├── OutputController.java │ │ │ │ ├── PartitionController.java │ │ │ │ └── partition │ │ │ │ ├── AllPartitionController.java │ │ │ │ ├── FieldsPartitionController.java │ │ │ │ ├── GlobalPartitionController.java │ │ │ │ ├── MarkerShufflePartitionController.java │ │ │ │ ├── PartialKeyGroupingController.java │ │ │ │ ├── ShufflePartitionController.java │ │ │ │ └── impl │ │ │ │ └── TupleUtils.java │ │ │ ├── execution │ │ │ ├── Clock.java │ │ │ ├── ExecutionGraph.java │ │ │ ├── ExecutionManager.java │ │ │ ├── ExecutionNode.java │ │ │ ├── RawExecutionGraph.java │ │ │ └── runtime │ │ │ │ ├── boltThread.java │ │ │ │ ├── collector │ │ │ │ ├── OutputCollector.java │ │ │ │ └── impl │ │ │ │ │ ├── BIDGenerator.java │ │ │ │ │ ├── BIDGenerator2.java │ │ │ │ │ ├── Meta.java │ │ │ │ │ └── MetaGroup.java │ │ │ │ ├── controllerThread.java │ │ │ │ ├── executorThread.java │ │ │ │ ├── spoutThread.java │ │ │ │ └── tuple │ │ │ │ ├── TransferTuple.java │ │ │ │ └── impl │ │ │ │ ├── Fields.java │ │ │ │ ├── Marker.java │ │ │ │ ├── Message.java │ │ │ │ ├── OutputFieldsDeclarer.java │ │ │ │ ├── StableValues.java │ │ │ │ ├── Tuple.java │ │ │ │ └── msgs │ │ │ │ ├── GeneralMsg.java │ │ │ │ ├── IntDoubleDoubleMsg.java │ │ │ │ ├── StringLongMsg.java │ │ │ │ ├── StringMsg.java │ │ │ │ └── Tuple2Msg.java │ │ │ ├── faulttolerance │ │ │ ├── State.java │ │ │ ├── Writer.java │ │ │ ├── compress.java │ │ │ └── impl │ │ │ │ └── ValueState.java │ │ │ ├── optimization │ │ │ ├── ExecutionPlan.java │ │ │ ├── OptimizationManager.java │ │ │ ├── Optimizer.java │ │ │ ├── impl │ │ │ │ ├── Decision.java │ │ │ │ ├── LocalPlanScheduler.java │ │ │ │ ├── PlanScheduler.java │ │ │ │ ├── SchedulingPlan.java │ │ │ │ ├── scaling │ │ │ │ │ ├── Parallelism.java │ │ │ │ │ └── scalingOptimization.java │ │ │ │ └── scheduling │ │ │ │ │ ├── BranchAndBound.java │ │ │ │ │ ├── TOFF.java │ │ │ │ │ ├── TOFF_hardConstraint.java │ │ │ │ │ ├── randomPlan_Constraints.java │ │ │ │ │ ├── randomPlan_NoConstraints.java │ │ │ │ │ ├── randomSearch_Constraints.java │ │ │ │ │ ├── randomSearch_NoConstrains.java │ │ │ │ │ ├── randomSearch_hardConstraints.java │ │ │ │ │ └── roundrobin.java │ │ │ ├── model │ │ │ │ ├── BackPressure.java │ │ │ │ ├── Constraints.java │ │ │ │ ├── GraphMetrics.java │ │ │ │ ├── RateModel.java │ │ │ │ ├── STAT.java │ │ │ │ ├── Variables.java │ │ │ │ └── cache.java │ │ │ └── routing │ │ │ │ ├── RoutingOptimizer.java │ │ │ │ └── RoutingPlan.java │ │ │ ├── queue │ │ │ ├── MPMCController.java │ │ │ ├── MPSCController.java │ │ │ ├── QueueController.java │ │ │ ├── SPMCController.java │ │ │ ├── SPSCController.java │ │ │ └── impl │ │ │ │ ├── P1C1OffHeapQueue.java │ │ │ │ ├── P1C1Queue.java │ │ │ │ ├── PaddedAtomicLong.java │ │ │ │ ├── PaddedLong.java │ │ │ │ ├── UnsafeAccess.java │ │ │ │ ├── UnsafeDirectByteBuffer.java │ │ │ │ └── wait │ │ │ │ ├── SleepWaitStrategy.java │ │ │ │ ├── WaitStrategy.java │ │ │ │ └── YieldingWaitStrategy.java │ │ │ ├── topology │ │ │ ├── AbstractTopology.java │ │ │ ├── BasicTopology.java │ │ │ ├── TopologyBuilder.java │ │ │ ├── TopologyComiler.java │ │ │ └── TopologySubmitter.java │ │ │ └── util │ │ │ ├── ExpiringMap.java │ │ │ ├── FileClean.java │ │ │ ├── FixedSizeQueue.java │ │ │ ├── ObjectSizeFetcher.java │ │ │ ├── PerfectHashMap.java │ │ │ ├── SlidingWindow.java │ │ │ ├── myIntegerMap.java │ │ │ └── queue │ │ │ ├── Nets1.java │ │ │ └── P1C1QueueStep0.java │ └── resources │ │ └── HUAWEI.cpuinfo │ └── test │ └── java │ └── streaming │ ├── Tests.java │ ├── affinity │ └── AffinityTests.java │ └── impl │ ├── demoTopology_testFT.java │ ├── demoTopology_testNormal.java │ ├── ft │ ├── bolt_ft.java │ ├── sink_ft.java │ └── spout_ft.java │ └── normal │ ├── bolt.java │ ├── sink.java │ └── spout.java ├── FlinkBenchmarks ├── pom.xml └── src │ ├── main │ └── java │ │ └── applications │ │ ├── AppDriver.java │ │ ├── FlinkRunner.java │ │ ├── bolts │ │ ├── AbstractBolt.java │ │ ├── Executor.java │ │ └── udf │ │ │ ├── comm │ │ │ ├── AbstractFilterBolt.java │ │ │ ├── AbstractScoreBolt.java │ │ │ ├── ForwardBolt.java │ │ │ ├── ParserBolt.java │ │ │ └── ParserBolt_latency.java │ │ │ ├── fd │ │ │ ├── FraudPredictorBolt.java │ │ │ └── FraudPredictor_latencyBolt.java │ │ │ ├── lg │ │ │ ├── GeoStatsBolt.java │ │ │ ├── GeoStatsBolt_latency.java │ │ │ ├── GeographyBolt.java │ │ │ ├── GeographyBolt_latency.java │ │ │ ├── StatusCountBolt.java │ │ │ ├── StatusCountBolt_latency.java │ │ │ ├── VolumeCountBolt.java │ │ │ └── VolumeCountBolt_latency.java │ │ │ ├── lr │ │ │ ├── AccidentDetectionBolt.java │ │ │ ├── AccidentDetectionBolt_latency.java │ │ │ ├── AccidentNotificationBolt.java │ │ │ ├── AccidentNotificationBolt_latency.java │ │ │ ├── AccountBalanceBolt.java │ │ │ ├── AccountBalanceBolt_latency.java │ │ │ ├── AverageVehicleSpeedBolt.java │ │ │ ├── CountVehiclesBolt.java │ │ │ ├── CountVehiclesBolt_latency.java │ │ │ ├── DailyExpenditureBolt.java │ │ │ ├── DailyExpenditureBolt_latency.java │ │ │ ├── DispatcherBolt.java │ │ │ ├── DispatcherBolt_latency.java │ │ │ ├── LatestAverageVelocityBolt.java │ │ │ ├── TollNotificationBolt.java │ │ │ ├── TollNotificationBolt_cv.java │ │ │ ├── TollNotificationBolt_las.java │ │ │ ├── TollNotificationBolt_latency.java │ │ │ └── TollNotificationBolt_pos.java │ │ │ ├── sd │ │ │ ├── MovingAverageBolt.java │ │ │ ├── MovingAverageBolt_latency.java │ │ │ ├── SpikeDetectionBolt.java │ │ │ └── SpikeDetectionBolt_latency.java │ │ │ ├── vs │ │ │ ├── ACDBolt.java │ │ │ ├── CTBolt.java │ │ │ ├── ECRBolt.java │ │ │ ├── ENCRBolt.java │ │ │ ├── FoFiRBolt.java │ │ │ ├── GlobalACDBolt.java │ │ │ ├── RCRBolt.java │ │ │ ├── ScoreBolt.java │ │ │ ├── URLBolt.java │ │ │ └── VoiceDispatcherBolt.java │ │ │ └── wc │ │ │ ├── SplitSentenceBolt.java │ │ │ └── WordCountBolt.java │ │ ├── datatypes │ │ ├── AbstractInputTuple.java │ │ ├── AbstractLRBTuple.java │ │ ├── AbstractOutputTuple.java │ │ ├── AccidentNotification.java │ │ ├── AccountBalanceRequest.java │ │ ├── DailyExpenditureRequest.java │ │ ├── Pair.java │ │ ├── PositionReport.java │ │ ├── TextInt.java │ │ ├── TollNotification.java │ │ ├── TravelTimeRequest.java │ │ ├── Tuple.java │ │ ├── internal │ │ │ ├── AccidentTuple.java │ │ │ ├── AvgSpeedTuple.java │ │ │ ├── AvgVehicleSpeedTuple.java │ │ │ ├── CountTuple.java │ │ │ └── LavTuple.java │ │ ├── toll │ │ │ ├── FileTollDataStore.java │ │ │ ├── MemoryTollDataStore.java │ │ │ └── TollDataStore.java │ │ └── util │ │ │ ├── AvgValue.java │ │ │ ├── CarCount.java │ │ │ ├── Constants.java │ │ │ ├── IPositionIdentifier.java │ │ │ ├── ISegmentIdentifier.java │ │ │ ├── PositionIdentifier.java │ │ │ ├── SegmentIdentifier.java │ │ │ ├── TimeStampExtractor.java │ │ │ └── TopologyControl.java │ │ ├── models │ │ ├── Accident.java │ │ ├── AccidentImmutable.java │ │ ├── AccountBalance.java │ │ ├── MinuteStatistics.java │ │ ├── NovLav.java │ │ ├── TollEntry.java │ │ ├── VehicleAccount.java │ │ └── VehicleInfo.java │ │ ├── sink │ │ ├── BaseSink.java │ │ ├── ForwardSink.java │ │ ├── ForwardSink_latency.java │ │ ├── MeasureSink.java │ │ ├── MeasureSink_latency.java │ │ ├── MeasureSink_latency_LR.java │ │ ├── formatter │ │ │ ├── BasicFormatter.java │ │ │ ├── Formatter.java │ │ │ └── FullInfoFormatter.java │ │ └── verbose │ │ │ └── ConsoleSink.java │ │ ├── spout │ │ ├── AbstractSpout.java │ │ ├── MemFileSpout.java │ │ └── MemFileSpout_latency.java │ │ ├── topology │ │ ├── AbstractTopology.java │ │ ├── BasicTopology.java │ │ └── benchmarks │ │ │ ├── FraudDetection.java │ │ │ ├── FraudDetection_latency.java │ │ │ ├── LinearRoad.java │ │ │ ├── LinearRoad_latency.java │ │ │ ├── LogProcessing.java │ │ │ ├── LogProcessing_latency.java │ │ │ ├── SpikeDetection.java │ │ │ ├── SpikeDetection_latency.java │ │ │ ├── VoIPSTREAM.java │ │ │ ├── WordCount.java │ │ │ └── WordCount_latency.java │ │ └── util │ │ └── Time.java │ └── test │ └── java │ └── TestFlinkRunner.java ├── HeronBenchmarks ├── pom.xml └── src │ ├── main │ └── java │ │ └── applications │ │ ├── AppDriver.java │ │ ├── HeronRunner.java │ │ ├── bolts │ │ ├── AbstractBolt.java │ │ ├── Executor.java │ │ ├── comm │ │ │ ├── AbstractFilterBolt.java │ │ │ ├── AbstractScoreBolt.java │ │ │ ├── ForwardBolt.java │ │ │ └── ParserBolt.java │ │ ├── fd │ │ │ └── FraudPredictorBolt.java │ │ ├── lg │ │ │ ├── GeoStatsBolt.java │ │ │ ├── GeographyBolt.java │ │ │ ├── StatusCountBolt.java │ │ │ └── VolumeCountBolt.java │ │ ├── lr │ │ │ ├── AccidentDetectionBolt.java │ │ │ ├── AccidentNotificationBolt.java │ │ │ ├── AccountBalanceBolt.java │ │ │ ├── AverageVehicleSpeedBolt.java │ │ │ ├── CountVehiclesBolt.java │ │ │ ├── DailyExpenditureBolt.java │ │ │ ├── DispatcherBolt.java │ │ │ ├── LatestAverageVelocityBolt.java │ │ │ └── TollNotificationBolt.java │ │ ├── sd │ │ │ ├── MovingAverageBolt.java │ │ │ └── SpikeDetectionBolt.java │ │ ├── vs │ │ │ ├── ACDBolt.java │ │ │ ├── CTBolt.java │ │ │ ├── ECRBolt.java │ │ │ ├── ENCRBolt.java │ │ │ ├── FoFiRBolt.java │ │ │ ├── GlobalACDBolt.java │ │ │ ├── RCRBolt.java │ │ │ ├── ScoreBolt.java │ │ │ ├── URLBolt.java │ │ │ └── VoiceDispatcherBolt.java │ │ └── wc │ │ │ ├── SplitSentenceBolt.java │ │ │ └── WordCountBolt.java │ │ ├── models │ │ └── lr │ │ │ ├── Accident.java │ │ │ ├── AccidentImmutable.java │ │ │ ├── AccountBalance.java │ │ │ ├── MinuteStatistics.java │ │ │ ├── NovLav.java │ │ │ ├── TollEntry.java │ │ │ ├── VehicleAccount.java │ │ │ └── VehicleInfo.java │ │ ├── sink │ │ ├── BaseSink.java │ │ ├── ForwardSink.java │ │ ├── MeasureSink.java │ │ ├── formatter │ │ │ ├── BasicFormatter.java │ │ │ └── Formatter.java │ │ └── verbose │ │ │ └── ConsoleSink.java │ │ ├── spout │ │ ├── AbstractSpout.java │ │ └── MemFileSpout.java │ │ ├── topology │ │ ├── AbstractTopology.java │ │ ├── BasicTopology.java │ │ └── benchmarks │ │ │ ├── FraudDetection.java │ │ │ ├── LinearRoad.java │ │ │ ├── LogProcessing.java │ │ │ ├── SpikeDetection.java │ │ │ ├── VoIPSTREAM.java │ │ │ └── WordCount.java │ │ └── util │ │ └── Time.java │ └── test │ └── java │ └── TestHeronRunner.java ├── LICENSE ├── README.md ├── StormBenchmarks ├── pom.xml └── src │ ├── main │ └── java │ │ └── applications │ │ ├── AppDriver.java │ │ ├── StormRunner.java │ │ ├── bolts │ │ ├── AbstractBolt.java │ │ ├── Executor.java │ │ └── udf │ │ │ ├── comm │ │ │ ├── AbstractFilterBolt.java │ │ │ ├── AbstractScoreBolt.java │ │ │ ├── ForwardBolt.java │ │ │ ├── ParserBolt.java │ │ │ └── ParserBolt_latency.java │ │ │ ├── fd │ │ │ ├── FraudPredictorBolt.java │ │ │ └── FraudPredictorBolt_latency.java │ │ │ ├── lg │ │ │ ├── GeoStatsBolt.java │ │ │ ├── GeoStatsBolt_latency.java │ │ │ ├── GeographyBolt.java │ │ │ ├── GeographyBolt_latency.java │ │ │ ├── StatusCountBolt.java │ │ │ ├── StatusCountBolt_latency.java │ │ │ ├── VolumeCountBolt.java │ │ │ └── VolumeCountBolt_latency.java │ │ │ ├── lr │ │ │ ├── AccidentDetectionBolt.java │ │ │ ├── AccidentDetection_latencyBolt.java │ │ │ ├── AccidentNotificationBolt.java │ │ │ ├── AccidentNotification_latencyBolt.java │ │ │ ├── AccountBalanceBolt.java │ │ │ ├── AccountBalance_latencyBolt.java │ │ │ ├── AverageVehicleSpeedBolt.java │ │ │ ├── AverageVehicleSpeedBolt_latency.java │ │ │ ├── CountVehiclesBolt.java │ │ │ ├── CountVehicles_latencyBolt.java │ │ │ ├── DailyExpenditureBolt.java │ │ │ ├── DailyExpenditure_latencyBolt.java │ │ │ ├── DispatcherBolt.java │ │ │ ├── DispatcherBolt_latency.java │ │ │ ├── LatestAverageVelocityBolt.java │ │ │ ├── TollNotificationBolt.java │ │ │ ├── TollNotificationBolt_cv.java │ │ │ ├── TollNotificationBolt_cv_latency.java │ │ │ ├── TollNotificationBolt_las.java │ │ │ ├── TollNotificationBolt_las_latency.java │ │ │ ├── TollNotificationBolt_pos.java │ │ │ └── TollNotificationBolt_pos_latency.java │ │ │ ├── sd │ │ │ ├── MovingAverageBolt.java │ │ │ ├── MovingAverageBolt_latency.java │ │ │ ├── SpikeDetectionBolt.java │ │ │ └── SpikeDetectionBolt_latency.java │ │ │ ├── vs │ │ │ ├── ACDBolt.java │ │ │ ├── CTBolt.java │ │ │ ├── ECRBolt.java │ │ │ ├── ENCRBolt.java │ │ │ ├── FoFiRBolt.java │ │ │ ├── GlobalACDBolt.java │ │ │ ├── RCRBolt.java │ │ │ ├── ScoreBolt.java │ │ │ ├── URLBolt.java │ │ │ └── VoiceDispatcherBolt.java │ │ │ └── wc │ │ │ ├── SplitSentenceBolt.java │ │ │ ├── SplitSentenceBolt_latency.java │ │ │ ├── WordCountBolt.java │ │ │ └── WordCountBolt_latency.java │ │ ├── datatypes │ │ ├── AbstractInputTuple.java │ │ ├── AbstractLRBTuple.java │ │ ├── AbstractOutputTuple.java │ │ ├── AccidentNotification.java │ │ ├── AccountBalanceRequest.java │ │ ├── DailyExpenditureRequest.java │ │ ├── Pair.java │ │ ├── PositionReport.java │ │ ├── TextInt.java │ │ ├── TollNotification.java │ │ ├── TravelTimeRequest.java │ │ ├── Tuple.java │ │ ├── internal │ │ │ ├── AccidentTuple.java │ │ │ ├── AvgSpeedTuple.java │ │ │ ├── AvgVehicleSpeedTuple.java │ │ │ ├── CountTuple.java │ │ │ └── LavTuple.java │ │ ├── toll │ │ │ ├── FileTollDataStore.java │ │ │ ├── MemoryTollDataStore.java │ │ │ └── TollDataStore.java │ │ └── util │ │ │ ├── AvgValue.java │ │ │ ├── CarCount.java │ │ │ ├── Constants.java │ │ │ ├── IPositionIdentifier.java │ │ │ ├── ISegmentIdentifier.java │ │ │ ├── PositionIdentifier.java │ │ │ ├── SegmentIdentifier.java │ │ │ ├── TimeStampExtractor.java │ │ │ └── TopologyControl.java │ │ ├── models │ │ ├── Accident.java │ │ ├── AccidentImmutable.java │ │ ├── AccountBalance.java │ │ ├── MinuteStatistics.java │ │ ├── NovLav.java │ │ ├── TollEntry.java │ │ ├── VehicleAccount.java │ │ └── VehicleInfo.java │ │ ├── sink │ │ ├── BaseSink.java │ │ ├── ForwardSink.java │ │ ├── ForwardSink_latency.java │ │ ├── MeasureSink.java │ │ ├── MeasureSink_latency.java │ │ ├── MeasureSink_latency_LR.java │ │ ├── formatter │ │ │ ├── BasicFormatter.java │ │ │ ├── Formatter.java │ │ │ └── FullInfoFormatter.java │ │ └── verbose │ │ │ └── ConsoleSink.java │ │ ├── spout │ │ ├── AbstractSpout.java │ │ ├── MemFileSpout.java │ │ └── MemFileSpout_latency.java │ │ ├── topology │ │ ├── AbstractTopology.java │ │ ├── BasicTopology.java │ │ └── benchmarks │ │ │ ├── FraudDetection.java │ │ │ ├── FraudDetection_latency.java │ │ │ ├── LinearRoad.java │ │ │ ├── LinearRoad_latency.java │ │ │ ├── LogProcessing.java │ │ │ ├── LogProcessing_latency.java │ │ │ ├── SpikeDetection.java │ │ │ ├── SpikeDetection_latency.java │ │ │ ├── VoIPSTREAM.java │ │ │ ├── WordCount.java │ │ │ └── WordCount_latency.java │ │ └── util │ │ └── Time.java │ └── test │ └── java │ └── TestStormRunner.java ├── affinity ├── affinity.iml ├── pom.xml └── src │ ├── main │ ├── c │ │ ├── Makefile │ │ ├── net_openhft_ticker_impl_JNIClock.cpp │ │ ├── software_chronicle_enterprise_internals_impl_NativeAffinity.cpp │ │ └── software_chronicle_enterprise_internals_impl_NativeAffinity_MacOSX.c │ └── java │ │ ├── java │ │ └── lang │ │ │ ├── ThreadLifecycleListener.java │ │ │ └── ThreadTrackingGroup.java │ │ ├── net │ │ └── openhft │ │ │ ├── affinity │ │ │ ├── Affinity.java │ │ │ ├── AffinityLock.java │ │ │ ├── AffinityStrategies.java │ │ │ ├── AffinityStrategy.java │ │ │ ├── AffinitySupport.java │ │ │ ├── AffinityThreadFactory.java │ │ │ ├── BootClassPath.java │ │ │ ├── CpuLayout.java │ │ │ ├── IAffinity.java │ │ │ ├── LockCheck.java │ │ │ ├── LockInventory.java │ │ │ ├── MicroJitterSampler.java │ │ │ ├── NonForkingAffinityLock.java │ │ │ └── impl │ │ │ │ ├── LinuxHelper.java │ │ │ │ ├── LinuxJNAAffinity.java │ │ │ │ ├── NoCpuLayout.java │ │ │ │ ├── NullAffinity.java │ │ │ │ ├── OSXJNAAffinity.java │ │ │ │ ├── PosixJNAAffinity.java │ │ │ │ ├── SolarisJNAAffinity.java │ │ │ │ ├── Utilities.java │ │ │ │ ├── VanillaCpuLayout.java │ │ │ │ ├── VersionHelper.java │ │ │ │ └── WindowsJNAAffinity.java │ │ │ └── ticker │ │ │ ├── ITicker.java │ │ │ ├── Ticker.java │ │ │ └── impl │ │ │ ├── JNIClock.java │ │ │ └── SystemClock.java │ │ └── software │ │ └── chronicle │ │ └── enterprise │ │ └── internals │ │ └── impl │ │ └── NativeAffinity.java │ └── test │ └── resources │ ├── HUAWEI.cpuinfo │ ├── amd64.dual.core.cpuinfo │ ├── amd64.quad.core.cpuinfo │ ├── core.duo.cpuinfo │ ├── dual.E5405.cpuinfo │ ├── dual.xeon.cpuinfo │ ├── i3.cpuinfo │ ├── i7.cpuinfo │ ├── i7.properties │ ├── q6600.noht.cpuinfo │ └── q6600.vm.cpuinfo ├── analysis.uxf ├── bitbucket-pipelines.yml ├── briskstream.iml ├── common ├── Brisk_scripts │ ├── benchmark.sh │ ├── hpi │ ├── jobdone.py │ ├── message.txt │ ├── mvn_install.sh │ ├── mycontacts.txt │ ├── nus │ ├── p100 │ ├── rtm.sh │ ├── run.bat │ ├── run_23-10-2017.sh │ ├── run_24_10_2017.sh │ ├── run_micro.sh │ └── test.xlsx ├── Flink-scripts │ ├── nus.sh │ ├── run.sh │ ├── run_app_flink.sh │ ├── run_app_flink_IC.sh │ ├── run_app_flink_OC.sh │ ├── run_app_flink_default.sh │ ├── run_app_flink_ss.sh │ ├── run_app_flink_ts.sh │ ├── run_app_flink_window.sh │ ├── start-flink.sh │ └── tools │ │ ├── ExecutionTimeanalysis.R │ │ ├── InstAnalysis2.R │ │ ├── _run_app - tmp_NUMA.bat │ │ ├── analysis_flink.sh │ │ ├── auto_setup.sh │ │ ├── bashrc │ │ ├── disable_socket0.sh │ │ ├── disable_socket1.sh │ │ ├── disable_socket2.sh │ │ ├── disable_socket3.sh │ │ ├── enable_all.sh │ │ ├── enable_socket0.sh │ │ ├── enable_socket1.sh │ │ ├── enable_socket2.sh │ │ ├── enable_socket3.sh │ │ ├── frontend_profile.sh │ │ ├── full_run_app_NUMA.sh │ │ ├── mem.py │ │ ├── numa.sh │ │ ├── profile.sh │ │ ├── profile_bacup.sh │ │ ├── profile_bacup2.sh │ │ ├── profile_full.sh │ │ ├── run_app.py │ │ └── stableanalysis.R ├── Spark-scripts │ ├── ExecutionTimeanalysis.R │ ├── install │ │ └── install.sh │ ├── passwordless.sh │ ├── run_app.py │ ├── run_app_spark.sh │ ├── run_app_spark_IC.sh │ ├── run_app_spark_OC.sh │ ├── run_app_spark_default.sh │ ├── run_app_spark_ss.sh │ ├── run_app_spark_ts.sh │ ├── run_app_spark_window.sh │ └── start-spark.sh ├── Storm-scripts │ ├── StormUIAPI │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── json.txt │ │ ├── query.py │ │ ├── requirements.txt │ │ └── stormuiapi.py │ ├── bin │ │ ├── Start-Storm.cmd │ │ ├── _run_app - tmp_NUMA.bat │ │ ├── flight.bash │ │ ├── start-nimbus.sh │ │ ├── start-supervisor.sh │ │ ├── start-ui.sh │ │ ├── storm │ │ ├── storm-config.cmd │ │ ├── storm.cmd │ │ └── storm.py │ ├── conf │ │ ├── storm.yaml │ │ └── storm_env.ini │ ├── nus.sh │ ├── run.sh │ ├── run_app_storm.sh │ ├── run_app_storm_IC.sh │ ├── run_app_storm_OC.sh │ ├── run_app_storm_default.sh │ ├── run_app_storm_ss.sh │ ├── run_app_storm_ts.sh │ ├── run_app_storm_window.sh │ ├── run_storm.sh │ ├── start-all.sh │ ├── start-nimbus.sh │ ├── start-supervisor.sh │ ├── start-ui.sh │ └── stop_storm.sh ├── common-scripts │ ├── kill_stormTopology.sh │ ├── ocprofile_PID.sh │ ├── passwordless.sh │ ├── run_kafka.sh │ ├── run_producer.sh │ ├── run_producer_default.sh │ ├── start_all.sh │ └── stop_all.sh ├── common.iml ├── lib │ ├── README.txt │ ├── classmexer.jar │ ├── flink-storm_2.11-1.5-SNAPSHOT.jar │ └── overseer.jar ├── pom.xml └── src │ ├── main │ ├── java │ │ ├── applications │ │ │ ├── CONTROL.java │ │ │ ├── Constants.java │ │ │ ├── abstractRunner.java │ │ │ └── utils.java │ │ ├── constants │ │ │ ├── BaseConstants.java │ │ │ ├── ClickAnalyticsConstants.java │ │ │ ├── CrossTableConstants.java │ │ │ ├── FraudDetectionConstants.java │ │ │ ├── LeaderboardConstants.java │ │ │ ├── LinearRoadConstants.java │ │ │ ├── LogProcessingConstants.java │ │ │ ├── MicroBenchmarkConstants.java │ │ │ ├── OnlineBidingSystemConstants.java │ │ │ ├── PositionKeepingConstants.java │ │ │ ├── SpikeDetectionConstants.java │ │ │ ├── TrafficMonitoringConstants.java │ │ │ ├── VoIPSTREAMConstants.java │ │ │ ├── VoterSStoreExampleConstants.java │ │ │ ├── WordCountConstants.java │ │ │ ├── WordCount_FTConstants.java │ │ │ └── streamingAnalysisConstants.java │ │ ├── helper │ │ │ ├── DataSource.java │ │ │ ├── Event.java │ │ │ ├── SimplePartitioner.java │ │ │ ├── helper.java │ │ │ ├── longrunning_sink_helper.java │ │ │ ├── parser │ │ │ │ └── Parser.java │ │ │ ├── sink_helper.java │ │ │ ├── stable_sink_helper.java │ │ │ ├── stable_sink_helper_bak.java │ │ │ ├── stable_sink_helper_multisink.java │ │ │ ├── stable_sink_helper_verbose.java │ │ │ ├── warmup_sink_helper.java │ │ │ └── wrapper │ │ │ │ ├── StringStatesWrapper.java │ │ │ │ ├── ValueStatesWrapper.java │ │ │ │ └── basic │ │ │ │ └── StateWrapper.java │ │ ├── machine │ │ │ ├── HP_Machine.java │ │ │ ├── HUAWEI_Machine.java │ │ │ ├── Platform.java │ │ │ └── RTM_Machine.java │ │ ├── model │ │ │ ├── ads │ │ │ │ └── AdEvent.java │ │ │ ├── cdr │ │ │ │ ├── CDRDataGenerator.java │ │ │ │ └── CallDetailRecord.java │ │ │ ├── finance │ │ │ │ ├── GoogleQuoteFetcher.java │ │ │ │ ├── IndicatorParameter.java │ │ │ │ ├── Quote.java │ │ │ │ ├── QuoteCollection.java │ │ │ │ ├── QuoteFetcher.java │ │ │ │ ├── TimeSeries.java │ │ │ │ └── YahooQuoteFetcher.java │ │ │ ├── geoip │ │ │ │ ├── GeoIP2Location.java │ │ │ │ ├── IPLocation.java │ │ │ │ ├── IPLocationFactory.java │ │ │ │ └── Location.java │ │ │ ├── gis │ │ │ │ ├── GPSRecord.java │ │ │ │ ├── Point.java │ │ │ │ ├── Polygon.java │ │ │ │ ├── Road.java │ │ │ │ ├── RoadGridList.java │ │ │ │ └── Sect.java │ │ │ ├── log │ │ │ │ ├── LogEntry.java │ │ │ │ ├── NotificationDetails.java │ │ │ │ └── Severity.java │ │ │ ├── metadata │ │ │ │ └── MachineMetadata.java │ │ │ └── predictor │ │ │ │ ├── IMarkovModelSource.java │ │ │ │ ├── MarkovModel.java │ │ │ │ ├── MarkovModelFileSource.java │ │ │ │ ├── MarkovModelPredictor.java │ │ │ │ ├── MarkovModelResourceSource.java │ │ │ │ ├── ModelBasedPredictor.java │ │ │ │ ├── Pair.java │ │ │ │ └── Prediction.java │ │ ├── parser │ │ │ ├── CommonLogParser.java │ │ │ ├── LBParser.java │ │ │ ├── SAParser.java │ │ │ ├── SensorParser.java │ │ │ ├── StringParser.java │ │ │ ├── TaxiTraceParser.java │ │ │ ├── TransactionParser.java │ │ │ └── voipParser.java │ │ ├── tasks │ │ │ ├── Executor.java │ │ │ ├── fully_stateful_task.java │ │ │ ├── partial_stateful_task.java │ │ │ ├── stateful_task.java │ │ │ ├── stateless_task.java │ │ │ └── stateless_taskImpl.java │ │ ├── tools │ │ │ ├── B_object.java │ │ │ ├── FastZipfGenerator.java │ │ │ ├── KB_object.java │ │ │ ├── KB_object_AsString.java │ │ │ ├── NUMA_analysis.java │ │ │ ├── RMA_analysis.java │ │ │ ├── Rankable.java │ │ │ ├── Rankings.java │ │ │ ├── SlidingWindowAggregator.java │ │ │ ├── SlidingWindowCounter.java │ │ │ ├── SlotBasedCounter.java │ │ │ ├── ZipfDistribution.java │ │ │ ├── ZipfGenerator.java │ │ │ ├── cacheSim │ │ │ │ ├── Cache.java │ │ │ │ ├── CacheSimulator.java │ │ │ │ ├── CaculateInst.java │ │ │ │ ├── Set.java │ │ │ │ ├── ZipfGenerator.java │ │ │ │ ├── cacheHitSimulator.java │ │ │ │ ├── cacheHitSimulator_Distribution.java │ │ │ │ ├── cacheHitSimulator_PI.java │ │ │ │ ├── cacheHitSimulator_PI_callflow.java │ │ │ │ ├── cacheHitSimulator_PI_generic.java │ │ │ │ ├── cacheHitSimulator_PI_generic_bak.java │ │ │ │ ├── cacheHitSimulator_PI_generic_cacheHit.java │ │ │ │ └── randomNumberGenerator.java │ │ │ ├── object.java │ │ │ ├── randomNumberGenerator.java │ │ │ ├── worker2.java │ │ │ └── zipf.java │ │ └── util │ │ │ ├── CacheInfo.java │ │ │ ├── ClassLoaderUtils.java │ │ │ ├── CompactHashMap │ │ │ ├── CompactHashMap.scala │ │ │ ├── FastHashSet.java │ │ │ ├── FastLinkedHashMap.java │ │ │ ├── FastLinkedHashSet.java │ │ │ └── QuickHashMap.java │ │ │ ├── Configuration.java │ │ │ ├── Constants.java │ │ │ ├── JavaUtils.java │ │ │ ├── MemoryMapReader.java │ │ │ ├── MemoryMapWriter.java │ │ │ ├── NUMA_analysis.java │ │ │ ├── OsUtils.java │ │ │ ├── STREAM.java │ │ │ ├── Time.java │ │ │ ├── collections │ │ │ ├── FixedMap.java │ │ │ ├── FixedSizeQueue.java │ │ │ └── LimitQueue.java │ │ │ ├── datatypes │ │ │ ├── DataTypeUtils.java │ │ │ ├── DateUtils.java │ │ │ ├── StreamValues.java │ │ │ └── TabularData.java │ │ │ ├── events │ │ │ ├── AccidentEvent.java │ │ │ ├── AccountBalanceEvent.java │ │ │ ├── ExpenditureEvent.java │ │ │ ├── HistoryEvent.java │ │ │ ├── LAVEvent.java │ │ │ ├── NOVEvent.java │ │ │ ├── PositionReportEvent.java │ │ │ └── TollCalculationEvent.java │ │ │ ├── hash │ │ │ ├── BloomCalculations.java │ │ │ ├── BloomFilter.java │ │ │ ├── MurmurHash.java │ │ │ └── ODTDBloomFilter.java │ │ │ ├── io │ │ │ ├── FileUtils.java │ │ │ └── IOUtils.java │ │ │ ├── lr │ │ │ └── Helper.java │ │ │ ├── maps │ │ │ └── BingMapsLookup.java │ │ │ ├── math │ │ │ ├── AverageTracker.java │ │ │ ├── Entropy.java │ │ │ ├── HistogramStat.java │ │ │ ├── MaximumLikelihoodNormalDistribution.java │ │ │ ├── OutlierTracker.java │ │ │ ├── RandomUtil.java │ │ │ ├── RunningMedianCalculator.java │ │ │ ├── SimpleStat.java │ │ │ ├── StateTransitionProbability.java │ │ │ ├── SummaryArchive.java │ │ │ └── VariableEWMA.java │ │ │ ├── model │ │ │ ├── ads │ │ │ │ └── AdEvent.java │ │ │ ├── finance │ │ │ │ ├── GoogleQuoteFetcher.java │ │ │ │ ├── IndicatorParameter.java │ │ │ │ ├── Quote.java │ │ │ │ ├── QuoteCollection.java │ │ │ │ ├── QuoteFetcher.java │ │ │ │ ├── TimeSeries.java │ │ │ │ └── YahooQuoteFetcher.java │ │ │ ├── log │ │ │ │ ├── LogEntry.java │ │ │ │ ├── NotificationDetails.java │ │ │ │ └── Severity.java │ │ │ ├── metadata │ │ │ │ └── MachineMetadata.java │ │ │ └── spam │ │ │ │ ├── OfflineTraining.java │ │ │ │ ├── Word.java │ │ │ │ └── WordMap.java │ │ │ └── window │ │ │ ├── SlidingWindow.java │ │ │ ├── SlidingWindowCallback.java │ │ │ └── SlidingWindowEntry.java │ └── resources │ │ ├── CountryCodes.json │ │ ├── config │ │ ├── CrossTables.properties │ │ ├── FraudDetection.properties │ │ ├── FraudDetection_latency.properties │ │ ├── Leaderboard.properties │ │ ├── LinearRoad.properties │ │ ├── LinearRoad_latency.properties │ │ ├── LogProcessing.properties │ │ ├── LogProcessing_latency.properties │ │ ├── MicroBenchmark.properties │ │ ├── OnlineBiding.properties │ │ ├── PositionKeeping.properties │ │ ├── SpikeDetection.properties │ │ ├── SpikeDetection_latency.properties │ │ ├── StreamingAnalysis.properties │ │ ├── TrafficMonitoring.properties │ │ ├── VoIPSTREAM.properties │ │ ├── WordCount.properties │ │ ├── WordCount_FT.properties │ │ └── WordCount_latency.properties │ │ ├── file.properties │ │ ├── log4j.properties │ │ └── standout.properties │ └── test │ └── java │ └── TestCommonRunner.java ├── executionTimeBreakdown.uxf ├── git_find_big.sh ├── list.sh ├── overseer-master ├── INSTALL ├── Makefile ├── examples │ ├── Makefile │ ├── java_agent.java │ ├── java_arch.java │ ├── java_hpc.java │ ├── java_ipmi.java │ ├── test.cpp │ ├── test_arch.cpp │ ├── test_java_agent.sh │ ├── test_java_arch.sh │ ├── test_java_hpc.sh │ ├── test_java_ipmi.sh │ └── test_mt.cpp ├── maven-install.sh └── src │ ├── hpcOverseer │ ├── Makefile │ ├── hpcOverseer.cpp │ ├── hpcOverseer.h │ ├── hpcOverseer_cpuinfo.cpp │ ├── hpcOverseer_rdtsc.cpp │ ├── hpcOverseer_scheduler.cpp │ └── hpcOverseer_tools.cpp │ ├── hpcOverseerWrapper │ ├── Makefile │ ├── ch_usi_overseerJNI_WrapperJNI.h │ └── hpcOverseerWrapper.cpp │ ├── java │ ├── Makefile │ └── ch │ │ └── usi │ │ └── overseer │ │ ├── OverAgent.java │ │ ├── OverHpc.java │ │ └── OverIpmi.java │ ├── overAgent │ ├── Makefile │ ├── ch_usi_overseerJNI_WrapperJNI.h │ ├── jniInterface.cpp │ ├── overAgent.cpp │ ├── overAgent.h │ ├── smartLock.h │ ├── threadList.cpp │ └── threadList.h │ └── overIpmi │ ├── Makefile │ ├── overIpmi.c │ └── overIpmi.h └── pom.xml /BriskBenchmarks/src/main/java/applications/bolts/udf/README.md: -------------------------------------------------------------------------------- 1 | udf: user-defined-function. 2 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/datatype/util/CarCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | package applications.datatype.util; 20 | 21 | 22 | /** 23 | * {@link CarCount} is an class that helps to count the number of cars in a segment. Its set_executor_ready count value_list is one. 24 | * 25 | * @author mjsax 26 | */ 27 | public final class CarCount { 28 | /** 29 | * The current count. 30 | */ 31 | public int count = 1; 32 | } 33 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/datatype/util/IPositionIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | package applications.datatype.util; 20 | 21 | import java.io.Serializable; 22 | 23 | 24 | /** 25 | * Each type that contains the three position identifier attributes XWAY, LANE, POSITION, DIR must implement this 26 | * interface. 27 | * 28 | * @author mjsax 29 | */ 30 | public interface IPositionIdentifier extends Serializable { 31 | 32 | Integer getXWay(); 33 | 34 | Short getLane(); 35 | 36 | Integer getPosition(); 37 | 38 | Short getDirection(); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/datatype/util/ISegmentIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | package applications.datatype.util; 20 | 21 | import java.io.Serializable; 22 | 23 | 24 | /** 25 | * Each type that contains the three segment identifier attributes XWAY, SEGMENT, DIR must implement this interface. 26 | * 27 | * @author mjsax 28 | */ 29 | public interface ISegmentIdentifier extends Serializable { 30 | 31 | Integer getXWay(); 32 | 33 | Short getSegment(); 34 | 35 | Short getDirection(); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/datatype/util/TimeStampExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | package applications.datatype.util; 20 | 21 | 22 | import org.apache.storm.tuple.Tuple; 23 | import org.apache.storm.tuple.Values; 24 | 25 | import java.io.Serializable; 26 | 27 | 28 | /** 29 | * {@link TimeStampExtractor} extract the timestamp from a given {@link Tuple} or {@link Values}. (Type {@code T} is 30 | * expected to be either {@link Tuple} (for usage in bolts) or {@link Values} (for usage in spouts).) 31 | * 32 | * @author mjsax 33 | */ 34 | public interface TimeStampExtractor extends Serializable { 35 | 36 | long getTs(T tuple); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/sink/EmptySink.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | import brisk.execution.runtime.tuple.TransferTuple; 4 | import brisk.execution.runtime.tuple.impl.Tuple; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | public class EmptySink extends BaseSink { 9 | private static final Logger LOG = LoggerFactory.getLogger(EmptySink.class); 10 | private static final long serialVersionUID = -2429018860900290157L; 11 | 12 | public EmptySink() { 13 | super(LOG); 14 | } 15 | 16 | 17 | @Override 18 | protected Logger getLogger() { 19 | return LOG; 20 | } 21 | 22 | @Override 23 | public void execute(Tuple in) throws InterruptedException { 24 | 25 | } 26 | 27 | @Override 28 | public void execute(TransferTuple in) throws InterruptedException { 29 | 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/sink/MeasureSink_FT.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | import brisk.execution.runtime.tuple.TransferTuple; 4 | import brisk.execution.runtime.tuple.impl.Marker; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | public class MeasureSink_FT extends MeasureSink { 9 | private static final Logger LOG = LoggerFactory.getLogger(MeasureSink_FT.class); 10 | private static final long serialVersionUID = 5481794109405775823L; 11 | 12 | 13 | @Override 14 | public void execute(TransferTuple input) { 15 | double results; 16 | int bound = input.length; 17 | for (int i = 0; i < bound; i++) { 18 | final Marker marker = input.getMarker(i); 19 | if (marker != null) { 20 | this.collector.ack(input, marker); 21 | continue; 22 | } 23 | 24 | results = helper.execute(input.getBID()); 25 | if (results != 0) { 26 | this.setResults(results); 27 | LOG.info("Sink finished:" + results); 28 | if (thisTaskId == graph.getSink().getExecutorID()) { 29 | measure_end(); 30 | } 31 | } 32 | } 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/sink/PKSink.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | import brisk.execution.runtime.tuple.impl.Tuple; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | public class PKSink extends MeasureSink { 8 | private static final Logger LOG = LoggerFactory.getLogger(PKSink.class); 9 | private static final long serialVersionUID = 5481794109405775823L; 10 | 11 | 12 | double success = 0; 13 | double failure = 0; 14 | 15 | @Override 16 | public void execute(Tuple input) { 17 | 18 | boolean result = input.getBoolean(0); 19 | if (result) { 20 | success++; 21 | } else 22 | failure++; 23 | 24 | double results; 25 | results = helper.execute(input.getBID()); 26 | if (results != 0) { 27 | this.setResults(results); 28 | LOG.info("Sink finished:" + results); 29 | if (thisTaskId == graph.getSink().getExecutorID()) { 30 | measure_end(); 31 | } 32 | } 33 | } 34 | 35 | 36 | public void display() { 37 | LOG.info("Spikes: " + success + "(" + (success / (success + failure)) + ")"); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/sink/formatter/BasicFormatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | import brisk.execution.runtime.tuple.impl.Tuple; 5 | 6 | /** 7 | * @author Maycon Viana Bordin 8 | */ 9 | public class BasicFormatter extends Formatter { 10 | 11 | @Override 12 | public String format(Tuple tuple) { 13 | Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); 14 | 15 | StringBuilder line = new StringBuilder(); 16 | 17 | for (int i = 0; i < tuple.fieldSize(); i++) { 18 | if (i != 0) { 19 | line.append(", "); 20 | } 21 | line.append(String.format("%s=%s", schema.get(i), tuple.getValue(i))); 22 | } 23 | 24 | return line.toString(); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/sink/formatter/Formatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import brisk.components.context.TopologyContext; 4 | import brisk.execution.runtime.tuple.impl.Tuple; 5 | import util.Configuration; 6 | 7 | public abstract class Formatter { 8 | TopologyContext context; 9 | 10 | public void initialize(Configuration config, TopologyContext context) { 11 | this.context = context; 12 | } 13 | 14 | public abstract String format(Tuple tuple); 15 | } 16 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/sink/formatter/FullInfoFormatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | import brisk.execution.runtime.tuple.impl.Tuple; 5 | 6 | /** 7 | * @author Maycon Viana Bordin 8 | */ 9 | public class FullInfoFormatter extends Formatter { 10 | private static final String TEMPLATE = "source: %s:%d, stream: %s, id: %s, values: [%s]"; 11 | 12 | @Override 13 | public String format(Tuple tuple) { 14 | Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); 15 | 16 | StringBuilder values = new StringBuilder(); 17 | for (int i = 0; i < tuple.fieldSize(); i++) { 18 | if (i != 0) { 19 | values.append(", "); 20 | } 21 | values.append(String.format("%s = %s", schema.get(i), tuple.getValue(i))); 22 | } 23 | 24 | return String.format(TEMPLATE, tuple.getSourceComponent(), tuple.getSourceTask(), 25 | tuple.getSourceStreamId(), values.toString()); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/spout/MemFileSpout_latency.java: -------------------------------------------------------------------------------- 1 | package applications.spout; 2 | 3 | public class MemFileSpout_latency extends MemFileSpout { 4 | private long msgID_counter = 0; 5 | 6 | @Override 7 | public void nextTuple() { 8 | 9 | counter++; 10 | if (counter == array_array.length) { 11 | counter = 0; 12 | } 13 | collector.emit_nowait(array_array[counter], msgID_counter++, System.nanoTime()); 14 | } 15 | 16 | } 17 | 18 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/spout/PhoneCallGeneratorSpout.java: -------------------------------------------------------------------------------- 1 | package applications.spout; 2 | 3 | import applications.spout.generator.PhoneCallGenerator; 4 | import brisk.components.operators.api.AbstractSpout; 5 | import brisk.execution.ExecutionGraph; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import static applications.Constants.DEFAULT_STREAM_ID; 10 | 11 | public class PhoneCallGeneratorSpout extends AbstractSpout { 12 | private static final Logger LOG = LoggerFactory.getLogger(PhoneCallGeneratorSpout.class); 13 | private static final long serialVersionUID = 7738169734935576086L; 14 | private PhoneCallGenerator callGenerator; 15 | 16 | public PhoneCallGeneratorSpout() { 17 | super(LOG); 18 | } 19 | 20 | @Override 21 | public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { 22 | super.initialize(thread_Id, thisTaskId, graph); 23 | int numContestants = 100; 24 | callGenerator = new PhoneCallGenerator(this.getContext().getThisTaskId(), numContestants); 25 | } 26 | 27 | @Override 28 | public void cleanup() { 29 | 30 | } 31 | 32 | @Override 33 | public void nextTuple() throws InterruptedException { 34 | collector.emit_bid(DEFAULT_STREAM_ID, callGenerator.next()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/spout/WCGeneratorSpout.java: -------------------------------------------------------------------------------- 1 | package applications.spout; 2 | 3 | import applications.spout.generator.WCGenerator; 4 | import brisk.components.operators.api.AbstractSpout; 5 | import brisk.execution.ExecutionGraph; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import static applications.Constants.DEFAULT_STREAM_ID; 10 | 11 | public class WCGeneratorSpout extends AbstractSpout { 12 | private static final Logger LOG = LoggerFactory.getLogger(WCGeneratorSpout.class); 13 | private static final long serialVersionUID = 7738169734935576086L; 14 | private WCGenerator Generator; 15 | 16 | public WCGeneratorSpout() { 17 | super(LOG); 18 | } 19 | 20 | @Override 21 | public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { 22 | super.initialize(thread_Id, thisTaskId, graph); 23 | String delimiter = ","; 24 | int number_of_words = 10; 25 | Generator = new WCGenerator(this.getContext().getThisTaskId(), number_of_words, delimiter); 26 | } 27 | 28 | @Override 29 | public void cleanup() { 30 | 31 | } 32 | 33 | @Override 34 | public void nextTuple() throws InterruptedException { 35 | collector.emit(DEFAULT_STREAM_ID, Generator.next()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/spout/generator/Generator.java: -------------------------------------------------------------------------------- 1 | package applications.spout.generator; 2 | 3 | import java.util.Random; 4 | 5 | public abstract class Generator { 6 | 7 | final Random rand = new Random(); 8 | 9 | public abstract T next(); 10 | } 11 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/applications/spout/generator/WCGenerator.java: -------------------------------------------------------------------------------- 1 | package applications.spout.generator; 2 | 3 | import java.util.Random; 4 | 5 | public class WCGenerator extends Generator { 6 | 7 | final Random rand = new Random(); 8 | private final String delimiter; 9 | private final int taskId; 10 | private final int number_of_words; 11 | 12 | public WCGenerator(int taskId, int number_of_words, String delimiter) { 13 | this.taskId = taskId; 14 | this.number_of_words = number_of_words; 15 | this.delimiter = delimiter; 16 | } 17 | 18 | @Override 19 | public char[] next() { 20 | 21 | StringBuffer sb = new StringBuffer(); 22 | 23 | for (int i = 0; i < number_of_words; i++) { 24 | sb.append(String.valueOf(rand.nextDouble())); 25 | if (i != number_of_words - 1) { 26 | sb.append(delimiter); 27 | } 28 | } 29 | 30 | return sb.toString().toCharArray(); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/exception/InvalidIDException.java: -------------------------------------------------------------------------------- 1 | package brisk.components.exception; 2 | 3 | /** 4 | * Custom exception 5 | */ 6 | public class InvalidIDException extends Exception { 7 | private static final long serialVersionUID = 4410379836219437416L; 8 | 9 | public InvalidIDException(String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/exception/UnhandledCaseException.java: -------------------------------------------------------------------------------- 1 | package brisk.components.exception; 2 | 3 | /** 4 | * Custom exception 5 | */ 6 | public class UnhandledCaseException extends Exception { 7 | private static final long serialVersionUID = 8308956830238052858L; 8 | 9 | public UnhandledCaseException(String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/grouping/AllGrouping.java: -------------------------------------------------------------------------------- 1 | package brisk.components.grouping; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | 5 | /** 6 | * Created by shuhaozhang on 12/7/16. 7 | */ 8 | public class AllGrouping extends Grouping { 9 | 10 | 11 | private static final long serialVersionUID = -2721357732532321681L; 12 | 13 | public AllGrouping(String componentId, String streamID) { 14 | super(componentId, streamID); 15 | } 16 | 17 | public AllGrouping(String componentId) { 18 | super(componentId); 19 | } 20 | 21 | @Override 22 | public Fields getFields() { 23 | return null; 24 | } 25 | 26 | 27 | } 28 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/grouping/FieldsGrouping.java: -------------------------------------------------------------------------------- 1 | package brisk.components.grouping; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | 5 | /** 6 | * Created by shuhaozhang on 12/7/16. 7 | */ 8 | public class FieldsGrouping extends Grouping { 9 | 10 | private static final long serialVersionUID = 6314571320061895100L; 11 | /** 12 | * The fields maintained in FieldsGrouping are "keys"... 13 | */ 14 | private Fields fields; 15 | 16 | 17 | public FieldsGrouping(String componentId, String streamID, Fields fields) { 18 | super(componentId, streamID); 19 | this.fields = fields; 20 | } 21 | 22 | public FieldsGrouping(String componentId, Fields fields) { 23 | super(componentId); 24 | this.fields = fields; 25 | } 26 | 27 | public Fields getFields() { 28 | return fields; 29 | } 30 | 31 | public void setFields(Fields fields) { 32 | this.fields = fields; 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/grouping/GlobalGrouping.java: -------------------------------------------------------------------------------- 1 | package brisk.components.grouping; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | 5 | /** 6 | * Created by shuhaozhang on 12/7/16. 7 | */ 8 | public class GlobalGrouping extends Grouping { 9 | 10 | 11 | private static final long serialVersionUID = -7376300455364041870L; 12 | 13 | public GlobalGrouping(String componentId, String streamID) { 14 | super(componentId, streamID); 15 | } 16 | 17 | public GlobalGrouping(String componentId) { 18 | super(componentId); 19 | } 20 | 21 | @Override 22 | public Fields getFields() { 23 | return null; 24 | } 25 | 26 | 27 | } 28 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/grouping/MarkerShuffleGrouping.java: -------------------------------------------------------------------------------- 1 | package brisk.components.grouping; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | 5 | /** 6 | * Created by shuhaozhang on 12/7/16. 7 | */ 8 | public class MarkerShuffleGrouping extends Grouping { 9 | 10 | 11 | private static final long serialVersionUID = 7558192140873152714L; 12 | 13 | public MarkerShuffleGrouping(String componentId, String streamID) { 14 | super(componentId, streamID); 15 | } 16 | 17 | public MarkerShuffleGrouping(String componentId) { 18 | super(componentId); 19 | } 20 | 21 | @Override 22 | public Fields getFields() { 23 | return null; 24 | } 25 | 26 | 27 | } 28 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/grouping/PartialKeyGrouping.java: -------------------------------------------------------------------------------- 1 | package brisk.components.grouping; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | 5 | /** 6 | * Created by shuhaozhang on 12/7/16. 7 | */ 8 | public class PartialKeyGrouping extends Grouping { 9 | 10 | private static final long serialVersionUID = 642110085528132186L; 11 | /** 12 | * The fields maintained in FieldsGrouping are "keys"... 13 | */ 14 | private Fields fields; 15 | 16 | 17 | public PartialKeyGrouping(String componentId, String streamID, Fields fields) { 18 | super(componentId, streamID); 19 | this.fields = fields; 20 | } 21 | 22 | public PartialKeyGrouping(String componentId, Fields fields) { 23 | super(componentId); 24 | this.fields = fields; 25 | } 26 | 27 | public Fields getFields() { 28 | return fields; 29 | } 30 | 31 | public void setFields(Fields fields) { 32 | this.fields = fields; 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/grouping/ShuffleGrouping.java: -------------------------------------------------------------------------------- 1 | package brisk.components.grouping; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | 5 | /** 6 | * Created by shuhaozhang on 12/7/16. 7 | */ 8 | public class ShuffleGrouping extends Grouping { 9 | 10 | 11 | private static final long serialVersionUID = 2906605250134845742L; 12 | 13 | public ShuffleGrouping(String componentId, String streamID) { 14 | super(componentId, streamID); 15 | } 16 | 17 | public ShuffleGrouping(String componentId) { 18 | super(componentId); 19 | } 20 | 21 | @Override 22 | public Fields getFields() { 23 | return null; 24 | } 25 | 26 | 27 | } 28 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/api/AbstractSpout.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.api; 2 | 3 | import brisk.execution.runtime.tuple.impl.Marker; 4 | import org.slf4j.Logger; 5 | 6 | /** 7 | * Abstract AbstractSpout is a special partition-pass Operator. 8 | */ 9 | public abstract class AbstractSpout extends Operator { 10 | 11 | private static final long serialVersionUID = -7455539617930687503L; 12 | 13 | 14 | //the following are used for checkpoint 15 | protected int myiteration = 0;//start from 1st iteration. 16 | protected boolean success = true; 17 | protected long boardcast_time; 18 | 19 | 20 | protected AbstractSpout(Logger log) { 21 | super(log, true, -1, 1); 22 | } 23 | 24 | protected String getConfigKey(String template) { 25 | return String.format(template, configPrefix); 26 | } 27 | 28 | public abstract void nextTuple() throws InterruptedException; 29 | 30 | public void nextTuple_nonblocking() throws InterruptedException { 31 | nextTuple(); 32 | } 33 | 34 | 35 | /** 36 | * When all my consumers callback_bolt, I can delete source message. 37 | * 38 | * @param callee 39 | * @param marker 40 | */ 41 | public void callback(int callee, Marker marker) { 42 | state.callback_spout(callee, marker, executor); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/api/AbstractWindowedBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.api; 2 | 3 | import brisk.components.windowing.TupleWindow; 4 | import brisk.execution.runtime.tuple.impl.Marker; 5 | import org.slf4j.Logger; 6 | 7 | import java.util.Map; 8 | 9 | public abstract class AbstractWindowedBolt extends Operator { 10 | private static final long serialVersionUID = -9211354361283989202L; 11 | 12 | AbstractWindowedBolt(Logger log, Map input_selectivity, Map output_selectivity, double branch_selectivity, double read_selectivity, boolean byP, double event_frequency, double window_size) { 13 | super(log, input_selectivity, output_selectivity, branch_selectivity, read_selectivity, byP, event_frequency, window_size); 14 | } 15 | 16 | AbstractWindowedBolt(double event_frequency, double w) { 17 | super(null, false, event_frequency, w); 18 | } 19 | 20 | 21 | public abstract void execute(TupleWindow in); 22 | 23 | 24 | @Override 25 | public void callback(int callee, Marker marker) { 26 | 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/api/BaseOperator.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.api; 2 | 3 | import org.slf4j.Logger; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * we don't have to differentiate parent operators or parent executors.. just use same list to represent them. 9 | */ 10 | public abstract class BaseOperator extends AbstractBolt { 11 | private static final long serialVersionUID = 9120945392080191838L; 12 | 13 | protected BaseOperator(Logger log, Map input_selectivity, Map output_selectivity, 14 | boolean byP, double event_frequency, double w) { 15 | super(log, input_selectivity, output_selectivity, byP, event_frequency, w); 16 | } 17 | 18 | protected BaseOperator(Logger log, Map input_selectivity, Map output_selectivity, 19 | double branch_selectivity, double read_selectivity, double event_frequency, double w) { 20 | super(log, input_selectivity, output_selectivity, branch_selectivity, read_selectivity, event_frequency, w); 21 | } 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/api/Checkpointable.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.api; 2 | 3 | import brisk.execution.runtime.tuple.impl.Marker; 4 | 5 | public interface Checkpointable { 6 | 7 | void forward_checkpoint(int sourceId, long bid, Marker marker) throws InterruptedException; 8 | 9 | void forward_checkpoint(int sourceTask, String streamId, long bid, Marker marker) throws InterruptedException; 10 | 11 | /** 12 | * Optionally relax_reset state before marker. 13 | * 14 | * @param marker 15 | */ 16 | void ack_checkpoint(Marker marker); 17 | 18 | 19 | void earlier_ack_checkpoint(Marker marker); 20 | } 21 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/api/IOperator.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.api; 2 | 3 | import brisk.components.context.TopologyContext; 4 | import brisk.execution.ExecutionGraph; 5 | import brisk.execution.runtime.collector.OutputCollector; 6 | import brisk.execution.runtime.tuple.impl.Marker; 7 | import brisk.execution.runtime.tuple.impl.OutputFieldsDeclarer; 8 | 9 | import java.io.Serializable; 10 | import java.util.Map; 11 | 12 | /** 13 | * Created by shuhaozhang on 12/7/16. 14 | */ 15 | public interface IOperator extends Serializable { 16 | /** 17 | * configure output fields through pass in a outputfields declarer. 18 | */ 19 | 20 | void declareOutputFields(OutputFieldsDeclarer declarer); 21 | 22 | void prepare(Map stormConf, TopologyContext context, OutputCollector collector); 23 | 24 | void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph); 25 | 26 | void cleanup(); 27 | 28 | void callback(int callee, Marker marker); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/api/WjoinBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.api; 2 | 3 | import org.slf4j.Logger; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * Created by I309939 on 8/28/2016. 9 | */ 10 | public abstract class WjoinBolt extends AbstractBolt { 11 | private static final long serialVersionUID = 3607973933453683397L; 12 | 13 | protected WjoinBolt(Logger log, Map input_selectivity, Map output_selectivity, boolean byP, double event_frequency, double w) { 14 | super(log, input_selectivity, output_selectivity, byP, event_frequency, w); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/base/MapBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.base; 2 | 3 | import brisk.components.operators.api.BaseOperator; 4 | import brisk.components.operators.api.Operator; 5 | import org.slf4j.Logger; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * Created by I309939 on 8/28/2016. 11 | * map Operator can only have partition input Operator. 12 | */ 13 | public abstract class MapBolt extends BaseOperator { 14 | private static final long serialVersionUID = 9189077925508299274L; 15 | 16 | protected MapBolt(Logger log, Map input_selectivity) { 17 | super(log, input_selectivity, null, false, 0, 1); 18 | } 19 | 20 | protected MapBolt(Map input_selectivity, Map output_selectivity) { 21 | super(null, input_selectivity, output_selectivity, 1, 1.0, 0, 1); 22 | } 23 | 24 | protected MapBolt(Logger log, double read_selectivity) { 25 | super(log, null, null, 1, read_selectivity, 0, 1); 26 | } 27 | 28 | 29 | protected MapBolt(Logger log) { 30 | super(log, null, null, false, 0, 1); 31 | } 32 | 33 | public String output_type() { 34 | return Operator.map; 35 | } 36 | 37 | 38 | } 39 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/base/filterBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.base; 2 | 3 | import brisk.components.operators.api.BaseOperator; 4 | import brisk.components.operators.api.Operator; 5 | import org.slf4j.Logger; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * Created by I309939 on 8/28/2016. 11 | */ 12 | public abstract class filterBolt extends BaseOperator { 13 | 14 | private static final long serialVersionUID = 234241824251364743L; 15 | 16 | protected filterBolt() { 17 | super(null, null, null, 0.5, 1, 0, 1); 18 | } 19 | 20 | protected filterBolt(Logger log, Map input_selectivity, Map output_selectivity) { 21 | super(log, input_selectivity, output_selectivity, false, 0, 1); 22 | } 23 | 24 | protected filterBolt(Logger log, Map input_selectivity, Map output_selectivity, double read_selectivity) { 25 | super(log, input_selectivity, output_selectivity, (double) 1, read_selectivity, 0, 1); 26 | } 27 | 28 | protected filterBolt(Logger log, Map output_selectivity) { 29 | super(log, null, output_selectivity, false, 0, 1); 30 | } 31 | 32 | public String output_type() { 33 | return Operator.filter; 34 | } 35 | 36 | 37 | } 38 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/base/splitBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.base; 2 | 3 | import brisk.components.operators.api.BaseOperator; 4 | import org.slf4j.Logger; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by tony on 5/8/2017. 10 | */ 11 | public abstract class splitBolt extends BaseOperator { 12 | 13 | private static final long serialVersionUID = 5684338552926136996L; 14 | 15 | public splitBolt(Logger log, Map input_selectivity, Map output_selectivity, 16 | double branch_selectivity, double read_selectivity) { 17 | super(log, input_selectivity, output_selectivity, branch_selectivity, read_selectivity, 1, 1); 18 | } 19 | 20 | public splitBolt(Logger log, Map output_selectivity) { 21 | super(log, null, output_selectivity, false, 0, 1); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/base/unionBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.base; 2 | 3 | import brisk.components.operators.api.BaseOperator; 4 | import brisk.components.operators.api.Operator; 5 | import org.slf4j.Logger; 6 | 7 | import java.util.Map; 8 | 9 | 10 | /** 11 | * Created by I309939 on 8/28/2016. 12 | */ 13 | public abstract class unionBolt extends BaseOperator { 14 | private static final long serialVersionUID = 4285498526255572237L; 15 | // private static final Logger LOG = LoggerFactory.getLogger(unionBolt.class); 16 | 17 | public unionBolt(Logger log, Map input_selectivity, Map output_selectivity) { 18 | super(log, input_selectivity, output_selectivity, false, 0, 1); 19 | } 20 | 21 | protected unionBolt(Logger log) { 22 | super(log, null, 23 | null, false, 0, 1); 24 | } 25 | 26 | protected unionBolt(Logger log, Map input_selectivity, Map output_selectivity, 27 | double branch_selectivity, double read_selectivity) { 28 | super(log, input_selectivity, output_selectivity, branch_selectivity, read_selectivity, 0, 0); 29 | 30 | } 31 | 32 | public String output_type() { 33 | return Operator.reduce; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/base/w_applyBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.base; 2 | 3 | import brisk.components.operators.api.BaseOperator; 4 | import brisk.components.operators.api.Operator; 5 | import org.slf4j.Logger; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * Created by I309939 on 8/28/2016. 11 | */ 12 | abstract class w_applyBolt extends BaseOperator { 13 | 14 | private static final long serialVersionUID = 742012390912849219L; 15 | 16 | public w_applyBolt(Logger log, Map input_selectivity, Map output_selectivity, boolean byP, double event_frequency, double w) { 17 | 18 | super(log, input_selectivity, output_selectivity, byP, event_frequency, w); 19 | } 20 | 21 | public w_applyBolt(Logger log, Map output_selectivity, boolean byP, double event_frequency, double w) { 22 | 23 | super(log, null, output_selectivity, byP, event_frequency, w); 24 | } 25 | 26 | public String output_type() { 27 | return Operator.w_apply; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/operators/base/w_joinBolt.java: -------------------------------------------------------------------------------- 1 | package brisk.components.operators.base; 2 | 3 | import brisk.components.operators.api.WjoinBolt; 4 | import org.slf4j.Logger; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by I309939 on 8/28/2016. 10 | */ 11 | abstract class w_joinBolt extends WjoinBolt { 12 | private static final long serialVersionUID = 5220064148086690758L; 13 | private final Logger LOG; 14 | 15 | public w_joinBolt(Logger log, Map input_selectivity, Map output_selectivity, boolean byP, double event_frequency, double w) { 16 | super(log, input_selectivity, output_selectivity, byP, event_frequency, w); 17 | LOG = log; 18 | } 19 | 20 | public w_joinBolt(Logger log, Map output_selectivity, boolean byP, double event_frequency, double w) { 21 | super(log, null, output_selectivity, byP, event_frequency, w); 22 | LOG = log; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/streaminfo.java: -------------------------------------------------------------------------------- 1 | package brisk.components; 2 | 3 | import brisk.execution.runtime.tuple.impl.Fields; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * Created by shuhaozhang on 13/7/16. 9 | */ 10 | public class streaminfo implements Serializable { 11 | private static final long serialVersionUID = 44L; 12 | 13 | 14 | /** 15 | * fields of the output stream.. 16 | * Don't be confused with fields maintained in FieldsGrouping --> those are "keys" declared by user application. 17 | */ 18 | private Fields fields; 19 | 20 | 21 | /** 22 | * TODO: add direct output type in future. 23 | * 24 | * @param fields 25 | * @param direct 26 | */ 27 | public streaminfo(Fields fields, boolean direct) { 28 | this.fields = fields; 29 | } 30 | 31 | 32 | public Fields getFields() { 33 | return fields; 34 | } 35 | 36 | 37 | public void setFields(Fields fields) { 38 | this.fields = fields; 39 | } 40 | 41 | 42 | } 43 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/BasicEvent.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | public class BasicEvent implements Event { 4 | private final T event; 5 | private final long ts; 6 | 7 | public BasicEvent(T event, long ts) { 8 | this.event = event; 9 | this.ts = ts; 10 | } 11 | 12 | @Override 13 | public long getTimestamp() { 14 | return ts; 15 | } 16 | 17 | @Override 18 | public T get() { 19 | return event; 20 | } 21 | 22 | @Override 23 | public boolean isWatermark() { 24 | return false; 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return "BasicEvent{" + 30 | "event=" + event + 31 | ", ts=" + ts + 32 | '}'; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/Event.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | 4 | /** 5 | * An event is a wrapper object that gets stored in the window. 6 | * 7 | * @param the type of the object thats wrapped. E.g Tuple 8 | */ 9 | public interface Event { 10 | /** 11 | * The event timestamp in millis. This could be the time 12 | * when the source generated the tuple or the time 13 | * when the tuple was received by a bolt. 14 | * 15 | * @return the event timestamp in milliseconds. 16 | */ 17 | long getTimestamp(); 18 | 19 | /** 20 | * Returns the wrapped object, E.g. a tuple 21 | * 22 | * @return the wrapped object. 23 | */ 24 | T get(); 25 | 26 | /** 27 | * If this is a watermark event or not. Watermark events are used 28 | * for tracking time while processing event based ts. 29 | * 30 | * @return true if this is a watermark event 31 | */ 32 | boolean isWatermark(); 33 | } 34 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/EvictionContext.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | /** 4 | * Context information that can be used by the eviction policy 5 | */ 6 | public interface EvictionContext { 7 | /** 8 | * Returns the reference time that the eviction policy could use to 9 | * evict the events. In the case of event time processing, this would be 10 | * the watermark time. 11 | * 12 | * @return the reference time in millis 13 | */ 14 | Long getReferenceTime(); 15 | 16 | /** 17 | * Returns the sliding count for count based windows 18 | * 19 | * @return the sliding count 20 | */ 21 | Long getSlidingCount(); 22 | 23 | 24 | /** 25 | * Returns the sliding interval for time based windows 26 | * 27 | * @return the sliding interval 28 | */ 29 | Long getSlidingInterval(); 30 | 31 | /** 32 | * Returns the current count of events in the queue up to the reference time 33 | * based on which count based evictions can be performed. 34 | * 35 | * @return the current count 36 | */ 37 | Long getCurrentCount(); 38 | } 39 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/TimestampExtractor.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | import brisk.execution.runtime.tuple.impl.Tuple; 4 | 5 | public interface TimestampExtractor { 6 | long extractTimestamp(Tuple tuple); 7 | } 8 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/TriggerHandler.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | /** 4 | * The callback_bolt fired by {@link TriggerPolicy} when the trigger 5 | * condition is satisfied. 6 | */ 7 | interface TriggerHandler { 8 | /** 9 | * The code to execute when the {@link TriggerPolicy} condition is satisfied. 10 | * 11 | * @return true if the window was evaluated with at least one event in the window, false otherwise 12 | */ 13 | boolean onTrigger(); 14 | } 15 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/TriggerPolicy.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | public interface TriggerPolicy { 4 | 5 | /** 6 | * Tracks the event and could use this to invoke the trigger. 7 | * 8 | * @param event the input event 9 | */ 10 | void track(Event event); 11 | 12 | /** 13 | * resets the trigger policy. 14 | */ 15 | void reset(); 16 | 17 | /** 18 | * Starts the trigger policy. This can be used 19 | * during recovery to start the triggers after 20 | * recovery is complete. 21 | */ 22 | void start(); 23 | 24 | /** 25 | * Any relax_reset up could be handled here. 26 | */ 27 | void shutdown(); 28 | 29 | /** 30 | * Return runtime state to be checkpointed by the framework for restoring the trigger policy 31 | * in case of failures. 32 | * 33 | * @return the state 34 | */ 35 | S getState(); 36 | 37 | /** 38 | * Restore the trigger policy from the state that was earlier checkpointed by the framework. 39 | * 40 | * @param state the state 41 | */ 42 | void restoreState(S state); 43 | 44 | } 45 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/TupleFieldTimestampExtractor.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | 4 | import brisk.execution.runtime.tuple.impl.Tuple; 5 | 6 | /** 7 | * A {@link org.apache.storm.windowing.TimestampExtractor} that extracts timestamp from a specific field in the tuple. 8 | */ 9 | public final class TupleFieldTimestampExtractor implements TimestampExtractor { 10 | private final String fieldName; 11 | 12 | private TupleFieldTimestampExtractor(String fieldName) { 13 | this.fieldName = fieldName; 14 | } 15 | 16 | public static TupleFieldTimestampExtractor of(String fieldName) { 17 | return new TupleFieldTimestampExtractor(fieldName); 18 | } 19 | 20 | @Override 21 | public long extractTimestamp(Tuple tuple) { 22 | return tuple.getLongByField(fieldName); 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return "TupleFieldTimestampExtractor{" + 28 | "fieldName='" + fieldName + '\'' + 29 | '}'; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/TupleWindow.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | import brisk.execution.runtime.tuple.impl.Tuple; 4 | 5 | /** 6 | * A {@link Window} that contains {@link Tuple} objects. 7 | */ 8 | public interface TupleWindow extends Window { 9 | } 10 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/components/windowing/WaterMarkEvent.java: -------------------------------------------------------------------------------- 1 | package brisk.components.windowing; 2 | 3 | 4 | /** 5 | * Watermark event used for tracking progress of time when 6 | * processing event based ts. 7 | */ 8 | public class WaterMarkEvent extends BasicEvent { 9 | public WaterMarkEvent(long ts) { 10 | super(null, ts); 11 | } 12 | 13 | @Override 14 | public boolean isWatermark() { 15 | return true; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return "WaterMarkEvent{} " + super.toString(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/controller/input/IISC.java: -------------------------------------------------------------------------------- 1 | package brisk.controller.input; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by shuhaozhang on 17/7/16. 7 | */ 8 | interface IISC extends Serializable { 9 | long serialVersionUID = 21L; 10 | } 11 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/RawExecutionGraph.java: -------------------------------------------------------------------------------- 1 | package brisk.execution; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.Serializable; 7 | import java.util.ArrayList; 8 | 9 | /** 10 | * Created by I309939 on 11/8/2016. 11 | */ 12 | public class RawExecutionGraph implements Serializable { 13 | private static final Logger LOG = LoggerFactory.getLogger(RawExecutionGraph.class); 14 | private static final long serialVersionUID = 5066635517599230626L; 15 | final ArrayList executionNodeArrayList; 16 | int vertex_id = 0;//Each executor has its unique vertex id. 17 | 18 | RawExecutionGraph() { 19 | executionNodeArrayList = new ArrayList<>(); 20 | } 21 | 22 | void addExecutor(ExecutionNode e) { 23 | 24 | executionNodeArrayList.add(e); 25 | } 26 | 27 | public ExecutionNode getExecutionNode(int id) { 28 | if (id == -1) { 29 | return null; 30 | } 31 | return executionNodeArrayList.get(id); 32 | } 33 | 34 | public ArrayList getExecutionNodeArrayList() { 35 | return executionNodeArrayList; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/collector/impl/BIDGenerator.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.collector.impl; 2 | 3 | import java.util.concurrent.atomic.AtomicInteger; 4 | 5 | /** 6 | * A singleton class 7 | */ 8 | public class BIDGenerator { 9 | private static BIDGenerator ourInstance = new BIDGenerator(); 10 | private final AtomicInteger bid = new AtomicInteger(); 11 | 12 | private BIDGenerator() { 13 | bid.set(0); 14 | } 15 | 16 | public static BIDGenerator getInstance() { 17 | return ourInstance; 18 | } 19 | 20 | public int getAndIncrement() { 21 | return bid.getAndIncrement(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/collector/impl/BIDGenerator2.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.collector.impl; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | 5 | /** 6 | * A singleton class 7 | */ 8 | public class BIDGenerator2 { 9 | private static BIDGenerator2 ourInstance = new BIDGenerator2(); 10 | private final AtomicLong bid = new AtomicLong(); 11 | 12 | private BIDGenerator2() { 13 | bid.set(0); 14 | } 15 | 16 | public static BIDGenerator2 getInstance() { 17 | return ourInstance; 18 | } 19 | 20 | public long getAndIncrement() { 21 | return bid.getAndIncrement(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/collector/impl/Meta.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.collector.impl; 2 | 3 | public class Meta { 4 | public final int src_id; 5 | public int index;//which index I have arrived. 6 | 7 | public Meta(int taskId) { 8 | this.src_id = taskId; 9 | index = 0; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/collector/impl/MetaGroup.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.collector.impl; 2 | 3 | import brisk.components.TopologyComponent; 4 | 5 | import java.util.HashMap; 6 | 7 | public class MetaGroup { 8 | private final int taskId; 9 | HashMap map = new HashMap<>();//every children op corresponds to one meta. 10 | 11 | public MetaGroup(int taskId) { 12 | this.taskId = taskId; 13 | } 14 | 15 | public Meta get(TopologyComponent childOP) { 16 | return map.get(childOP); 17 | } 18 | 19 | public void put(TopologyComponent childrenOP, Meta meta) { 20 | map.put(childrenOP, meta); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/tuple/impl/Message.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.tuple.impl; 2 | 3 | import java.io.Serializable; 4 | 5 | public abstract class Message implements Serializable { 6 | private static final long serialVersionUID = 17L; 7 | //three components are being serialized. 8 | //public int sourceId; //Executor who owns this tuple. Executor who receives this Brisk.execution.runtime.tuple would be then be able to know "who" creats and sents this Brisk.execution.runtime.tuple. 9 | public final String streamId; 10 | final int field_size; 11 | 12 | protected Message(String streamId, int field_size) { 13 | 14 | // this.sourceId = sourceId; 15 | this.streamId = streamId; 16 | this.field_size = field_size; 17 | } 18 | 19 | public abstract Object getValue(); 20 | 21 | public abstract Object getValue(int index_fields); 22 | 23 | // public abstract Object getMsg(); 24 | 25 | public String getStreamId() { 26 | return streamId; 27 | } 28 | 29 | public abstract boolean isMarker(); 30 | 31 | public abstract Marker getMarker(); 32 | } 33 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/tuple/impl/OutputFieldsDeclarer.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.tuple.impl; 2 | 3 | import brisk.components.streaminfo; 4 | 5 | import java.util.HashMap; 6 | 7 | import static applications.Constants.DEFAULT_STREAM_ID; 8 | 9 | /** 10 | * Created by shuhaozhang on 13/7/16. 11 | */ 12 | public class OutputFieldsDeclarer { 13 | 14 | private final HashMap _fields = new HashMap<>(); 15 | 16 | public void declare(Fields fields) { 17 | declare(false, fields); 18 | } 19 | 20 | private void declare(boolean direct, Fields fields) { 21 | declareStream(DEFAULT_STREAM_ID, direct, fields); 22 | } 23 | 24 | public void declareStream(String streamId, Fields fields) { 25 | 26 | declareStream(streamId, false, fields); 27 | } 28 | 29 | private void declareStream(String streamId, boolean direct, Fields fields) { 30 | if (_fields.containsKey(streamId)) { 31 | throw new IllegalArgumentException("Fields for " + streamId + " already set"); 32 | } 33 | _fields.put(streamId, new streaminfo(fields, direct)); 34 | } 35 | 36 | 37 | public HashMap getFieldsDeclaration() { 38 | 39 | return _fields; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/tuple/impl/StableValues.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.tuple.impl; 2 | 3 | import org.apache.commons.lang.SerializationUtils; 4 | import util.datatypes.StreamValues; 5 | 6 | import java.io.Serializable; 7 | import java.util.ArrayList; 8 | 9 | /** 10 | * Created by I309939 on 12/7/2016. 11 | */ 12 | class StableValues extends ArrayList { 13 | 14 | 15 | private static final long serialVersionUID = 4056690381745622263L; 16 | 17 | public static StreamValues create(Object... vals) { 18 | 19 | Object[] ret = new Object[vals.length]; 20 | int i = 0; 21 | for (Object v : vals) { 22 | ret[i++] = SerializationUtils.clone((Serializable) v); 23 | } 24 | 25 | return new StreamValues(ret); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/tuple/impl/msgs/GeneralMsg.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.tuple.impl.msgs; 2 | 3 | import brisk.execution.runtime.tuple.impl.Marker; 4 | import brisk.execution.runtime.tuple.impl.Message; 5 | 6 | import java.util.Arrays; 7 | import java.util.LinkedList; 8 | 9 | /** 10 | * Created by tony on 5/30/2017. 11 | *

12 | * General Message. 13 | * For performance consideration, do not use this. 14 | */ 15 | public class GeneralMsg extends Message { 16 | private static final long serialVersionUID = -1840662997355503904L; 17 | public final LinkedList values = new LinkedList<>(); 18 | 19 | public GeneralMsg(String streamId, T... values) { 20 | super(streamId, values.length); 21 | this.values.addAll(Arrays.asList(values)); 22 | } 23 | 24 | public GeneralMsg(String streamId, T values) { 25 | super(streamId, 1); 26 | this.values.add(values); 27 | } 28 | 29 | @Override 30 | public Marker getMarker() { 31 | return null; 32 | } 33 | 34 | @Override 35 | public Object getValue() { 36 | return values; 37 | } 38 | 39 | @Override 40 | public T getValue(int index_fields) { 41 | return values.get(index_fields); 42 | } 43 | 44 | 45 | public boolean isMarker() { 46 | return false; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/tuple/impl/msgs/StringLongMsg.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.tuple.impl.msgs; 2 | 3 | import brisk.execution.runtime.tuple.impl.Marker; 4 | import brisk.execution.runtime.tuple.impl.Message; 5 | 6 | public class StringLongMsg extends Message { 7 | private static final long serialVersionUID = -285716889341771284L; 8 | public final char[] str; 9 | public final long value; 10 | 11 | public StringLongMsg(String streamId, char[] str, long value) { 12 | super(streamId, 2); 13 | this.str = str; 14 | this.value = value; 15 | } 16 | 17 | @Override 18 | public Object getValue() { 19 | return str + "" + value; 20 | } 21 | 22 | @Override 23 | public Object getValue(int index_fields) { 24 | switch (index_fields) { 25 | case 0: 26 | return str; 27 | case 1: 28 | return value; 29 | default: 30 | throw new IndexOutOfBoundsException(String.valueOf(index_fields)); 31 | } 32 | } 33 | 34 | @Override 35 | public boolean isMarker() { 36 | return false; 37 | } 38 | 39 | @Override 40 | public Marker getMarker() { 41 | return null; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/tuple/impl/msgs/StringMsg.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.tuple.impl.msgs; 2 | 3 | import brisk.execution.runtime.tuple.impl.Marker; 4 | import brisk.execution.runtime.tuple.impl.Message; 5 | 6 | public class StringMsg extends Message { 7 | private static final long serialVersionUID = -8382182763339974896L; 8 | public final char[] str; 9 | 10 | public StringMsg(String streamId, char[] str) { 11 | super(streamId, 1); 12 | this.str = str; 13 | } 14 | 15 | @Override 16 | public char[] getValue() { 17 | return str; 18 | } 19 | 20 | @Override 21 | public char[] getValue(int index_fields) { 22 | return str; 23 | } 24 | 25 | @Override 26 | public boolean isMarker() { 27 | return false; 28 | } 29 | 30 | @Override 31 | public Marker getMarker() { 32 | return null; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/execution/runtime/tuple/impl/msgs/Tuple2Msg.java: -------------------------------------------------------------------------------- 1 | package brisk.execution.runtime.tuple.impl.msgs; 2 | 3 | import brisk.execution.runtime.tuple.impl.Marker; 4 | import brisk.execution.runtime.tuple.impl.Message; 5 | 6 | public class Tuple2Msg extends Message { 7 | private static final long serialVersionUID = -1296552220431181613L; 8 | public final T0 value0; 9 | public final T1 value1; 10 | 11 | public Tuple2Msg(String streamId, T0 value0, T1 value1) { 12 | super(streamId, 2); 13 | this.value0 = value0; 14 | this.value1 = value1; 15 | } 16 | 17 | @Override 18 | public Object getValue() { 19 | return value0 + "" + value1; 20 | } 21 | 22 | @Override 23 | public Object getValue(int index_fields) { 24 | 25 | switch (index_fields) { 26 | case 0: 27 | return value0; 28 | case 1: 29 | return value1; 30 | default: 31 | throw new IndexOutOfBoundsException(String.valueOf(index_fields)); 32 | } 33 | } 34 | 35 | @Override 36 | public boolean isMarker() { 37 | return false; 38 | } 39 | 40 | @Override 41 | public Marker getMarker() { 42 | return null; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/faulttolerance/compress.java: -------------------------------------------------------------------------------- 1 | package brisk.faulttolerance; 2 | 3 | import net.jpountz.lz4.LZ4BlockOutputStream; 4 | 5 | import java.io.FileInputStream; 6 | import java.io.FileOutputStream; 7 | import java.io.IOException; 8 | 9 | public class compress { 10 | 11 | 12 | public static void LZ4compress(String filename, String lz4file) { 13 | byte[] buf = new byte[2048]; 14 | try { 15 | String outFilename = lz4file; 16 | LZ4BlockOutputStream out = new LZ4BlockOutputStream(new FileOutputStream(outFilename), 32 * 1024 * 1024); 17 | FileInputStream in = new FileInputStream(filename); 18 | int len; 19 | while ((len = in.read(buf)) > 0) { 20 | out.write(buf, 0, len); 21 | } 22 | in.close(); 23 | out.close(); 24 | } catch (IOException ignored) { 25 | 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/faulttolerance/impl/ValueState.java: -------------------------------------------------------------------------------- 1 | package brisk.faulttolerance.impl; 2 | 3 | 4 | import brisk.faulttolerance.State; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * Single value_list State 10 | */ 11 | public class ValueState extends State { 12 | private static final long serialVersionUID = 5982252287746705128L; 13 | E value; 14 | 15 | public ValueState() { 16 | 17 | } 18 | 19 | /** 20 | * Updates the operator state accessible by ValueState.value_list() to the given value_list. 21 | * 22 | * @param value 23 | */ 24 | @Override 25 | public void update(E value) { 26 | this.value = value; 27 | } 28 | 29 | @Override 30 | public E value() { 31 | return value; 32 | } 33 | 34 | @Override 35 | public void clean() { 36 | value = null; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/optimization/impl/Decision.java: -------------------------------------------------------------------------------- 1 | package brisk.optimization.impl; 2 | 3 | import brisk.execution.ExecutionNode; 4 | 5 | public class Decision { 6 | public final ExecutionNode producer; 7 | public final ExecutionNode consumer; 8 | 9 | public Decision(ExecutionNode producer, ExecutionNode consumer) { 10 | this.producer = producer; 11 | this.consumer = consumer; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/optimization/impl/scaling/Parallelism.java: -------------------------------------------------------------------------------- 1 | package brisk.optimization.impl.scaling; 2 | 3 | import java.util.HashMap; 4 | 5 | 6 | public class Parallelism extends HashMap { 7 | private static final long serialVersionUID = 3574295926004443115L; 8 | 9 | public int total_num; 10 | 11 | public Parallelism(Parallelism parallelism) { 12 | for (String topo : parallelism.keySet()) { 13 | this.put(topo, parallelism.get(topo)); 14 | } 15 | this.total_num = parallelism.total_num; 16 | } 17 | 18 | public Parallelism() { 19 | 20 | } 21 | 22 | public void increment(String id, int inc) { 23 | this.put(id, this.getOrDefault(id, 0) + inc); 24 | this.total_num += inc; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/optimization/impl/scheduling/randomPlan_Constraints.java: -------------------------------------------------------------------------------- 1 | package brisk.optimization.impl.scheduling; 2 | 3 | import brisk.execution.ExecutionGraph; 4 | import brisk.execution.ExecutionNode; 5 | import brisk.optimization.impl.SchedulingPlan; 6 | import brisk.optimization.model.Constraints; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import util.Configuration; 10 | 11 | import java.util.ArrayList; 12 | 13 | public class randomPlan_Constraints extends randomSearch_Constraints { 14 | private final static Logger LOG = LoggerFactory.getLogger(randomPlan_Constraints.class); 15 | 16 | public randomPlan_Constraints(ExecutionGraph graph, int numNodes, int numCPUs, Constraints cons, Configuration conf) { 17 | super(graph, numNodes, numCPUs, cons, conf); 18 | } 19 | 20 | @Override 21 | public SchedulingPlan Search(boolean worst_plan, int timeoutMs) { 22 | initilize(worst_plan, conf); 23 | //if it is not used in set_executor_ready plan.. 24 | 25 | LOG.info("Randomly search for a plan"); 26 | //main course. 27 | final ArrayList sort_opList = graph.sort(); 28 | return Packing(new SchedulingPlan(currentPlan, false), graph, sort_opList); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/optimization/impl/scheduling/randomPlan_NoConstraints.java: -------------------------------------------------------------------------------- 1 | package brisk.optimization.impl.scheduling; 2 | 3 | import brisk.execution.ExecutionGraph; 4 | import brisk.execution.ExecutionNode; 5 | import brisk.optimization.impl.SchedulingPlan; 6 | import brisk.optimization.model.Constraints; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import util.Configuration; 10 | 11 | import java.util.ArrayList; 12 | 13 | public class randomPlan_NoConstraints extends randomSearch_NoConstrains { 14 | private final static Logger LOG = LoggerFactory.getLogger(randomPlan_NoConstraints.class); 15 | 16 | public randomPlan_NoConstraints(ExecutionGraph graph, int numNodes, int numCPUs, Constraints cons, Configuration conf) { 17 | super(graph, numNodes, numCPUs, cons, conf); 18 | } 19 | 20 | @Override 21 | public SchedulingPlan Search(boolean worst_plan, int timeoutMs) { 22 | initilize(worst_plan, conf); 23 | //if it is not used in set_executor_ready plan.. 24 | 25 | LOG.info("Randomly search for a plan"); 26 | //main course. 27 | final ArrayList sort_opList = graph.sort(); 28 | final SchedulingPlan currentplan = new SchedulingPlan(currentPlan, false); 29 | return Packing(currentplan, graph, sort_opList); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/optimization/impl/scheduling/roundrobin.java: -------------------------------------------------------------------------------- 1 | package brisk.optimization.impl.scheduling; 2 | 3 | import brisk.execution.ExecutionGraph; 4 | import brisk.execution.ExecutionNode; 5 | import brisk.optimization.impl.SchedulingPlan; 6 | import brisk.optimization.model.Constraints; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import util.Configuration; 10 | 11 | import java.util.ArrayList; 12 | import java.util.Iterator; 13 | 14 | /** 15 | * Created by tony on 7/11/2017. 16 | */ 17 | public class roundrobin extends randomPlan_NoConstraints { 18 | private final static Logger LOG = LoggerFactory.getLogger(roundrobin.class); 19 | 20 | public roundrobin(ExecutionGraph graph, int numNodes, int numCPUs, Constraints cons, Configuration conf) { 21 | super(graph, numNodes, numCPUs, cons, conf); 22 | LOG.info("Round robin based scheduling."); 23 | } 24 | 25 | SchedulingPlan Packing(SchedulingPlan sp, ExecutionGraph graph, ArrayList sort_opList) { 26 | 27 | final Iterator iterator = sort_opList.iterator(); 28 | int s = 0; 29 | while (iterator.hasNext()) { 30 | ExecutionNode executor = iterator.next(); 31 | sp.allocate(executor, s++ % numNodes); 32 | } 33 | sp.set_success(); 34 | return sp; 35 | } 36 | } -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/optimization/model/GraphMetrics.java: -------------------------------------------------------------------------------- 1 | package brisk.optimization.model; 2 | 3 | import applications.Constants; 4 | import brisk.execution.ExecutionGraph; 5 | import brisk.execution.ExecutionNode; 6 | import brisk.optimization.impl.SchedulingPlan; 7 | 8 | 9 | /** 10 | * Created by I309939 on 11/8/2016. 11 | */ 12 | public class GraphMetrics { 13 | private final ExecutionNode virtualGround; 14 | private final SchedulingPlan schedulingPlan; 15 | private final boolean backPressure; 16 | 17 | public GraphMetrics(SchedulingPlan schedulingPlan, boolean backPressure) { 18 | this.backPressure = backPressure; 19 | ExecutionGraph graph = schedulingPlan.graph; 20 | virtualGround = graph.getvirtualGround(); 21 | this.schedulingPlan = schedulingPlan; 22 | } 23 | 24 | public double getOutput_rate(boolean bound) { 25 | if (backPressure && !schedulingPlan.BP_calculated) { 26 | BackPressure.BP(schedulingPlan); 27 | } 28 | return virtualGround.CleangetOutput_rate(Constants.DEFAULT_STREAM_ID, Constants.DEFAULT_STREAM_ID, schedulingPlan, bound); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/optimization/model/Variables.java: -------------------------------------------------------------------------------- 1 | package brisk.optimization.model; 2 | 3 | import java.io.Serializable; 4 | 5 | import static util.Constants.default_sourceRate; 6 | 7 | /** 8 | * Created by tony on 7/4/2017. 9 | */ 10 | public class Variables implements Serializable { 11 | private static final long serialVersionUID = 40L; 12 | public double SOURCE_RATE = default_sourceRate / 1.0E+09;//event/ns 13 | 14 | public Variables(Variables variables) { 15 | this.SOURCE_RATE = variables.SOURCE_RATE; 16 | } 17 | 18 | public Variables() { 19 | 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/queue/QueueController.java: -------------------------------------------------------------------------------- 1 | package brisk.queue; 2 | 3 | import brisk.execution.ExecutionNode; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.io.Serializable; 8 | import java.util.HashMap; 9 | import java.util.Queue; 10 | 11 | public abstract class QueueController implements Serializable { 12 | private static final long serialVersionUID = 12L; 13 | 14 | private static final Logger LOG = LoggerFactory.getLogger(QueueController.class); 15 | final HashMap downExecutor_list; 16 | 17 | QueueController(HashMap downExecutor_list) { 18 | this.downExecutor_list = downExecutor_list; 19 | } 20 | 21 | public abstract Queue get_queue(int executor); 22 | 23 | 24 | public abstract void allocate_queue(boolean linked, int desired_elements_epoch_per_core); 25 | 26 | public abstract boolean isEmpty(); 27 | } 28 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/queue/impl/PaddedAtomicLong.java: -------------------------------------------------------------------------------- 1 | package brisk.queue.impl; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | 5 | class PaddedAtomicLong extends AtomicLong { 6 | private static final long serialVersionUID = 5826979353123727160L; 7 | public volatile long p1, p2, p3, p4, p5, p6 = 7; 8 | 9 | public PaddedAtomicLong() { 10 | } 11 | 12 | public PaddedAtomicLong(final long initialValue) { 13 | super(initialValue); 14 | } 15 | } -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/queue/impl/PaddedLong.java: -------------------------------------------------------------------------------- 1 | package brisk.queue.impl; 2 | 3 | public class PaddedLong { 4 | public long value = 0, p1, p2, p3, p4, p5, p6; 5 | } -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/queue/impl/UnsafeAccess.java: -------------------------------------------------------------------------------- 1 | package brisk.queue.impl; 2 | 3 | import sun.misc.Unsafe; 4 | 5 | import java.lang.reflect.Field; 6 | 7 | class UnsafeAccess { 8 | public static final Unsafe unsafe; 9 | 10 | static { 11 | try { 12 | // This is a bit of voodoo to force the unsafe object into 13 | // visibility and acquire it. 14 | // This is not playing nice, but as an established back door it is 15 | // not likely to be 16 | // taken away. 17 | Field field = Unsafe.class.getDeclaredField("theUnsafe"); 18 | field.setAccessible(true); 19 | unsafe = (Unsafe) field.get(null); 20 | } catch (Exception e) { 21 | throw new RuntimeException(e); 22 | } 23 | } 24 | 25 | } -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/queue/impl/wait/WaitStrategy.java: -------------------------------------------------------------------------------- 1 | package brisk.queue.impl.wait; 2 | 3 | 4 | import brisk.queue.impl.PaddedLong; 5 | 6 | import java.util.concurrent.atomic.AtomicLong; 7 | 8 | public interface WaitStrategy { 9 | 10 | int waitFor(AtomicLong tail, AtomicLong head, int capacity, PaddedLong headCache) throws InterruptedException; 11 | 12 | void signalAllWhenBlocking(); 13 | } -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/topology/TopologyComiler.java: -------------------------------------------------------------------------------- 1 | package brisk.topology; 2 | 3 | import brisk.components.Topology; 4 | import brisk.execution.ExecutionGraph; 5 | import util.Configuration; 6 | 7 | /** 8 | * Created by shuhaozhang on 11/7/16. 9 | */ 10 | class TopologyComiler { 11 | 12 | //TODO: It is possible to have a static-lanuch here. For example, code-generation techniques. 13 | public ExecutionGraph generateEG(Topology topology, Configuration conf) { 14 | //Construct Brisk.execution Graph structure based on information from this Brisk.topology. 15 | return new ExecutionGraph(topology, null, conf); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/util/ExpiringMap.java: -------------------------------------------------------------------------------- 1 | package brisk.util; 2 | 3 | class ExpiringMap { 4 | 5 | // public ExpiringMap(){ 6 | // // Create hash map with default set_executor_ready capacity, load factor, number of threads, 7 | //// and eviction scheduler 8 | //// An instance of SingleThreadEvictionScheduler is used in this case 9 | // ConcurrentMapWithTimedEviction map = 10 | // new ConcurrentHashMapWithTimedEviction<>(); 11 | // 12 | //// Create delayed Task eviction scheduler 13 | // EvictionScheduler scheduler = new DelayedTaskEvictionScheduler<>(); 14 | //// Create hash map with default set_executor_ready capacity, load factor, and transaction level, 15 | //// and the previously created scheduler 16 | // ConcurrentMapWithTimedEviction map = 17 | // new ConcurrentHashMapWithTimedEviction<>(scheduler); 18 | // 19 | // 20 | // } 21 | 22 | 23 | } 24 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/util/ObjectSizeFetcher.java: -------------------------------------------------------------------------------- 1 | package brisk.util; 2 | 3 | import java.lang.instrument.Instrumentation; 4 | 5 | class ObjectSizeFetcher { 6 | private static Instrumentation instrumentation; 7 | 8 | public static void premain(String args, Instrumentation inst) { 9 | instrumentation = inst; 10 | } 11 | 12 | public static long getObjectSize(Object o) { 13 | return instrumentation.getObjectSize(o); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/util/SlidingWindow.java: -------------------------------------------------------------------------------- 1 | package brisk.util; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by shuhaozhang on 21/11/16. 7 | */ 8 | public class SlidingWindow implements Serializable { 9 | private static final long serialVersionUID = 7L; 10 | private final V[] storage; 11 | private int ct = 0; 12 | 13 | public SlidingWindow(int size) { 14 | storage = (V[]) new Object[size]; 15 | } 16 | 17 | public boolean build() { 18 | return get(storage.length - 1) != null; 19 | } 20 | 21 | public V get(int i) { 22 | return storage[i]; 23 | } 24 | 25 | public void put(V item) { 26 | storage[ct % storage.length] = item; 27 | ct++; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /BriskBenchmarks/src/main/java/brisk/util/myIntegerMap.java: -------------------------------------------------------------------------------- 1 | package brisk.util; 2 | 3 | 4 | import java.util.LinkedHashMap; 5 | 6 | /** 7 | * Created by tony on 7/8/2017. 8 | */ 9 | public class myIntegerMap extends LinkedHashMap { 10 | 11 | private static final long serialVersionUID = 4202592582267245248L; 12 | 13 | public Integer get(Object key) { 14 | return super.get(key) == null ? 0 : super.get(key); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/datatypes/util/CarCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | package applications.datatypes.util; 20 | 21 | 22 | /** 23 | * {@link CarCount} is an class that helps to count the number of cars in a segment. Its initial count value is one. 24 | * 25 | * @author mjsax 26 | */ 27 | public final class CarCount { 28 | /** 29 | * The current count. 30 | */ 31 | public int count = 1; 32 | } 33 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/datatypes/util/IPositionIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | package applications.datatypes.util; 20 | 21 | import java.io.Serializable; 22 | 23 | 24 | /** 25 | * Each type that contains the three position identifier attributes XWAY, LANE, POSITION, DIR must implement this 26 | * interface. 27 | * 28 | * @author mjsax 29 | */ 30 | public interface IPositionIdentifier extends Serializable { 31 | static final long serialVersionUID = 25L; 32 | 33 | Integer getXWay(); 34 | 35 | Short getLane(); 36 | 37 | Integer getPosition(); 38 | 39 | Short getDirection(); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/datatypes/util/ISegmentIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | package applications.datatypes.util; 20 | 21 | import java.io.Serializable; 22 | 23 | 24 | /** 25 | * Each type that contains the three segment identifier attributes XWAY, SEGMENT, DIR must implement this interface. 26 | * 27 | * @author mjsax 28 | */ 29 | public interface ISegmentIdentifier extends Serializable { 30 | static final long serialVersionUID = 26L; 31 | 32 | Integer getXWay(); 33 | 34 | Short getSegment(); 35 | 36 | Short getDirection(); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/sink/BaseSink.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | import applications.bolts.AbstractBolt; 4 | import applications.sink.formatter.BasicFormatter; 5 | import applications.sink.formatter.Formatter; 6 | import constants.BaseConstants.BaseConf; 7 | import org.apache.storm.tuple.Fields; 8 | import org.slf4j.Logger; 9 | import util.ClassLoaderUtils; 10 | 11 | 12 | public abstract class BaseSink extends AbstractBolt { 13 | protected Formatter formatter; 14 | 15 | @Override 16 | public void initialize() { 17 | String formatterClass = config.getString(getConfigKey(BaseConf.SINK_FORMATTER), null); 18 | 19 | if (formatterClass == null) { 20 | formatter = new BasicFormatter(); 21 | } else { 22 | formatter = (Formatter) ClassLoaderUtils.newInstance(formatterClass, "formatter", getLogger()); 23 | } 24 | 25 | formatter.initialize(config, context); 26 | } 27 | 28 | @Override 29 | public Fields getDefaultFields() { 30 | return new Fields(""); 31 | } 32 | 33 | protected String getConfigKey(String template) { 34 | return String.format(template, configPrefix); 35 | } 36 | 37 | protected abstract Logger getLogger(); 38 | 39 | protected void killTopology() { 40 | System.exit(0); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/sink/ForwardSink.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | 4 | import helper.stable_sink_helper; 5 | import org.apache.storm.tuple.Tuple; 6 | import org.apache.storm.tuple.Values; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * @author mayconbordin 12 | */ 13 | public class ForwardSink extends BaseSink { 14 | private static final Logger LOG = LoggerFactory.getLogger(ForwardSink.class); 15 | stable_sink_helper helper; 16 | 17 | @Override 18 | public void initialize() { 19 | 20 | helper = new stable_sink_helper(LOG 21 | , config.getInt("runtimeInSeconds") 22 | , config.getString("metrics.output"), config.getDouble("predict", 0), 0, context.getThisTaskId()); 23 | } 24 | 25 | @Override 26 | public void execute(Tuple input) { 27 | // if (stat != null) stat.start_measure(); 28 | collector.emit(new Values(input)); 29 | // if (stat != null) stat.end_measure(); 30 | } 31 | 32 | @Override 33 | protected Logger getLogger() { 34 | return LOG; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/sink/formatter/BasicFormatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.tuple.Fields; 4 | import org.apache.storm.tuple.Tuple; 5 | 6 | /** 7 | * @author Maycon Viana Bordin 8 | */ 9 | public class BasicFormatter extends Formatter { 10 | 11 | @Override 12 | public String format(Tuple tuple) { 13 | Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); 14 | 15 | String line = ""; 16 | 17 | for (int i = 0; i < tuple.size(); i++) { 18 | if (i != 0) line += ", "; 19 | line += String.format("%s=%s", schema.get(i), tuple.getValue(i)); 20 | } 21 | 22 | return line; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/sink/formatter/Formatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.task.TopologyContext; 4 | import org.apache.storm.tuple.Tuple; 5 | import util.Configuration; 6 | 7 | /** 8 | * @author Maycon Viana Bordin 9 | */ 10 | public abstract class Formatter { 11 | protected Configuration config; 12 | protected TopologyContext context; 13 | 14 | public void initialize(Configuration config, TopologyContext context) { 15 | this.config = config; 16 | this.context = context; 17 | } 18 | 19 | public abstract String format(Tuple tuple); 20 | } 21 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/main/java/applications/sink/formatter/FullInfoFormatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.tuple.Fields; 4 | import org.apache.storm.tuple.Tuple; 5 | 6 | /** 7 | * @author Maycon Viana Bordin 8 | */ 9 | public class FullInfoFormatter extends Formatter { 10 | private static final String TEMPLATE = "source: %s:%d, stream: %s, id: %s, values: [%s]"; 11 | 12 | @Override 13 | public String format(Tuple tuple) { 14 | Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); 15 | 16 | String values = ""; 17 | for (int i = 0; i < tuple.size(); i++) { 18 | if (i != 0) values += ", "; 19 | values += String.format("%s = %s", schema.get(i), tuple.getValue(i)); 20 | } 21 | 22 | return String.format(TEMPLATE, tuple.getSourceComponent(), tuple.getSourceTask(), 23 | tuple.getSourceStreamId(), tuple.getMessageId().toString(), values); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /FlinkBenchmarks/src/test/java/TestFlinkRunner.java: -------------------------------------------------------------------------------- 1 | import applications.FlinkRunner; 2 | import org.junit.Test; 3 | 4 | /** 5 | * Created by I309939 on 8/4/2016. 6 | */ 7 | public class TestFlinkRunner { 8 | private String topic = "topic"; 9 | 10 | @Test 11 | public void Test() { 12 | String[] args; 13 | args = new String[]{ 14 | " --microbenchmark", 15 | "-a", 16 | "WordCount", 17 | "-tt", 18 | "25" 19 | }; 20 | 21 | try { 22 | FlinkRunner.main(args); 23 | } catch (Exception e) { 24 | e.printStackTrace(); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /HeronBenchmarks/src/main/java/applications/bolts/wc/WordCountBolt.java: -------------------------------------------------------------------------------- 1 | package applications.bolts.udf.wc; 2 | 3 | import applications.bolts.AbstractBolt; 4 | import constants.WordCountConstants.Field; 5 | import org.apache.commons.lang.mutable.MutableLong; 6 | import org.apache.storm.tuple.Fields; 7 | import org.apache.storm.tuple.Tuple; 8 | import org.apache.storm.tuple.Values; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | 16 | public class WordCountBolt extends AbstractBolt { 17 | private static final Logger LOG = LoggerFactory.getLogger(WordCountBolt.class); 18 | private final Map counts = new HashMap<>(); 19 | 20 | @Override 21 | public Fields getDefaultFields() { 22 | return new Fields(Field.WORD, Field.COUNT); 23 | } 24 | 25 | @Override 26 | public void execute(Tuple input) { 27 | String word = input.getStringByField(Field.WORD); 28 | MutableLong count = counts.computeIfAbsent(word, k -> new MutableLong(0)); 29 | count.increment(); 30 | Values objects = new Values(word, count.longValue()); //StableValues.create(word, count.longValue()); 31 | collector.emit(objects); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /HeronBenchmarks/src/main/java/applications/sink/BaseSink.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | import applications.bolts.AbstractBolt; 4 | import applications.sink.formatter.BasicFormatter; 5 | import applications.sink.formatter.Formatter; 6 | import constants.BaseConstants.BaseConf; 7 | import org.apache.storm.tuple.Fields; 8 | import org.slf4j.Logger; 9 | import util.ClassLoaderUtils; 10 | 11 | 12 | public abstract class BaseSink extends AbstractBolt { 13 | protected Formatter formatter; 14 | 15 | @Override 16 | public void initialize() { 17 | String formatterClass = config.getString(getConfigKey(BaseConf.SINK_FORMATTER), null); 18 | 19 | if (formatterClass == null) { 20 | formatter = new BasicFormatter(); 21 | } else { 22 | formatter = (Formatter) ClassLoaderUtils.newInstance(formatterClass, "formatter", getLogger()); 23 | } 24 | 25 | formatter.initialize(config, context); 26 | } 27 | 28 | @Override 29 | public Fields getDefaultFields() { 30 | return new Fields(""); 31 | } 32 | 33 | protected String getConfigKey(String template) { 34 | return String.format(template, configPrefix); 35 | } 36 | 37 | protected abstract Logger getLogger(); 38 | 39 | protected void killTopology() { 40 | System.exit(0); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /HeronBenchmarks/src/main/java/applications/sink/ForwardSink.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | 4 | import helper.stable_sink_helper; 5 | import org.apache.storm.tuple.Tuple; 6 | import org.apache.storm.tuple.Values; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * @author mayconbordin 12 | */ 13 | public class ForwardSink extends BaseSink { 14 | private static final Logger LOG = LoggerFactory.getLogger(ForwardSink.class); 15 | stable_sink_helper helper; 16 | 17 | @Override 18 | public void initialize() { 19 | 20 | helper = new stable_sink_helper(LOG 21 | , config.getInt("runtimeInSeconds") 22 | , config.getString("metrics.output"), config.getDouble("predict", 0), 0, context.getThisTaskId()); 23 | } 24 | 25 | @Override 26 | public void execute(Tuple input) { 27 | // if (stat != null) stat.start_measure(); 28 | collector.emit(new Values(input.getValues())); 29 | // if (stat != null) stat.end_measure(); 30 | } 31 | 32 | @Override 33 | protected Logger getLogger() { 34 | return LOG; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /HeronBenchmarks/src/main/java/applications/sink/formatter/BasicFormatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.tuple.Fields; 4 | import org.apache.storm.tuple.Tuple; 5 | 6 | /** 7 | * @author Maycon Viana Bordin 8 | */ 9 | public class BasicFormatter extends Formatter { 10 | 11 | @Override 12 | public String format(Tuple tuple) { 13 | Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); 14 | 15 | String line = ""; 16 | 17 | for (int i = 0; i < tuple.size(); i++) { 18 | if (i != 0) line += ", "; 19 | line += String.format("%s=%s", schema.get(i), tuple.getValue(i)); 20 | } 21 | 22 | return line; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /HeronBenchmarks/src/main/java/applications/sink/formatter/Formatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.task.TopologyContext; 4 | import org.apache.storm.tuple.Tuple; 5 | import util.Configuration; 6 | 7 | /** 8 | * @author Maycon Viana Bordin 9 | */ 10 | public abstract class Formatter { 11 | protected Configuration config; 12 | protected TopologyContext context; 13 | 14 | public void initialize(Configuration config, TopologyContext context) { 15 | this.config = config; 16 | this.context = context; 17 | } 18 | 19 | public abstract String format(Tuple tuple); 20 | } 21 | -------------------------------------------------------------------------------- /HeronBenchmarks/src/test/java/TestHeronRunner.java: -------------------------------------------------------------------------------- 1 | import applications.HeronRunner; 2 | import org.junit.Test; 3 | 4 | /** 5 | * Created by I309939 on 8/4/2016. 6 | */ 7 | public class TestHeronRunner { 8 | private String topic = "topic"; 9 | 10 | @Test 11 | public void Test() { 12 | System.out.println("Start test clean;"); 13 | String[] args; 14 | args = new String[]{ 15 | "-a", 16 | "FraudDetection", 17 | "-tt", 18 | "25" 19 | }; 20 | try { 21 | HeronRunner.main(args); 22 | } catch (Exception e) { 23 | e.printStackTrace(); 24 | } 25 | 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/datatypes/util/CarCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | 20 | package applications.datatypes.util; 21 | 22 | 23 | /** 24 | * {@link CarCount} is an class that helps to count the number of cars in a segment. Its initial count value is one. 25 | * 26 | * @author mjsax 27 | */ 28 | public final class CarCount { 29 | /** 30 | * The current count. 31 | */ 32 | public int count = 1; 33 | } 34 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/datatypes/util/IPositionIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | 20 | package applications.datatypes.util; 21 | 22 | import java.io.Serializable; 23 | 24 | 25 | /** 26 | * Each type that contains the three position identifier attributes XWAY, LANE, POSITION, DIR must implement this 27 | * interface. 28 | * 29 | * @author mjsax 30 | */ 31 | public interface IPositionIdentifier extends Serializable { 32 | 33 | Integer getXWay(); 34 | 35 | Short getLane(); 36 | 37 | Integer getPosition(); 38 | 39 | Short getDirection(); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/datatypes/util/ISegmentIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | 20 | package applications.datatypes.util; 21 | 22 | import java.io.Serializable; 23 | 24 | 25 | /** 26 | * Each type that contains the three segment identifier attributes XWAY, SEGMENT, DIR must implement this interface. 27 | * 28 | * @author mjsax 29 | */ 30 | public interface ISegmentIdentifier extends Serializable { 31 | 32 | Integer getXWay(); 33 | 34 | Short getSegment(); 35 | 36 | Short getDirection(); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/datatypes/util/TimeStampExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * #! 3 | * % 4 | * Copyright (C) 2014 - 2015 Humboldt-Universität zu Berlin 5 | * % 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * #_ 18 | */ 19 | 20 | package applications.datatypes.util; 21 | 22 | 23 | import org.apache.storm.tuple.Tuple; 24 | import org.apache.storm.tuple.Values; 25 | 26 | import java.io.Serializable; 27 | 28 | 29 | /** 30 | * {@link TimeStampExtractor} extract the timestamp from a given {@link Tuple} or {@link Values}. (Type {@code T} is 31 | * expected to be either {@link Tuple} (for usage in bolts) or {@link Values} (for usage in spouts).) 32 | * 33 | * @author mjsax 34 | */ 35 | public interface TimeStampExtractor extends Serializable { 36 | 37 | long getTs(T tuple); 38 | 39 | } 40 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/sink/ForwardSink.java: -------------------------------------------------------------------------------- 1 | package applications.sink; 2 | 3 | 4 | import helper.stable_sink_helper; 5 | import org.apache.storm.tuple.Tuple; 6 | import org.apache.storm.tuple.Values; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * @author mayconbordin 12 | */ 13 | public class ForwardSink extends BaseSink { 14 | private static final Logger LOG = LoggerFactory.getLogger(ForwardSink.class); 15 | stable_sink_helper helper; 16 | 17 | public ForwardSink() { 18 | super(); 19 | } 20 | 21 | @Override 22 | public void initialize() { 23 | 24 | helper = new stable_sink_helper(LOG 25 | , config.getInt("runtimeInSeconds") 26 | , config.getString("metrics.output"), config.getDouble("predict", 0), 0, context.getThisTaskId()); 27 | } 28 | 29 | @Override 30 | public void execute(Tuple input) { 31 | 32 | collector.emit(new Values(input)); 33 | } 34 | 35 | @Override 36 | protected Logger getLogger() { 37 | return LOG; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/sink/formatter/BasicFormatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.tuple.Fields; 4 | import org.apache.storm.tuple.Tuple; 5 | 6 | /** 7 | * @author Maycon Viana Bordin 8 | */ 9 | public class BasicFormatter extends Formatter { 10 | 11 | @Override 12 | public String format(Tuple tuple) { 13 | Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); 14 | 15 | StringBuilder line = new StringBuilder(); 16 | 17 | for (int i = 0; i < tuple.size(); i++) { 18 | if (i != 0) { 19 | line.append(", "); 20 | } 21 | line.append(String.format("%s=%s", schema.get(i), tuple.getValue(i))); 22 | } 23 | 24 | return line.toString(); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/sink/formatter/Formatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.task.TopologyContext; 4 | import org.apache.storm.tuple.Tuple; 5 | import util.Configuration; 6 | 7 | /** 8 | * @author Maycon Viana Bordin 9 | */ 10 | public abstract class Formatter { 11 | protected Configuration config; 12 | protected TopologyContext context; 13 | 14 | public void initialize(Configuration config, TopologyContext context) { 15 | this.config = config; 16 | this.context = context; 17 | } 18 | 19 | public abstract String format(Tuple tuple); 20 | } 21 | -------------------------------------------------------------------------------- /StormBenchmarks/src/main/java/applications/sink/formatter/FullInfoFormatter.java: -------------------------------------------------------------------------------- 1 | package applications.sink.formatter; 2 | 3 | import org.apache.storm.tuple.Fields; 4 | import org.apache.storm.tuple.Tuple; 5 | 6 | /** 7 | * @author Maycon Viana Bordin 8 | */ 9 | public class FullInfoFormatter extends Formatter { 10 | private static final String TEMPLATE = "source: %s:%d, stream: %s, id: %s, values: [%s]"; 11 | 12 | @Override 13 | public String format(Tuple tuple) { 14 | Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); 15 | 16 | StringBuilder values = new StringBuilder(); 17 | for (int i = 0; i < tuple.size(); i++) { 18 | if (i != 0) { 19 | values.append(", "); 20 | } 21 | values.append(String.format("%s = %s", schema.get(i), tuple.getValue(i))); 22 | } 23 | 24 | return String.format(TEMPLATE, tuple.getSourceComponent(), tuple.getSourceTask(), 25 | tuple.getSourceStreamId(), tuple.getMessageId().toString(), values.toString()); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /StormBenchmarks/src/test/java/TestStormRunner.java: -------------------------------------------------------------------------------- 1 | import applications.StormRunner; 2 | import org.junit.Test; 3 | 4 | /** 5 | * Created by I309939 on 8/4/2016. 6 | */ 7 | public class TestStormRunner { 8 | private String topic = "topic"; 9 | 10 | @Test 11 | public void Test() { 12 | System.out.println("Start test clean;"); 13 | String[] args; 14 | args = new String[]{ 15 | "-a", 16 | "LogProcessing", 17 | "-tt", 18 | "25" 19 | }; 20 | try { 21 | StormRunner.main(args); 22 | } catch (Exception e) { 23 | e.printStackTrace(); 24 | } 25 | 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /affinity/src/main/java/net/openhft/affinity/AffinityStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 higherfrequencytrading.com 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package net.openhft.affinity; 19 | 20 | /** 21 | * Allow you define a strategy for find the a cpu relative to another select cpu. 22 | * 23 | * @author peter.lawrey 24 | */ 25 | public interface AffinityStrategy { 26 | /** 27 | * @param cpuId to cpuId to compare 28 | * @param cpuId2 with a second cpuId 29 | * @return true if it matches the criteria. 30 | */ 31 | boolean matches(int cpuId, int cpuId2); 32 | } 33 | -------------------------------------------------------------------------------- /affinity/src/main/java/net/openhft/affinity/AffinitySupport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 higherfrequencytrading.com 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package net.openhft.affinity; 19 | 20 | /** 21 | * For backward compatibility with Affinity 2.x 22 | */ 23 | @Deprecated 24 | public class AffinitySupport { 25 | 26 | public static int getThreadId() { 27 | return Affinity.getThreadId(); 28 | } 29 | 30 | public static void setThreadId() { 31 | Affinity.setThreadId(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /affinity/src/main/java/net/openhft/ticker/ITicker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 higherfrequencytrading.com 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package net.openhft.ticker; 19 | 20 | /* 21 | * Created by Peter Lawrey on 13/07/15. 22 | */ 23 | public interface ITicker { 24 | long nanoTime(); 25 | 26 | long ticks(); 27 | 28 | long toNanos(long ticks); 29 | 30 | double toMicros(double ticks); 31 | } 32 | -------------------------------------------------------------------------------- /affinity/src/test/resources/i7.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 higherfrequencytrading.com 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # 17 | 0=0,0,0 18 | 1=0,1,0 19 | 2=0,2,0 20 | 3=0,3,0 21 | 4=0,0,1 22 | 5=0,1,1 23 | 6=0,2,1 24 | 7=0,3,1 25 | -------------------------------------------------------------------------------- /bitbucket-pipelines.yml: -------------------------------------------------------------------------------- 1 | # This is a sample build configuration for Java (Maven). 2 | # Check our guides at https://confluence.atlassian.com/x/zd-5Mw for more examples. 3 | # Only use spaces to indent your .yml configuration. 4 | # ----- 5 | # You can specify a custom docker image from Docker Hub as your build environment. 6 | image: maven:3.3.9 7 | 8 | pipelines: 9 | default: 10 | - step: 11 | caches: 12 | - maven 13 | script: # Modify the commands below to build your repository. 14 | - mvn validate 15 | - mvn -X verify # -B batch mode makes Maven less verbose -------------------------------------------------------------------------------- /briskstream.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /common/Brisk_scripts/message.txt: -------------------------------------------------------------------------------- 1 | done 2 | -------------------------------------------------------------------------------- /common/Brisk_scripts/mvn_install.sh: -------------------------------------------------------------------------------- 1 | mvn install:install-file -Dfile=$HOME/briskstream/common/lib/overseer.jar -DgroupId=org.Overseer -DartifactId=Overseer -Dversion=1.0 -Dpackaging=jar -DlocalRepositoryPath=$HOME/briskstream/common/lib 2 | -------------------------------------------------------------------------------- /common/Brisk_scripts/mycontacts.txt: -------------------------------------------------------------------------------- 1 | tony tonyzhang19900609@gmail.com 2 | -------------------------------------------------------------------------------- /common/Brisk_scripts/run.bat: -------------------------------------------------------------------------------- 1 | SET JRE_HOME="C:\Program Files\Java\jdk1.8.0_131" 2 | SET arg_benchmark= -a WordCount --THz 450000 --sim --num_socket 8 --num_cpu 8 --monte -st 5 -sit 5 3 | 4 | set /a P=1 5 | :loopP 6 | set /a A=1 7 | :loopA 8 | set /a B=1 9 | :loopB 10 | for /l %%x in (1, 1, 1000) do ( 11 | %JRE_HOME%\bin\java.exe -jar C:\Users\tony\Documents\briskstream\BriskBenchamrks\target\briskstream-1.2.0-jar-with-dependencies.jar %arg_benchmark% -pt %P% -ct1 %A% -ct2 %B% >> monte_%P%_%A%_%B%.txt 12 | ) 13 | 14 | set /a "B = B * 2" 15 | if %B% LEQ 32 goto loopB 16 | 17 | set /a "A = A * 2" 18 | if %A% LEQ 32 goto loopA 19 | 20 | set /a "P = P * 2" 21 | if %P% LEQ 32 goto loopP 22 | -------------------------------------------------------------------------------- /common/Brisk_scripts/test.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xtra-Computing/briskstream/21220007589bd5352b6a8d9053a4b5ba2921b927/common/Brisk_scripts/test.xlsx -------------------------------------------------------------------------------- /common/Flink-scripts/run_app_flink.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | run_app_flink_IC.sh $1 3 | run_app_flink_OC.sh $1 4 | run_app_flink_window.sh $1 5 | run_app_flink_ss.sh $1 -------------------------------------------------------------------------------- /common/Flink-scripts/start-flink.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | FLINK_HOME="$HOME/Documents/flink-1.3.2" 3 | $FLINK_HOME/bin/stop-cluster.sh 4 | sleep 5 5 | $FLINK_HOME/bin/start-cluster.sh 6 | sleep 15 7 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/auto_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #apt-get install vim -y && 3 | #sh ~/Downloads/parallel_studio_xe_2016_update2/install.sh&& 4 | tar -xf ~/Downloads/jdk-8u77-linux-x64.tar.gz -C ~/Documents && 5 | tar -xf ~/Downloads/smartgit-linux-7_1_2.tar.gz -C ~/Documents && 6 | tar -xf ~/Downloads/apache-storm-0.9.5.tar.gz -C ~/Documents && 7 | tar -xf ~/Downloads/ideaIC-2016.1.1.tar.gz -C ~/Documents && 8 | tar -xf ~/Downloads/zookeeper-3.4.8.tar.gz -C ~/Documents && 9 | 10 | echo "JAVA_HOME=~/Documents/jdk1.8.0_77" >> ~/.bashrc && 11 | echo "export PATH=$PATH:~/parallel_studio/vtune_amplifier_xe_2016/bin64:~/Documents/apache-compatibility-1.0.0/bin">> ~/.bashrc && 12 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections && 13 | sudo add-apt-repository ppa:webupd8team/java && 14 | sudo apt-get update && 15 | sudo apt-get install oracle-java8-installer -y 16 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/disable_socket0.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket0 : 0 4 8 12 16 20 24 28 3 | echo 0 | sudo tee /sys/devices/system/cpu/cpu0/online 4 | echo 0 | sudo tee /sys/devices/system/cpu/cpu4/online 5 | echo 0 | sudo tee /sys/devices/system/cpu/cpu8/online 6 | echo 0 | sudo tee /sys/devices/system/cpu/cpu12/online 7 | echo 0 | sudo tee /sys/devices/system/cpu/cpu16/online 8 | echo 0 | sudo tee /sys/devices/system/cpu/cpu20/online 9 | echo 0 | sudo tee /sys/devices/system/cpu/cpu24/online 10 | echo 0 | sudo tee /sys/devices/system/cpu/cpu28/online 11 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/disable_socket1.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket1 : 1 5 9 13 17 21 25 29 3 | echo 0 | sudo tee /sys/devices/system/cpu/cpu1/online 4 | echo 0 | sudo tee /sys/devices/system/cpu/cpu5/online 5 | echo 0 | sudo tee /sys/devices/system/cpu/cpu9/online 6 | echo 0 | sudo tee /sys/devices/system/cpu/cpu13/online 7 | echo 0 | sudo tee /sys/devices/system/cpu/cpu17/online 8 | echo 0 | sudo tee /sys/devices/system/cpu/cpu21/online 9 | echo 0 | sudo tee /sys/devices/system/cpu/cpu25/online 10 | echo 0 | sudo tee /sys/devices/system/cpu/cpu29/online 11 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/disable_socket2.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket2 : 2 6 10 14 18 22 26 30 3 | echo 0 | sudo tee /sys/devices/system/cpu/cpu2/online 4 | echo 0 | sudo tee /sys/devices/system/cpu/cpu6/online 5 | echo 0 | sudo tee /sys/devices/system/cpu/cpu10/online 6 | echo 0 | sudo tee /sys/devices/system/cpu/cpu14/online 7 | echo 0 | sudo tee /sys/devices/system/cpu/cpu18/online 8 | echo 0 | sudo tee /sys/devices/system/cpu/cpu22/online 9 | echo 0 | sudo tee /sys/devices/system/cpu/cpu26/online 10 | echo 0 | sudo tee /sys/devices/system/cpu/cpu30/online 11 | 12 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/disable_socket3.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket3 : 3 7 11 15 19 23 27 31 3 | echo 0 | sudo tee /sys/devices/system/cpu/cpu3/online 4 | echo 0 | sudo tee /sys/devices/system/cpu/cpu7/online 5 | echo 0 | sudo tee /sys/devices/system/cpu/cpu11/online 6 | echo 0 | sudo tee /sys/devices/system/cpu/cpu15/online 7 | echo 0 | sudo tee /sys/devices/system/cpu/cpu19/online 8 | echo 0 | sudo tee /sys/devices/system/cpu/cpu23/online 9 | echo 0 | sudo tee /sys/devices/system/cpu/cpu27/online 10 | echo 0 | sudo tee /sys/devices/system/cpu/cpu31/online 11 | 12 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/enable_all.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | . ~/enable_socket0.sh 3 | . ~/enable_socket1.sh 4 | . ~/enable_socket2.sh 5 | . ~/enable_socket3.sh 6 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/enable_socket0.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket0 : 0 4 8 12 16 21 24 28 3 | echo 1 | sudo tee /sys/devices/system/cpu/cpu0/online 4 | echo 1 | sudo tee /sys/devices/system/cpu/cpu4/online 5 | echo 1 | sudo tee /sys/devices/system/cpu/cpu8/online 6 | echo 1 | sudo tee /sys/devices/system/cpu/cpu12/online 7 | echo 1 | sudo tee /sys/devices/system/cpu/cpu16/online 8 | echo 1 | sudo tee /sys/devices/system/cpu/cpu21/online 9 | echo 1 | sudo tee /sys/devices/system/cpu/cpu24/online 10 | echo 1 | sudo tee /sys/devices/system/cpu/cpu28/online 11 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/enable_socket1.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket1 : 1 5 9 13 17 21 25 29 3 | echo 1 | sudo tee /sys/devices/system/cpu/cpu1/online 4 | echo 1 | sudo tee /sys/devices/system/cpu/cpu5/online 5 | echo 1 | sudo tee /sys/devices/system/cpu/cpu9/online 6 | echo 1 | sudo tee /sys/devices/system/cpu/cpu13/online 7 | echo 1 | sudo tee /sys/devices/system/cpu/cpu17/online 8 | echo 1 | sudo tee /sys/devices/system/cpu/cpu21/online 9 | echo 1 | sudo tee /sys/devices/system/cpu/cpu25/online 10 | echo 1 | sudo tee /sys/devices/system/cpu/cpu29/online 11 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/enable_socket2.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket2 : 2 6 10 14 18 22 26 30 3 | echo 1 | sudo tee /sys/devices/system/cpu/cpu2/online 4 | echo 1 | sudo tee /sys/devices/system/cpu/cpu6/online 5 | echo 1 | sudo tee /sys/devices/system/cpu/cpu10/online 6 | echo 1 | sudo tee /sys/devices/system/cpu/cpu14/online 7 | echo 1 | sudo tee /sys/devices/system/cpu/cpu18/online 8 | echo 1 | sudo tee /sys/devices/system/cpu/cpu22/online 9 | echo 1 | sudo tee /sys/devices/system/cpu/cpu26/online 10 | echo 1 | sudo tee /sys/devices/system/cpu/cpu30/online 11 | 12 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/enable_socket3.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | #socket3 : 3 7 11 15 19 23 27 31 3 | echo 1 | sudo tee /sys/devices/system/cpu/cpu3/online 4 | echo 1 | sudo tee /sys/devices/system/cpu/cpu7/online 5 | echo 1 | sudo tee /sys/devices/system/cpu/cpu11/online 6 | echo 1 | sudo tee /sys/devices/system/cpu/cpu15/online 7 | echo 1 | sudo tee /sys/devices/system/cpu/cpu19/online 8 | echo 1 | sudo tee /sys/devices/system/cpu/cpu23/online 9 | echo 1 | sudo tee /sys/devices/system/cpu/cpu27/online 10 | echo 1 | sudo tee /sys/devices/system/cpu/cpu31/online 11 | 12 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/frontend_profile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ocperf.py stat -e \ 3 | cpu_clk_unhalted.thread_p_any,\ 4 | dtlb_store_misses.stlb_hit,\ 5 | dtlb_store_misses.walk_duration,\ 6 | dtlb_load_misses.stlb_hit,\ 7 | dtlb_load_misses.walk_duration,\ 8 | icache.misses,\ 9 | idq.empty,\ 10 | idq_uops_not_delivered.core,\ 11 | ild_stall.iq_full,\ 12 | ild_stall.lcp,\ 13 | inst_retired.any,\ 14 | int_misc.recovery_cycles,\ 15 | itlb_misses.stlb_hit,\ 16 | itlb_misses.walk_duration,\ 17 | ld_blocks.store_forward,\ 18 | ld_blocks_partial.address_alias,\ 19 | uops_issued.any,\ 20 | uops_retired.core_stall_cycles,\ 21 | uops_retired.retire_slots,\ 22 | stalled-cycles-frontend -p $1 -o $2 -a sleep 30 23 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/mem.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | import numpy 3 | result = [numpy.random.bytes(1024*1024) for x in xrange(1024)] 4 | print len(result) 5 | -------------------------------------------------------------------------------- /common/Flink-scripts/tools/numa.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | numactl -N 1 -m 1 stress -m 1 --vm-bytes 128G --vm-keep & 3 | numactl -N 2 -m 2 stress -m 1 --vm-bytes 128G --vm-keep & 4 | #numactl -N 3 -m 3 stress -m 1 --vm-bytes 128G --vm-keep & 5 | -------------------------------------------------------------------------------- /common/Spark-scripts/install/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # stop Brisk.execution on error 4 | set -e 5 | 6 | # installation of Oracle Java JDK. 7 | sudo apt-get -y update 8 | sudo apt-get -y install python-software-properties 9 | sudo add-apt-repository -y ppa:webupd8team/java 10 | sudo apt-get -y update 11 | sudo apt-get -y install oracle-java7-installer 12 | 13 | # Installation of commonly used python scipy tools 14 | sudo apt-get -y install python-numpy python-scipy python-matplotlib ipython ipython-notebook python-pandas python-sympy python-nose 15 | 16 | # Installation of scala 17 | wget http://www.scala-lang.org/files/archive/scala-2.11.1.deb 18 | sudo dpkg -i scala-2.11.1.deb 19 | sudo apt-get -y update 20 | sudo apt-get -y install scala 21 | 22 | # Installation of sbt 23 | wget http://scalasbt.artifactoryonline.com/scalasbt/sbt-native-packages/org/scala-sbt/sbt//0.12.3/sbt.deb 24 | sudo dpkg -i sbt.deb 25 | sudo apt-get -y update 26 | sudo apt-get -y install sbt 27 | 28 | # Downloading spark 29 | wget http://apache.websitebeheerjd.nl/spark/spark-1.6.0/spark-1.6.0.tgz 30 | tar -zxf spark-1.6.0.tgz 31 | cd spark-1.6.0 32 | 33 | # Building spark 34 | ./sbt/sbt assembly 35 | 36 | # Clean-up 37 | rm scala-2.11.1.deb 38 | rm sbt.deb 39 | rm spark-1.0.0.tgz 40 | rm install.sh -------------------------------------------------------------------------------- /common/Spark-scripts/passwordless.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 1. ssh-keygen -t rsa 3 | Press enter for each line 4 | 2. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys 5 | 3. chmod og-wx ~/.ssh/authorized_keys 6 | 4. chmod 750 $HOME -------------------------------------------------------------------------------- /common/Spark-scripts/run_app_spark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | run_app_spark_IC.sh $1 3 | run_app_spark_OC.sh $1 4 | run_app_spark_window.sh $1 5 | run_app_spark_ss.sh $1 6 | -------------------------------------------------------------------------------- /common/Spark-scripts/start-spark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SPARK_HOME=$HOME/Documents/spark-2.0.0-bin-hadoop2.7 3 | $SPARK_HOME/sbin/stop-all.sh 4 | 5 | config=$SPARK_HOME/conf/spark-defaults.conf 6 | exe=$SPARK_HOME/sbin/start-slave.sh 7 | $SPARK_HOME/sbin/start-all.sh 8 | sleep 15 9 | 10 | -------------------------------------------------------------------------------- /common/Storm-scripts/StormUIAPI/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-executionNode.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.executionNode 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | -------------------------------------------------------------------------------- /common/Storm-scripts/StormUIAPI/json.txt: -------------------------------------------------------------------------------- 1 | {u'topologies': [{u'status': u'ACTIVE', u'uptime': u'50s', u'requestedTotalMem': 0.0, u'name': u'voipstream--1038967807', u'uptimeSeconds': 50, u'assignedMemOnHeap': 409856.0, u'assignedCpu': 0.0, u'workersTotal': 4, u'executorsTotal': 43, u'requestedMemOffHeap': 0.0, u'schedulerInfo': None, u'encodedId': u'voipstream--1038967807-7-1467516371', u'assignedTotalMem': 409856.0, u'assignedMemOffHeap': 0.0, u'owner': u'storm', u'requestedMemOnHeap': 0.0, u'requestedCpu': 0.0, u'id': u'voipstream--1038967807-7-1467516371', u'tasksTotal': 43, u'replicationCount': 1}], u'schedulerDisplayResource': False} 2 | -------------------------------------------------------------------------------- /common/Storm-scripts/StormUIAPI/query.py: -------------------------------------------------------------------------------- 1 | import stormuiapi 2 | import sys 3 | 4 | storm = stormuiapi.StormUIAPI(stormhost="localhost", stormport=8080) 5 | #print storm.getClusterConfiguration() 6 | print storm.getTopologyByName(sys.argv[1]) 7 | print sys.argv[1] 8 | -------------------------------------------------------------------------------- /common/Storm-scripts/StormUIAPI/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.7.0 2 | -------------------------------------------------------------------------------- /common/Storm-scripts/bin/Start-Storm.cmd: -------------------------------------------------------------------------------- 1 | storm nimbus && storm supervisor && storm ui -------------------------------------------------------------------------------- /common/Storm-scripts/bin/start-nimbus.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -fr ~/storm-local/* 3 | rm ~/Documents/apache-storm-1.0.0/logs/* 4 | numactl --cpunodebind=0 --membind=0 ~/Documents/zookeeper-3.4.8/bin/zkServer.sh start 5 | numactl --cpunodebind=0 --membind=0 ~/Documents/apache-storm-1.0.0/bin/storm nimbus 6 | -------------------------------------------------------------------------------- /common/Storm-scripts/bin/start-supervisor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #numactl --cpunodebind=1 --membind=1 ~/Documents/apache-compatibility-1.0.0/bin/compatibility supervisor 3 | ~/Documents/apache-storm-1.0.0/bin/storm supervisor 4 | -------------------------------------------------------------------------------- /common/Storm-scripts/bin/start-ui.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | numactl --cpunodebind=0 --membind=0 ~/Documents/apache-storm-1.0.0/bin/storm ui 3 | -------------------------------------------------------------------------------- /common/Storm-scripts/conf/storm_env.ini: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | 20 | # Environment variables in the following section will be used 21 | # in storm python script. They override the environment variables 22 | # set in the shell. 23 | [environment] 24 | 25 | # The java implementation to use. If JAVA_HOME is not found we expect java to be in path 26 | #JAVA_HOME:home 27 | 28 | # JVM options to be used in "storm jar" commad 29 | #STORM_JAR_JVM_OPTS: 30 | 31 | -------------------------------------------------------------------------------- /common/Storm-scripts/run_app_storm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | run_app_storm_IC.sh $1 3 | run_app_storm_OC.sh $1 4 | run_app_storm_window.sh $1 5 | run_app_storm_ss.sh $1 -------------------------------------------------------------------------------- /common/Storm-scripts/run_storm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #./run_app_storm_ts.sh $1 3 | #./run_app_storm_configureTasks.sh $1 4 | run_app_storm_IC.sh $1 5 | run_app_storm_OC.sh $1 6 | run_app_storm_window.sh $1 7 | run_app_storm_ss.sh $1 8 | -------------------------------------------------------------------------------- /common/Storm-scripts/start-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | start-nimbus.sh & 3 | start-ui.sh & 4 | start-supervisor.sh & 5 | -------------------------------------------------------------------------------- /common/Storm-scripts/start-nimbus.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf ~/storm-local/* 3 | rm -rf logs/* 4 | numactl --localalloc -N 1 ~/Documents/zookeeper-3.5.0-alpha/bin/zkServer.sh start 5 | numactl --localalloc -N 1 ~/Documents/apache-storm-1.1.1/bin/storm nimbus 6 | -------------------------------------------------------------------------------- /common/Storm-scripts/start-supervisor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | numactl --localalloc -N 1 ~/Documents/apache-storm-1.1.1/bin/storm supervisor 3 | -------------------------------------------------------------------------------- /common/Storm-scripts/start-ui.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ~/Documents/apache-storm-1.1.1/bin/storm ui 3 | -------------------------------------------------------------------------------- /common/Storm-scripts/stop_storm.sh: -------------------------------------------------------------------------------- 1 | killall -9 java 2 | -------------------------------------------------------------------------------- /common/common-scripts/kill_stormTopology.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | JAR_PATH="$HOME/NUMA-streamBenchmarks/common/target/common-1.0-SNAPSHOT-jar-with-dependencies.jar" 3 | MAIN="applications.utils" 4 | java -jar $JAR_PATH $MAIN 2 5 | 6 | -------------------------------------------------------------------------------- /common/common-scripts/ocprofile_PID.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ocperf.py stat -e \ 3 | cpu_clk_unhalted.thread_p_any,\ 4 | offcore_response.all_demand_mlc_pref_reads.llc_miss.any_response,\ 5 | offcore_response.all_demand_mlc_pref_reads.llc_miss.local_dram,\ 6 | offcore_response.all_demand_mlc_pref_reads.llc_miss.remote_hitm_hit_forward,\ 7 | idq_uops_not_delivered.core,\ 8 | dtlb_load_misses.stlb_hit,\ 9 | dtlb_load_misses.miss_causes_a_walk,\ 10 | icache.misses,\ 11 | mem_load_uops_retired.l2_hit,\ 12 | mem_load_uops_retired.llc_hit,\ 13 | int_misc.recovery_cycles,\ 14 | itlb_misses.stlb_hit,\ 15 | itlb_misses.miss_causes_a_walk,\ 16 | uops_retired.all,\ 17 | uops_retired.retire_slots,\ 18 | uops_issued.any -o $1 -a -p $2 -D 10000 -S 19 | -------------------------------------------------------------------------------- /common/common-scripts/passwordless.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 1. ssh-keygen -t rsa 3 | Press enter for each line 4 | 2. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys 5 | 3. chmod og-wx ~/.ssh/authorized_keys 6 | 4. chmod 750 $HOME 7 | 8 | 9 | 10 | 11 | putty: 12 | 13 | Ok, it is fixed however I don't see how this is different from what I tried already. 14 | 15 | What I did: 16 | 17 | generate a key pair with puttygen.exe (length: 1024 bits) 18 | load the private key in the PuTTY profile 19 | enter the public key in ~/.ssh/authorized_keys in one line (needs to start with ssh-rsa) 20 | chmod 700 ~/.ssh 21 | chmod 600 ~/.ssh/authorized_keys 22 | chown $USER:$USER ~/.ssh -R 23 | change /etc/ssh/sshd_config so it contains AuthorizedKeysFile %h/.ssh/authorized_keys 24 | sudo service ssh restart 25 | For troubleshooting do # tail -f /var/log/auth.log. 26 | 27 | Thanks for your help! -------------------------------------------------------------------------------- /common/common-scripts/run_kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | KAFKA_HOME=~/kafka/ 3 | $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties 4 | -------------------------------------------------------------------------------- /common/common-scripts/run_producer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | JAR_PATH="$HOME/NUMA-streamBenchmarks/common/target/common-1.0-SNAPSHOT-jar-with-dependencies.jar" 3 | MAIN="applications.utils" 4 | KAFKA_HOME="$HOME/kafka" 5 | cores="$(getconf _NPROCESSORS_ONLN)" 6 | #Only spark need multiple partitions... 7 | $KAFKA_HOME/bin/kafka-topics.sh --create --topic "mb" --replication 1 --zookeeper localhost:2181 --partitions 1 8 | #'function', 'tuple_size', 'skew', 'do not test', `do not verbose`. 9 | java -jar $JAR_PATH $MAIN 1 $1 $2 false $3 10 | -------------------------------------------------------------------------------- /common/common-scripts/run_producer_default.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #`do not verbose` 3 | run_producer.sh 8 0 false 4 | -------------------------------------------------------------------------------- /common/common-scripts/start_all.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | rm -rf ~/kafka-logs/* 3 | rm -rf ~/zookeeper/* 4 | ~/Documents/zookeeper-3.5.2-alpha/bin/zkServer.sh start 5 | $HOME/kafka/bin/kafka-topics.sh --delete --topic "mb" --zookeeper localhost:2181 6 | run_kafka.sh 7 | -------------------------------------------------------------------------------- /common/common-scripts/stop_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | killall -9 java 3 | -------------------------------------------------------------------------------- /common/lib/classmexer.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xtra-Computing/briskstream/21220007589bd5352b6a8d9053a4b5ba2921b927/common/lib/classmexer.jar -------------------------------------------------------------------------------- /common/lib/flink-storm_2.11-1.5-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xtra-Computing/briskstream/21220007589bd5352b6a8d9053a4b5ba2921b927/common/lib/flink-storm_2.11-1.5-SNAPSHOT.jar -------------------------------------------------------------------------------- /common/lib/overseer.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xtra-Computing/briskstream/21220007589bd5352b6a8d9053a4b5ba2921b927/common/lib/overseer.jar -------------------------------------------------------------------------------- /common/src/main/java/applications/CONTROL.java: -------------------------------------------------------------------------------- 1 | package applications; 2 | 3 | public interface CONTROL { 4 | 5 | //application related. 6 | int NUM_EVENTS = 1_000_000; //500_000 : 1_000_000; //1_000_000 for real use cases; 7 | 8 | 9 | //latency related. 10 | boolean enable_latency_measurement = true;// 11 | 12 | //profile related. 13 | boolean enable_profile = false;//enable this only when we want to test for breakdown. 14 | boolean enable_debug = false;//some critical debug section. 15 | 16 | 17 | } -------------------------------------------------------------------------------- /common/src/main/java/constants/ClickAnalyticsConstants.java: -------------------------------------------------------------------------------- 1 | package constants; 2 | 3 | public interface ClickAnalyticsConstants extends BaseConstants { 4 | String PREFIX = "ca"; 5 | 6 | interface Conf extends BaseConf { 7 | String REPEATS_THREADS = "ca.repeats.threads"; 8 | String GEOGRAPHY_THREADS = "ca.geography.threads"; 9 | String TOTAL_STATS_THREADS = "ca.total_stats.threads"; 10 | String GEO_STATS_THREADS = "ca.geo_stats.threads"; 11 | } 12 | 13 | interface Field { 14 | String IP = "ip"; 15 | String URL = "url"; 16 | String CLIENT_KEY = "clientKey"; 17 | String COUNTRY = "country"; 18 | String COUNTRY_NAME = "country_name"; 19 | String CITY = "city"; 20 | String UNIQUE = "unique"; 21 | String COUNTRY_TOTAL = "countryTotal"; 22 | String CITY_TOTAL = "cityTotal"; 23 | String TOTAL_COUNT = "totalCount"; 24 | String TOTAL_UNIQUE = "totalUnique"; 25 | } 26 | 27 | interface Component extends BaseComponent { 28 | String REPEATS = "repeatsBolt"; 29 | String GEOGRAPHY = "geographyBolt"; 30 | String TOTAL_STATS = "totalStats"; 31 | String GEO_STATS = "geoStats"; 32 | String SINK_VISIT = "sinkVisit"; 33 | String SINK_LOCATION = "sinkLocation"; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/java/constants/CrossTableConstants.java: -------------------------------------------------------------------------------- 1 | package constants; 2 | 3 | public interface CrossTableConstants extends BaseConstants { 4 | String PREFIX = "ct"; 5 | 6 | interface Field { 7 | String TIME = "time"; 8 | String TEXT = "text"; 9 | String STATE = "state"; 10 | String FLAG = "flag"; 11 | } 12 | 13 | interface Conf extends BaseConf { 14 | String DEG_THREADS = "deposit.generator.threads"; 15 | String TEG_THREADS = "txn.generator.threads"; 16 | String DT_THREADS = "deposit.transaction.threads"; 17 | String TT_THREADS = "txn.transaction.threads"; 18 | String CT_THREADS = "ct.transaction.threads"; 19 | } 20 | 21 | interface Component extends BaseComponent { 22 | String DEG = "depositGenerator"; 23 | String TEG = "txnGenerator"; 24 | 25 | String DT = "depositTxn"; 26 | String TT = "transferTxn"; 27 | 28 | String CT = "CTBolt"; 29 | } 30 | 31 | interface Constant { 32 | 33 | int NUM_ACCOUNTS = 100_000; 34 | int NUM_BOOK_ENTRIES = 100_000; 35 | String ACCOUNT_ID_PREFIX = "";//ACCT- 36 | String BOOK_ENTRY_ID_PREFIX = "";//BOOK- 37 | long MAX_ACCOUNT_TRANSFER = 10_000; 38 | long MAX_BOOK_TRANSFER = 1_000; 39 | long MIN_BALANCE = 0; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /common/src/main/java/constants/LeaderboardConstants.java: -------------------------------------------------------------------------------- 1 | package constants; 2 | 3 | public interface LeaderboardConstants { 4 | String PREFIX = "lb"; 5 | int max_hz = 450000; 6 | 7 | interface Field extends BaseConstants.BaseField { 8 | String WORD = "word"; 9 | String COUNT = "count"; 10 | String LargeData = "LD"; 11 | } 12 | 13 | interface Conf extends BaseConstants.BaseConf { 14 | String VOTER_THREADS = "lb.voter.threads"; 15 | String MAINTAINER_THREADS = "lb.maintainer.threads"; 16 | String DELETER_THREADS = "lb.deleter.threads"; 17 | String WINDOW_THREADS = "lb.window.threads"; 18 | String WINDOW_TRIGGER_THREADS = "lb.window_tigger.threads"; 19 | String Leaderboard_THREADS = "lb.Leaderboard.threads"; 20 | } 21 | 22 | interface Component extends BaseConstants.BaseComponent { 23 | String VOTER = "voter"; 24 | String MAINTAINER = "maintainer"; 25 | String WINDOW = "window"; 26 | String WINDOW_TRIGGER = "trigger"; 27 | String DELETER = "deleter"; 28 | String Leaderboard = "Leaderboard"; 29 | } 30 | 31 | 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/constants/MicroBenchmarkConstants.java: -------------------------------------------------------------------------------- 1 | package constants; 2 | 3 | public interface MicroBenchmarkConstants extends BaseConstants { 4 | String PREFIX = "mb"; 5 | 6 | interface Field { 7 | String TIME = "time"; 8 | String TEXT = "text"; 9 | String STATE = "state"; 10 | String FLAG = "flag"; 11 | } 12 | 13 | interface Conf extends BaseConf { 14 | String INSERTOR_THREADS = "mb.insert.threads"; 15 | String SELECTOR_THREADS = "mb.selector.threads"; 16 | String Executor_Threads = "mb.executor.threads"; 17 | } 18 | 19 | interface Component extends BaseComponent { 20 | String SELECTOR = "selector"; 21 | String INSERTOR = "insertor"; 22 | String SEQUNCER = "sequencer"; 23 | String EXECUTOR = "executor"; 24 | } 25 | 26 | interface Constant { 27 | int FREQUENCY_MICRO = 100; 28 | int VALUE_LEN = 32;// 32 bytes --> one cache line. 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/constants/OnlineBidingSystemConstants.java: -------------------------------------------------------------------------------- 1 | package constants; 2 | 3 | public interface OnlineBidingSystemConstants extends BaseConstants { 4 | String PREFIX = "ob"; 5 | 6 | interface Field { 7 | String TIME = "time"; 8 | String TEXT = "text"; 9 | String STATE = "state"; 10 | String FLAG = "flag"; 11 | } 12 | 13 | interface Conf extends BaseConf { 14 | String OB_THREADS = "ob.transaction.threads"; 15 | } 16 | 17 | interface Component extends BaseComponent { 18 | String OB = "online_biding"; 19 | } 20 | 21 | interface Constant { 22 | 23 | int NUM_ACCESSES_PER_BUY = 1;// each time only bid one item.. 24 | int NUM_ACCESSES_PER_TOP = 20;// each time top up 20 items. 25 | int NUM_ACCESSES_PER_ALERT = 20;// each time alert 20 items. 26 | 27 | long MAX_BUY_Transfer = 20; 28 | long MAX_TOP_UP = 20; 29 | int MAX_Price = 100; 30 | 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/constants/PositionKeepingConstants.java: -------------------------------------------------------------------------------- 1 | package constants; 2 | 3 | public interface PositionKeepingConstants extends BaseConstants { 4 | String PREFIX = "pk"; 5 | 6 | interface Field { 7 | String DEVICE_ID = "device_Id"; 8 | } 9 | 10 | interface Conf extends BaseConf { 11 | String PK_THREADS = "pk.transaction.threads"; 12 | } 13 | 14 | interface Component extends BaseComponent { 15 | String PK = "PKBolt"; 16 | } 17 | 18 | interface Constant { 19 | int SIZE_EVENT = 30; 20 | int SIZE_VALUE = 50;//vary its computing complexity. 21 | int NUM_MACHINES = 40; 22 | int MOVING_AVERAGE_WINDOW = 1_000; 23 | double SpikeThreshold = 0.3; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /common/src/main/java/constants/streamingAnalysisConstants.java: -------------------------------------------------------------------------------- 1 | package constants; 2 | 3 | public interface streamingAnalysisConstants extends BaseConstants { 4 | String PREFIX = "sa"; 5 | int max_hz = 120000; 6 | 7 | interface Field extends BaseField { 8 | String TIME = "time"; 9 | String KEY = "key"; 10 | String VALUE = "value"; 11 | 12 | } 13 | 14 | interface Conf extends BaseConf { 15 | String EXECUTOR_THREADS1 = "threads1"; 16 | String EXECUTOR_THREADS2 = "threads2"; 17 | String EXECUTOR_THREADS3 = "threads3"; 18 | String EXECUTOR_THREADS4 = "threads4"; 19 | } 20 | 21 | interface Component extends BaseComponent { 22 | String FILTER = "filter"; 23 | String MEDIAN = "median"; 24 | String RANK = "rank"; 25 | 26 | String MEDIAN2 = "median2"; 27 | String RANK2 = "rank2"; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/helper/SimplePartitioner.java: -------------------------------------------------------------------------------- 1 | package helper; 2 | 3 | import kafka.producer.Partitioner; 4 | import kafka.utils.VerifiableProperties; 5 | 6 | public class SimplePartitioner implements Partitioner { 7 | public SimplePartitioner(VerifiableProperties props) { 8 | } 9 | 10 | public int partition(Object key, int a_numPartitions) { 11 | // int partition = 0; 12 | // String stringKey = (String) key; 13 | // int offset = stringKey.lastIndexOf('.'); 14 | // if (offset > 0) { 15 | // partition = Integer.parseInt( stringKey.substring(offset+1)) % a_numPartitions; 16 | // } 17 | final int i = Integer.parseInt((String) key) % a_numPartitions; 18 | //LOG.info(String.valueOf(i)); 19 | return i; 20 | } 21 | 22 | } -------------------------------------------------------------------------------- /common/src/main/java/helper/parser/Parser.java: -------------------------------------------------------------------------------- 1 | package helper.parser; 2 | 3 | //import applications.utils.Configuration; 4 | 5 | import util.Configuration; 6 | import util.datatypes.StreamValues; 7 | 8 | import java.io.Serializable; 9 | import java.util.List; 10 | 11 | public abstract class Parser implements Serializable { 12 | private static final long serialVersionUID = -1221926672447206098L; 13 | protected Configuration config; 14 | 15 | public void initialize(Configuration config) { 16 | this.config = config; 17 | } 18 | 19 | public abstract T parse(char[] str); 20 | 21 | public abstract List parse(String value); 22 | 23 | 24 | //public abstract List parse(String[] input); 25 | } -------------------------------------------------------------------------------- /common/src/main/java/helper/sink_helper.java: -------------------------------------------------------------------------------- 1 | package helper; 2 | 3 | import org.slf4j.Logger; 4 | 5 | /** 6 | * Created by I309939 on 7/23/2016. 7 | */ 8 | public class sink_helper extends helper { 9 | long warm_start = 0, warm_end = 0; 10 | // private int size; 11 | 12 | public sink_helper(Logger LOG, int runtime, String metric_path, double predict, int size, int thisTaskId) { 13 | super(runtime, predict, size, metric_path, thisTaskId, false); 14 | super.size = size; 15 | warm_start = System.nanoTime(); 16 | need_warm_up = false; 17 | } 18 | 19 | public double execute(long bid) { 20 | // this.size = size; 21 | return execute("default", bid); 22 | } 23 | 24 | public double execute(String sourceComponent, long bid) { 25 | StartMeasurement(); 26 | return Measurement(sourceComponent, bid); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /common/src/main/java/helper/wrapper/StringStatesWrapper.java: -------------------------------------------------------------------------------- 1 | package helper.wrapper; 2 | 3 | import helper.wrapper.basic.StateWrapper; 4 | 5 | public class StringStatesWrapper extends StateWrapper { 6 | 7 | 8 | private static final long serialVersionUID = -6970166503629636382L; 9 | 10 | public StringStatesWrapper(boolean verbose, int size) { 11 | super(verbose, size); 12 | } 13 | 14 | public StringStatesWrapper(int tuple_size) { 15 | super(tuple_size); 16 | } 17 | } -------------------------------------------------------------------------------- /common/src/main/java/machine/RTM_Machine.java: -------------------------------------------------------------------------------- 1 | package machine; 2 | 3 | public class RTM_Machine extends Platform { 4 | 5 | 6 | public RTM_Machine() { 7 | this.latency_L2 = 11.2;//measured latency in ns for each cache line sized tuple access. 8 | this.latency_LLC = 50;//measured latency in ns for each cache line sized tuple access. 9 | this.latency_LOCAL_MEM = 110;//measured latency in ns for each cache line access. 10 | this.cache_line = 64.0;//bytes 11 | this.CLOCK_RATE = 1.9;//GHz 12 | this.num_socket = 4; 13 | this.CoresPerSocket = 10;//cores per socket 14 | this.num_cores = 40; 15 | bandwidth_map = new double[][]{ //in MB/s 16 | 17 | }; 18 | 19 | latency_map = new double[][]{// in ns 20 | 21 | }; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /common/src/main/java/model/finance/IndicatorParameter.java: -------------------------------------------------------------------------------- 1 | package model.finance; 2 | 3 | import org.joda.time.DateTime; 4 | 5 | /** 6 | * User: warrenhenning Date: 9/20/12 Time: 4:11 AM 7 | */ 8 | public class IndicatorParameter { 9 | 10 | private DateTime time; 11 | private int period; 12 | 13 | public IndicatorParameter(DateTime time, int period) { 14 | this.time = time; 15 | this.period = period; 16 | } 17 | 18 | public DateTime getTime() { 19 | return time; 20 | } 21 | 22 | public int getPeriod() { 23 | return period; 24 | } 25 | 26 | @Override 27 | public boolean equals(Object o) { 28 | if (this == o) { 29 | return true; 30 | } 31 | if (o == null || getClass() != o.getClass()) { 32 | return false; 33 | } 34 | 35 | IndicatorParameter that = (IndicatorParameter) o; 36 | 37 | return period == that.period && time.equals(that.time); 38 | } 39 | 40 | @Override 41 | public int hashCode() { 42 | int result = time.hashCode(); 43 | result = 31 * result + period; 44 | return result; 45 | } 46 | } -------------------------------------------------------------------------------- /common/src/main/java/model/geoip/IPLocation.java: -------------------------------------------------------------------------------- 1 | package model.geoip; 2 | 3 | /** 4 | * @author mayconbordin 5 | */ 6 | public interface IPLocation { 7 | Location resolve(String ip); 8 | } 9 | -------------------------------------------------------------------------------- /common/src/main/java/model/geoip/IPLocationFactory.java: -------------------------------------------------------------------------------- 1 | package model.geoip; 2 | 3 | 4 | import constants.BaseConstants; 5 | import util.Configuration; 6 | import util.OsUtils; 7 | 8 | public class IPLocationFactory { 9 | public static final String GEOIP2 = "geoip2"; 10 | 11 | public static IPLocation create(String name, Configuration config) { 12 | if (name.equals(GEOIP2)) { 13 | String OS_prefix = null; 14 | if (OsUtils.isWindows()) { 15 | OS_prefix = "win."; 16 | } else { 17 | OS_prefix = "unix."; 18 | } 19 | return new GeoIP2Location(config.getString(OS_prefix.concat(BaseConstants.BaseConf.GEOIP2_DB))); 20 | } else { 21 | throw new IllegalArgumentException(name + " is not a valid IP locator name"); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/model/geoip/Location.java: -------------------------------------------------------------------------------- 1 | package model.geoip; 2 | 3 | /** 4 | * @author mayconbordin 5 | */ 6 | public class Location { 7 | private String countryName; 8 | private String countryCode; 9 | private String city; 10 | private String ip; 11 | 12 | public String getCountryName() { 13 | return countryName; 14 | } 15 | 16 | public void setCountryName(String countryName) { 17 | this.countryName = countryName; 18 | } 19 | 20 | public String getCountryCode() { 21 | return countryCode; 22 | } 23 | 24 | public void setCountryCode(String countryCode) { 25 | this.countryCode = countryCode; 26 | } 27 | 28 | public String getCity() { 29 | return city; 30 | } 31 | 32 | public void setCity(String city) { 33 | this.city = city; 34 | } 35 | 36 | public String getIp() { 37 | return ip; 38 | } 39 | 40 | public void setIp(String ip) { 41 | this.ip = ip; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return "Location{" + "countryName=" + countryName + ", countryCode=" + countryCode + ", city=" + city + ", ip=" + ip + '}'; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /common/src/main/java/model/gis/GPSRecord.java: -------------------------------------------------------------------------------- 1 | package model.gis; 2 | 3 | public class GPSRecord extends Point { 4 | private int vel; 5 | private int bearing; 6 | 7 | public GPSRecord() { 8 | } 9 | 10 | public GPSRecord(double x, double y, int vel, int bearing) { 11 | this.x = x; 12 | this.y = y; 13 | this.vel = vel; 14 | this.bearing = bearing; 15 | } 16 | 17 | public int getVel() { 18 | return vel; 19 | } 20 | 21 | public int getBearing() { 22 | return bearing; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/model/gis/Point.java: -------------------------------------------------------------------------------- 1 | package model.gis; 2 | 3 | public class Point { 4 | protected double x; 5 | protected double y; 6 | 7 | public Point() { 8 | } 9 | 10 | public Point(double x, double y) { 11 | this.x = x; 12 | this.y = y; 13 | } 14 | 15 | public double getX() { 16 | return x; 17 | } 18 | 19 | public double getY() { 20 | return y; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /common/src/main/java/model/gis/Road.java: -------------------------------------------------------------------------------- 1 | package model.gis; 2 | 3 | 4 | import util.collections.FixedSizeQueue; 5 | 6 | public class Road { 7 | private final int roadID; 8 | private final FixedSizeQueue roadSpeed; 9 | private int averageSpeed; 10 | private int count; 11 | 12 | public Road(int roadID) { 13 | this.roadID = roadID; 14 | this.roadSpeed = new FixedSizeQueue<>(30); 15 | } 16 | 17 | public int getRoadID() { 18 | return roadID; 19 | } 20 | 21 | public int getAverageSpeed() { 22 | return averageSpeed; 23 | } 24 | 25 | public void setAverageSpeed(int averageSpeed) { 26 | this.averageSpeed = averageSpeed; 27 | } 28 | 29 | public int getCount() { 30 | return count; 31 | } 32 | 33 | public void setCount(int count) { 34 | this.count = count; 35 | } 36 | 37 | public void incrementCount() { 38 | this.count++; 39 | } 40 | 41 | public FixedSizeQueue getRoadSpeed() { 42 | return roadSpeed; 43 | } 44 | 45 | public boolean addRoadSpeed(int speed) { 46 | return roadSpeed.add(speed); 47 | } 48 | 49 | public int getRoadSpeedSize() { 50 | return roadSpeed.size(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /common/src/main/java/model/gis/Sect.java: -------------------------------------------------------------------------------- 1 | package model.gis; 2 | 3 | import java.util.ArrayList; 4 | 5 | public class Sect extends Polygon { 6 | private int id; 7 | private int roadWidth; 8 | private String mapID; 9 | 10 | public Sect(ArrayList points, int id) { 11 | super(points); 12 | this.id = id; 13 | } 14 | 15 | public Sect(ArrayList points, int id, int roadWidth, String mapID) { 16 | super(points); 17 | this.id = id; 18 | this.roadWidth = roadWidth; 19 | this.mapID = mapID; 20 | } 21 | 22 | public int getID() { 23 | return this.id; 24 | } 25 | 26 | public int getroadWidth() { 27 | return this.roadWidth; 28 | } 29 | 30 | 31 | public String getMapID() { 32 | return this.mapID; 33 | } 34 | } -------------------------------------------------------------------------------- /common/src/main/java/model/log/NotificationDetails.java: -------------------------------------------------------------------------------- 1 | package model.log; 2 | 3 | public class NotificationDetails { 4 | private String to; 5 | private Severity severity; 6 | private String message; 7 | 8 | public NotificationDetails(String to, Severity severity, String message) { 9 | this.to = to; 10 | this.severity = severity; 11 | this.message = message; 12 | } 13 | 14 | public String getTo() { 15 | return to; 16 | } 17 | 18 | public Severity getSeverity() { 19 | return severity; 20 | } 21 | 22 | public String getMessage() { 23 | return message; 24 | } 25 | } -------------------------------------------------------------------------------- /common/src/main/java/model/log/Severity.java: -------------------------------------------------------------------------------- 1 | package model.log; 2 | 3 | public enum Severity { 4 | LOW, MEDIUM, HIGH 5 | } -------------------------------------------------------------------------------- /common/src/main/java/model/predictor/IMarkovModelSource.java: -------------------------------------------------------------------------------- 1 | package model.predictor; 2 | 3 | /** 4 | * @author maycon 5 | */ 6 | public interface IMarkovModelSource { 7 | String getModel(String key); 8 | } 9 | -------------------------------------------------------------------------------- /common/src/main/java/model/predictor/MarkovModelFileSource.java: -------------------------------------------------------------------------------- 1 | package model.predictor; 2 | 3 | import java.io.IOException; 4 | import java.nio.ByteBuffer; 5 | import java.nio.charset.Charset; 6 | import java.nio.file.Files; 7 | import java.nio.file.Paths; 8 | 9 | /** 10 | * @author maycon 11 | */ 12 | public class MarkovModelFileSource implements IMarkovModelSource { 13 | private Charset charset; 14 | 15 | public MarkovModelFileSource() { 16 | charset = Charset.defaultCharset(); 17 | } 18 | 19 | @Override 20 | public String getModel(String key) { 21 | byte[] encoded; 22 | try { 23 | encoded = Files.readAllBytes(Paths.get(System.getProperty("user.home").concat("/data/app/").concat(key))); 24 | return charset.decode(ByteBuffer.wrap(encoded)).toString(); 25 | } catch (IOException ex) { 26 | return null; 27 | } 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/model/predictor/MarkovModelResourceSource.java: -------------------------------------------------------------------------------- 1 | package model.predictor; 2 | 3 | import com.google.common.io.Resources; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.io.IOException; 8 | import java.net.URL; 9 | import java.nio.charset.Charset; 10 | 11 | /** 12 | * @author mayconbordin 13 | */ 14 | public class MarkovModelResourceSource implements IMarkovModelSource { 15 | private static final Logger LOG = LoggerFactory.getLogger(MarkovModelResourceSource.class); 16 | private Charset charset; 17 | 18 | public MarkovModelResourceSource() { 19 | charset = Charset.defaultCharset(); 20 | } 21 | 22 | @Override 23 | public String getModel(String key) { 24 | try { 25 | URL url = Resources.getResource(key); 26 | return Resources.toString(url, charset); 27 | } catch (IOException ex) { 28 | LOG.error("Unable to load markov model from resource " + key, ex); 29 | return null; 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/model/predictor/ModelBasedPredictor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * beymani: Outlier and anamoly detection 3 | * Author: Pranab Ghosh 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you 6 | * may not use this file except in compliance with the License. You may 7 | * obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | * implied. See the License for the specific language governing 15 | * permissions and limitations under the License. 16 | */ 17 | 18 | package model.predictor; 19 | 20 | /** 21 | * Base class for all model based predictors 22 | * 23 | * @author pranab 24 | */ 25 | public abstract class ModelBasedPredictor { 26 | public abstract Prediction execute(char[] entityID, char[] record); 27 | 28 | public abstract Prediction execute(String entityID, String record); 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/model/predictor/Prediction.java: -------------------------------------------------------------------------------- 1 | package model.predictor; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Prediction implements Serializable { 6 | private static final long serialVersionUID = -669815404243309408L; 7 | private char[] entityId; 8 | private double score; 9 | private String[] states; 10 | private boolean outlier; 11 | 12 | public Prediction(char[] entityId, double score, String[] states, boolean outlier) { 13 | this.entityId = entityId; 14 | this.score = score; 15 | this.states = states; 16 | this.outlier = outlier; 17 | } 18 | 19 | public char[] getEntityId() { 20 | return entityId; 21 | } 22 | 23 | public void setEntityId(char[] entityId) { 24 | this.entityId = entityId; 25 | } 26 | 27 | public double getScore() { 28 | return score; 29 | } 30 | 31 | public void setScore(double score) { 32 | this.score = score; 33 | } 34 | 35 | public String[] getStates() { 36 | return states; 37 | } 38 | 39 | public void setStates(String[] states) { 40 | this.states = states; 41 | } 42 | 43 | public boolean isOutlier() { 44 | return outlier; 45 | } 46 | 47 | public void setOutlier(boolean outlier) { 48 | this.outlier = outlier; 49 | } 50 | } -------------------------------------------------------------------------------- /common/src/main/java/parser/LBParser.java: -------------------------------------------------------------------------------- 1 | package parser; 2 | 3 | import helper.parser.Parser; 4 | import util.datatypes.StreamValues; 5 | 6 | import java.util.Arrays; 7 | 8 | /** 9 | * @author Maycon Viana Bordin 10 | */ 11 | public class LBParser extends Parser { 12 | private static final long serialVersionUID = 5531330878367909877L; 13 | // private static final Logger LOG = LoggerFactory.getLogger(StringParser.class); 14 | 15 | @Override 16 | public Object[] parse(char[] str) { 17 | if (str.length <= 0) { 18 | return null; 19 | } 20 | 21 | return new char[][]{ 22 | // new String(new_copy) 23 | // sb.toString() 24 | Arrays.copyOf(str, str.length) 25 | };//str+"" 26 | } 27 | 28 | @Override 29 | public StreamValues parse(String value) { 30 | return null; 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /common/src/main/java/parser/SAParser.java: -------------------------------------------------------------------------------- 1 | package parser; 2 | 3 | import helper.parser.Parser; 4 | import util.datatypes.StreamValues; 5 | 6 | /** 7 | * Created by tony on 6/20/2017. 8 | */ 9 | public class SAParser extends Parser { 10 | public static final String split_expression = "\t"; 11 | private static final long serialVersionUID = 7448797732626243378L; 12 | 13 | @Override 14 | public Object[] parse(char[] input) { 15 | String[] split = new String(input).split(split_expression); 16 | // return ImmutableList.of(new StreamValues(Long.valueOf(split[0]), split[1], split[2]));//time, key, value. 17 | return new Object[]{Long.valueOf(split[0]), split[1].toCharArray(), split[2].toCharArray()};//time, key, value. 18 | } 19 | 20 | @Override 21 | public StreamValues parse(String value) { 22 | return null; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/tasks/stateful_task.java: -------------------------------------------------------------------------------- 1 | package tasks; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by I309939 on 7/30/2016. 7 | */ 8 | public abstract class stateful_task implements Serializable { 9 | private static final long serialVersionUID = 10L; 10 | 11 | public abstract int execute(Long time, String key, String value); 12 | 13 | public abstract String execute(Long time, String key, String value, long stateful_process_start); 14 | } 15 | -------------------------------------------------------------------------------- /common/src/main/java/tools/B_object.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | import java.util.ArrayList; 4 | 5 | /** 6 | * Created by tony on 6/5/2017. 7 | */ 8 | public class B_object extends object { 9 | private static final long serialVersionUID = 2225245845078346454L; 10 | // byte[] value; 11 | byte value; 12 | 13 | public B_object() { 14 | value = 0; 15 | //new byte[1];//1 byte per object 16 | //r.nextBytes(value); 17 | } 18 | 19 | @Override 20 | public ArrayList create_myObjectList(int size_state) { 21 | ArrayList myOjbectList = new ArrayList<>(); 22 | for (int i = 0; i < size_state; i++) { 23 | myOjbectList.add(new B_object()); 24 | } 25 | return myOjbectList; 26 | 27 | } 28 | 29 | @Override 30 | public Object getValue() { 31 | return value; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/tools/FastZipfGenerator.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | import java.util.NavigableMap; 4 | import java.util.Random; 5 | import java.util.TreeMap; 6 | 7 | public class FastZipfGenerator { 8 | 9 | private Random random = new Random(0); 10 | private NavigableMap map = new TreeMap<>(); 11 | 12 | 13 | public FastZipfGenerator(int size, double skew, int offset) { 14 | 15 | 16 | computeMap(size, skew, offset); 17 | } 18 | 19 | private void computeMap( 20 | int size, double skew, int offset) { 21 | 22 | double div = 0; 23 | for (int i = 1; i <= size; i++) { 24 | div += (1 / Math.pow(i, skew)); 25 | } 26 | 27 | double sum = 0; 28 | for (int i = 1; i <= size; i++) { 29 | double p = (1.0d / Math.pow(i, skew)) / div; 30 | sum += p; 31 | map.put(sum, i - 1 + offset); 32 | } 33 | 34 | } 35 | 36 | public int next() { 37 | double value = random.nextDouble(); 38 | 39 | return map.ceilingEntry(value).getValue(); 40 | } 41 | 42 | public void show_sample() { 43 | for (int i = 0; i < 100; i++) { 44 | System.out.println(this.next()); 45 | } 46 | } 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /common/src/main/java/tools/KB_object.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | 4 | // (per socket):20M 5 | // * 8 bits --> 0.1M long is significantly less than L3. 6 | 7 | import java.io.Serializable; 8 | import java.util.ArrayList; 9 | import java.util.Random; 10 | 11 | /** 12 | * each myObject is 8 bytes 13 | * let's change it to 1 MB!! that is 1*1024*1024 bytes. This is too slow.. 14 | * let's change it to 1 KB that is 1*1024 bytes. 15 | */ 16 | public class KB_object implements Serializable { 17 | private static final long serialVersionUID = -9073964378669329849L; 18 | final int MB = 1024 * 1024; 19 | final int KB = 1024; 20 | 21 | byte[] value; 22 | 23 | /*public myObject(byte[] value) { 24 | this.value = value; 25 | }*/ 26 | Random r = new Random(); 27 | 28 | public KB_object() { 29 | value = new byte[KB]; 30 | r.nextBytes(value); 31 | } 32 | 33 | public static ArrayList create_myObjectList(int size_state) { 34 | ArrayList myOjbectList = new ArrayList<>(); 35 | for (int i = 0; i < size_state; i++) { 36 | myOjbectList.add(new KB_object()); 37 | } 38 | return myOjbectList; 39 | } 40 | 41 | public byte[] getValue() { 42 | return value; 43 | } 44 | 45 | public void setValue(byte[] value) { 46 | this.value = value; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /common/src/main/java/tools/KB_object_AsString.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | 4 | // (per socket):20M 5 | // * 8 bits --> 0.1M long is significantly less than L3. 6 | 7 | import java.util.ArrayList; 8 | 9 | /** 10 | * Use char as the basic element. 11 | */ 12 | public class KB_object_AsString extends object { 13 | private static final long serialVersionUID = -397143864082731725L; 14 | char[] value; 15 | 16 | public KB_object_AsString() { 17 | value = new char[KB / 2];//each char occupy two bytes in java. 18 | for (int i = 0; i < KB / 2; i++) { 19 | value[i] = alphabet.charAt(r.nextInt(N)); 20 | } 21 | } 22 | 23 | 24 | public ArrayList create_myObjectList(int size_state) { 25 | ArrayList myOjbectList = new ArrayList<>(); 26 | for (int i = 0; i < size_state; i++) { 27 | myOjbectList.add(new KB_object_AsString()); 28 | } 29 | return myOjbectList; 30 | } 31 | 32 | public String getValue() { 33 | return String.valueOf(value); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/java/tools/Rankable.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | public interface Rankable extends Comparable { 4 | 5 | Object getObject(); 6 | 7 | long getCount(); 8 | 9 | /** 10 | * Note: We do not defensively copy the object wrapped by the Rankable. It is passed as is. 11 | * 12 | * @return a defensive copy 13 | */ 14 | Rankable copy(); 15 | } -------------------------------------------------------------------------------- /common/src/main/java/tools/cacheSim/randomNumberGenerator.java: -------------------------------------------------------------------------------- 1 | package tools.cacheSim; 2 | 3 | import java.util.Random; 4 | 5 | /** 6 | * Created by I309939 on 7/29/2016. 7 | */ 8 | public class randomNumberGenerator { 9 | public static int generate(int min, int max) { 10 | final Random rn = new Random(System.nanoTime()); 11 | int result = rn.nextInt(max - min + 1) + min; 12 | //System.out.println(result); 13 | return result; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /common/src/main/java/tools/object.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.Random; 6 | 7 | /** 8 | * Created by tony on 6/5/2017. 9 | */ 10 | public abstract class object implements Serializable { 11 | private static final long serialVersionUID = 5884774060475035093L; 12 | final int MB = 1024 * 1024; 13 | final int KB = 1024; 14 | final String alphabet = "0123456789ABCDE"; 15 | final int N = alphabet.length(); 16 | 17 | Random r = new Random(); 18 | 19 | public abstract ArrayList create_myObjectList(int size_state); 20 | 21 | public abstract Object getValue(); 22 | } 23 | -------------------------------------------------------------------------------- /common/src/main/java/tools/randomNumberGenerator.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | import java.util.Random; 4 | 5 | /** 6 | * Created by I309939 on 7/29/2016. 7 | */ 8 | public class randomNumberGenerator { 9 | public static int generate(int min, int max) { 10 | final Random rn = new Random(System.nanoTime()); 11 | int result = rn.nextInt(max - min + 1) + min; 12 | //System.out.println(result); 13 | return result; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /common/src/main/java/tools/worker2.java: -------------------------------------------------------------------------------- 1 | package tools; 2 | 3 | /** 4 | * Created by szhang026 on 5/30/2016. 5 | */ 6 | public class worker2 { 7 | final Control control = new Control(); 8 | 9 | public static void main(String[] args) { 10 | try { 11 | worker2 test = new worker2(); 12 | test.test(); 13 | } catch (Exception e) { 14 | e.printStackTrace(); 15 | } 16 | } 17 | 18 | private void test() { 19 | mapMatcher main = new mapMatcher(); 20 | speedCal help = new speedCal(); 21 | 22 | new Thread(main).start(); 23 | new Thread(help).start(); 24 | } 25 | 26 | class Control { 27 | public volatile boolean flag = false; 28 | } 29 | 30 | class mapMatcher implements Runnable { 31 | @Override 32 | public void run() { 33 | while (!control.flag) { 34 | 35 | } 36 | } 37 | } 38 | 39 | class speedCal implements Runnable { 40 | @Override 41 | public void run() { 42 | while (!control.flag) { 43 | 44 | } 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /common/src/main/java/util/CacheInfo.java: -------------------------------------------------------------------------------- 1 | package util; 2 | 3 | import java.io.Serializable; 4 | import java.util.HashMap; 5 | 6 | public class CacheInfo implements Serializable { 7 | private static final long serialVersionUID = 8130687873482833491L; 8 | 9 | //1st string: executionNode.getOP() + srcNode.getOP() 10 | //2nd string: string information in files. 11 | private HashMap cachedInformation = new HashMap<>(); 12 | 13 | public synchronized void updateInfo(String key, String value) { 14 | cachedInformation.put(key, value); 15 | } 16 | 17 | public synchronized boolean isEmpty(String key) { 18 | return cachedInformation.get(key) == null; 19 | } 20 | 21 | public String Info(String key) { 22 | return cachedInformation.get(key); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/util/Constants.java: -------------------------------------------------------------------------------- 1 | package util; 2 | 3 | /** 4 | * Created by I309939 on 12/7/2016. 5 | * Updated at BriskRunner line 262 6 | */ 7 | public class Constants { 8 | public volatile static double default_sourceRate = 1000_000;//xxx K events / second 9 | } 10 | -------------------------------------------------------------------------------- /common/src/main/java/util/JavaUtils.java: -------------------------------------------------------------------------------- 1 | package util; 2 | 3 | public class JavaUtils { 4 | public static boolean isJar() { 5 | String className = JavaUtils.class.getName().replace('.', '/'); 6 | String classJar = JavaUtils.class.getResource("/" + className + ".class").toString(); 7 | 8 | return classJar.startsWith("jar:"); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /common/src/main/java/util/collections/FixedMap.java: -------------------------------------------------------------------------------- 1 | package util.collections; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.Map; 5 | 6 | public class FixedMap extends LinkedHashMap { 7 | private static final long serialVersionUID = -6542891947990885228L; 8 | private int max_capacity; 9 | 10 | public FixedMap(int initial_capacity, int max_capacity) { 11 | super(initial_capacity, 0.75f, false); 12 | this.max_capacity = max_capacity; 13 | } 14 | 15 | @Override 16 | protected boolean removeEldestEntry(Map.Entry eldest) { 17 | return size() > this.max_capacity; 18 | } 19 | 20 | } -------------------------------------------------------------------------------- /common/src/main/java/util/datatypes/DataTypeUtils.java: -------------------------------------------------------------------------------- 1 | package util.datatypes; 2 | 3 | /** 4 | * @author mayconbordin 5 | */ 6 | public class DataTypeUtils { 7 | public static boolean isInteger(String str) { 8 | if (str == null) { 9 | return false; 10 | } 11 | int length = str.length(); 12 | if (length == 0) { 13 | return false; 14 | } 15 | int i = 0; 16 | if (str.charAt(0) == '-') { 17 | if (length == 1) { 18 | return false; 19 | } 20 | i = 1; 21 | } 22 | for (; i < length; i++) { 23 | char c = str.charAt(i); 24 | if (c <= '/' || c >= ':') { 25 | return false; 26 | } 27 | } 28 | return true; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/util/datatypes/StreamValues.java: -------------------------------------------------------------------------------- 1 | package util.datatypes; 2 | 3 | 4 | import java.util.ArrayList; 5 | 6 | public class StreamValues extends ArrayList { 7 | private static final long serialVersionUID = 7498957996924776844L; 8 | private Object messageId; 9 | private String streamId = "default"; 10 | 11 | // public ImmutableList getValue() { 12 | // return value; 13 | // } 14 | 15 | // private ImmutableList value; 16 | //public E get(int idx) { 17 | // return value.get(idx); 18 | // } 19 | public StreamValues() { 20 | } 21 | 22 | public StreamValues(E... vals) { 23 | 24 | // super(vals.length); 25 | // this.addAll(ImmutableList.copyOf(vals)); 26 | // value = ImmutableList.copyOf(vals); 27 | 28 | super(vals.length); 29 | for (E o : vals) { 30 | add(o); 31 | } 32 | } 33 | 34 | public Object getMessageId() { 35 | return messageId; 36 | } 37 | 38 | public void setMessageId(Object messageId) { 39 | this.messageId = messageId; 40 | } 41 | 42 | public String getStreamId() { 43 | return streamId; 44 | } 45 | 46 | public void setStreamId(String streamId) { 47 | this.streamId = streamId; 48 | } 49 | 50 | // 51 | } 52 | -------------------------------------------------------------------------------- /common/src/main/java/util/events/AccidentEvent.java: -------------------------------------------------------------------------------- 1 | package util.events; 2 | 3 | /** 4 | * Created by szhang026 on 20/2/2016. 5 | */ 6 | public class AccidentEvent { 7 | public int vid1;//ID of the first vehicle 8 | public int vid2; 9 | public byte xway; 10 | public byte mile; 11 | public byte dir; 12 | public long time; 13 | 14 | public AccidentEvent(int vid1, int vid2, byte xway, byte mile, byte dir, long t) { 15 | this.vid1 = vid1; 16 | this.vid2 = vid2; 17 | this.xway = xway; 18 | this.mile = mile; 19 | this.dir = dir; 20 | this.time = t; 21 | } 22 | 23 | public AccidentEvent() { 24 | 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return "AccidentEvent [vid1 = " + vid1 + ", vid2 = " + vid2 + ", xway=" + xway + ", mile=" 30 | + mile + ", dir=" + dir + " time=" + time + "]"; 31 | } 32 | 33 | public String toCompressedString() { 34 | return "" + vid1 + " " + vid2 + " " + xway + " " + mile + " " + dir + " " + time; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /common/src/main/java/util/events/AccountBalanceEvent.java: -------------------------------------------------------------------------------- 1 | package util.events; 2 | 3 | /** 4 | * @author miyuru 5 | */ 6 | public class AccountBalanceEvent { 7 | 8 | public long time; //A timestamp measured in seconds since the start of the simulation 9 | public int vid; //Vehicle identifier 10 | public int qid; //Query ID 11 | 12 | public AccountBalanceEvent(String[] fields) { 13 | this.time = Long.parseLong(fields[1]);//Seconds since start of simulation 14 | this.vid = Integer.parseInt(fields[2]);//Car ID 15 | this.qid = Integer.parseInt(fields[9]);//Query ID 16 | } 17 | 18 | public AccountBalanceEvent(long ttime, int tvid, int tqid) { 19 | this.time = ttime;//Seconds since start of simulation 20 | this.vid = tvid;//Car ID 21 | this.qid = tqid;//Query ID 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | return "AccountBalanceEvent [time=" + time + ", vid=" + vid + ", qid=" 27 | + qid + "]"; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/util/events/ExpenditureEvent.java: -------------------------------------------------------------------------------- 1 | package util.events; 2 | 3 | /** 4 | * @author miyuru 5 | */ 6 | public class ExpenditureEvent { 7 | public long time; //A timestamp measured in seconds since the start of the simulation 8 | public int vid; //vehicle identifier 9 | public int qid; //Query ID 10 | public byte xWay; //Express way number 0 .. 9 11 | public int day; //The day for which the daily expenditure value is needed 12 | 13 | public ExpenditureEvent(String[] fields) { 14 | this.time = Long.parseLong(fields[1]);//Seconds since start of simulation 15 | this.vid = Integer.parseInt(fields[2]);//Car ID 16 | this.qid = Integer.parseInt(fields[9]);//Query ID 17 | this.xWay = Byte.parseByte(fields[4]);//Expressway number 18 | this.day = Integer.parseInt(fields[14]);//Day 19 | } 20 | 21 | public ExpenditureEvent() { 22 | 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return "ExpenditureEvent [time=" + time + ", vid=" + vid + ", qid=" 28 | + qid + ", xWay=" + xWay + "]"; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/util/events/HistoryEvent.java: -------------------------------------------------------------------------------- 1 | package util.events; 2 | 3 | /** 4 | * @author miyuru 5 | */ 6 | public class HistoryEvent { 7 | //carid, d, x, daily_exp 8 | public int carid; 9 | public int d; 10 | public int x; 11 | public int daily_exp; 12 | 13 | public HistoryEvent(int carid, int d, int x, int daily_exp) { 14 | super(); 15 | this.carid = carid; 16 | this.d = d; 17 | this.x = x; 18 | this.daily_exp = daily_exp; 19 | } 20 | 21 | 22 | } 23 | -------------------------------------------------------------------------------- /common/src/main/java/util/events/LAVEvent.java: -------------------------------------------------------------------------------- 1 | package util.events; 2 | 3 | 4 | /** 5 | * @author miyuru 6 | * LAV is the Latest Average Velocity in each direction for each one-mile segment. This is computed as 7 | * the average speeds of the vehicles in each segment, and is computed every minute by averaging the speeds 8 | * of all position reports issued within the previous 5 minutes. 9 | */ 10 | public class LAVEvent { 11 | public byte segment; //A segement is in the range 0..99; It corresponds to a mile in the high way system 12 | public float lav; //Latest Average Velocity 13 | public byte dir; //Direction of travel (west = 0; East = 1) 14 | 15 | 16 | public LAVEvent(byte seg, float velocity, byte dir) { 17 | this.segment = seg; 18 | this.lav = velocity; 19 | this.dir = dir; 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return "LAVEvent [segment=" + segment + ", lav=" + lav + ", dir=" + dir 25 | + "]"; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /common/src/main/java/util/events/NOVEvent.java: -------------------------------------------------------------------------------- 1 | package util.events; 2 | 3 | /** 4 | * Created by szhang026 on 20/2/2016. 5 | */ 6 | public class NOVEvent { 7 | public int minute; // Current Minute 8 | public byte segment; //A segment is in the range 0..99; It corresponds to a mile in the high way system 9 | public int nov; //Number of vehicles in this particular Segment 10 | 11 | public NOVEvent(int current_minute, byte mile, int numVehicles) { 12 | this.minute = current_minute; 13 | this.segment = mile; 14 | this.nov = numVehicles; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /common/src/main/java/util/events/TollCalculationEvent.java: -------------------------------------------------------------------------------- 1 | package util.events; 2 | 3 | /** 4 | * @author miyuru 5 | */ 6 | public class TollCalculationEvent { 7 | 8 | public int vid; //vehicle identifier 9 | public int toll; //The toll 10 | public byte segment; //The mile 11 | 12 | public TollCalculationEvent() { 13 | 14 | } 15 | 16 | public int getVid() { 17 | return vid; 18 | } 19 | 20 | public void setVid(int vid) { 21 | this.vid = vid; 22 | } 23 | 24 | public int getToll() { 25 | return toll; 26 | } 27 | 28 | public void setToll(int toll) { 29 | this.toll = toll; 30 | } 31 | 32 | public byte getSegment() { 33 | return segment; 34 | } 35 | 36 | public void setSegment(byte segment) { 37 | this.segment = segment; 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return "TollCalculationEvent [vid=" + vid + ", toll=" + toll + ", segment=" + segment + "]"; 43 | } 44 | 45 | public String toCompressedString() { 46 | return "" + vid + " " + toll + " " + segment; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /common/src/main/java/util/io/FileUtils.java: -------------------------------------------------------------------------------- 1 | package util.io; 2 | 3 | public class FileUtils { 4 | public static String humanReadableByteCount(long bytes) { 5 | return humanReadableByteCount(bytes, false); 6 | } 7 | 8 | public static String humanReadableByteCount(long bytes, boolean si) { 9 | int unit = si ? 1000 : 1024; 10 | if (bytes < unit) return bytes + " B"; 11 | int exp = (int) (Math.log(bytes) / Math.log(unit)); 12 | String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); 13 | return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); 14 | } 15 | } -------------------------------------------------------------------------------- /common/src/main/java/util/io/IOUtils.java: -------------------------------------------------------------------------------- 1 | package util.io; 2 | 3 | import java.io.InputStream; 4 | import java.util.Scanner; 5 | 6 | public class IOUtils { 7 | public static String convertStreamToString(InputStream is) { 8 | Scanner s = null; 9 | try { 10 | s = new Scanner(is).useDelimiter("\\A"); 11 | } catch (Exception ex) { 12 | System.out.println("FUCKING error!" + ex.getMessage()); 13 | } 14 | return s.hasNext() ? s.next() : ""; 15 | } 16 | 17 | public static String convertStreamToString(InputStream is, String charsetName) { 18 | Scanner s = new Scanner(is, charsetName).useDelimiter("\\A"); 19 | return s.hasNext() ? s.next() : ""; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /common/src/main/java/util/math/AverageTracker.java: -------------------------------------------------------------------------------- 1 | package util.math; 2 | 3 | /** 4 | * A helper class to track the average per each house and each plug. 5 | * Author: Thilina 6 | * Date: 10/31/14 7 | */ 8 | public class AverageTracker { 9 | private int count; 10 | private double total; 11 | 12 | public void track(double value) { 13 | total += value; 14 | count++; 15 | } 16 | 17 | public double retrieve() { 18 | return total / count; 19 | } 20 | 21 | public void reset() { 22 | count = 0; 23 | total = 0; 24 | } 25 | 26 | } -------------------------------------------------------------------------------- /common/src/main/java/util/math/Entropy.java: -------------------------------------------------------------------------------- 1 | package util.math; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | public class Entropy { 7 | public static double calculateEntropyHistogram(int totalCount, Map> histogram) { 8 | double entropy = 0.0; 9 | 10 | for (Map.Entry> entry : histogram.entrySet()) { 11 | double prob = (double) entry.getValue().size() / totalCount; 12 | entropy -= prob * log2(prob); 13 | } 14 | 15 | return entropy; 16 | } 17 | 18 | public static double calculateEntropyNormalDistribution(double sigma) { 19 | if (sigma == 0) { 20 | sigma = 0.000001; 21 | } 22 | return 0.5 * Math.log(2 * Math.PI * Math.E * Math.pow(sigma, 2)); 23 | } 24 | 25 | 26 | private static double log2(double value) { 27 | return Math.log(value) / Math.log(2); 28 | } 29 | } -------------------------------------------------------------------------------- /common/src/main/java/util/math/MaximumLikelihoodNormalDistribution.java: -------------------------------------------------------------------------------- 1 | package util.math; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | public class MaximumLikelihoodNormalDistribution { 7 | private double mu; 8 | private double sigma; 9 | 10 | public MaximumLikelihoodNormalDistribution(int totalCount, Map> histogram) { 11 | mu = 0; 12 | sigma = 0; 13 | 14 | if (totalCount == 0) { 15 | return; 16 | } 17 | 18 | for (Map.Entry> entry : histogram.entrySet()) { 19 | mu += entry.getKey() * entry.getValue().size(); 20 | } 21 | mu /= totalCount; 22 | 23 | for (Map.Entry> entry : histogram.entrySet()) { 24 | double value = Math.pow((entry.getKey() - mu), 2); 25 | sigma += value * entry.getValue().size(); 26 | // System.out.println("(" + entry.getKey() + " - " + mu + ")^2 = " + value); 27 | } 28 | 29 | sigma = sigma / (totalCount - 1); 30 | // sigma = totalCount / (totalCount - 1) * sigma; 31 | sigma = Math.sqrt(sigma); 32 | } 33 | 34 | public double getMu() { 35 | return mu; 36 | } 37 | 38 | public double getSigma() { 39 | return sigma; 40 | } 41 | } -------------------------------------------------------------------------------- /common/src/main/java/util/math/OutlierTracker.java: -------------------------------------------------------------------------------- 1 | package util.math; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Author: Thilina 8 | * Date: 12/6/14 9 | */ 10 | public class OutlierTracker { 11 | private Set completeSet = new HashSet<>(); 12 | private Set outlierSet = new HashSet<>(); 13 | 14 | public void addMember(String key) { 15 | completeSet.add(key); 16 | } 17 | 18 | public void addOutlier(String key) { 19 | outlierSet.add(key); 20 | } 21 | 22 | public void removeOutlier(String key) { 23 | outlierSet.remove(key); 24 | } 25 | 26 | public boolean isOutlier(String key) { 27 | return outlierSet.contains(key); 28 | } 29 | 30 | public boolean isMember(String key) { 31 | return completeSet.contains(key); 32 | } 33 | 34 | public double getCurrentPercentage() { 35 | return (outlierSet.size() * 1.0) / (completeSet.size()); 36 | } 37 | } -------------------------------------------------------------------------------- /common/src/main/java/util/math/SimpleStat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * chombo: Hadoop Map Reduce utility 3 | * Author: Pranab Ghosh 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); you 6 | * may not use this file except in compliance with the License. You may 7 | * obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | * implied. See the License for the specific language governing 15 | * permissions and limitations under the License. 16 | */ 17 | 18 | package util.math; 19 | 20 | /** 21 | * @author pranab 22 | */ 23 | public class SimpleStat { 24 | private double sum; 25 | private int count; 26 | 27 | public void add(double value) { 28 | sum += value; 29 | ++count; 30 | } 31 | 32 | public double getMean() { 33 | return sum / count; 34 | } 35 | } -------------------------------------------------------------------------------- /common/src/main/java/util/model/finance/IndicatorParameter.java: -------------------------------------------------------------------------------- 1 | package util.model.finance; 2 | 3 | import org.joda.time.DateTime; 4 | 5 | /** 6 | * User: warrenhenning Date: 9/20/12 Time: 4:11 AM 7 | */ 8 | public class IndicatorParameter { 9 | 10 | private DateTime time; 11 | private int period; 12 | 13 | public IndicatorParameter(DateTime time, int period) { 14 | this.time = time; 15 | this.period = period; 16 | } 17 | 18 | public DateTime getTime() { 19 | return time; 20 | } 21 | 22 | public int getPeriod() { 23 | return period; 24 | } 25 | 26 | @Override 27 | public boolean equals(Object o) { 28 | if (this == o) { 29 | return true; 30 | } 31 | if (o == null || getClass() != o.getClass()) { 32 | return false; 33 | } 34 | 35 | IndicatorParameter that = (IndicatorParameter) o; 36 | 37 | return period == that.period && time.equals(that.time); 38 | } 39 | 40 | @Override 41 | public int hashCode() { 42 | int result = time.hashCode(); 43 | result = 31 * result + period; 44 | return result; 45 | } 46 | } -------------------------------------------------------------------------------- /common/src/main/java/util/model/log/NotificationDetails.java: -------------------------------------------------------------------------------- 1 | package util.model.log; 2 | 3 | public class NotificationDetails { 4 | private String to; 5 | private Severity severity; 6 | private String message; 7 | 8 | public NotificationDetails(String to, Severity severity, String message) { 9 | this.to = to; 10 | this.severity = severity; 11 | this.message = message; 12 | } 13 | 14 | public String getTo() { 15 | return to; 16 | } 17 | 18 | public Severity getSeverity() { 19 | return severity; 20 | } 21 | 22 | public String getMessage() { 23 | return message; 24 | } 25 | } -------------------------------------------------------------------------------- /common/src/main/java/util/model/log/Severity.java: -------------------------------------------------------------------------------- 1 | package util.model.log; 2 | 3 | public enum Severity { 4 | LOW, MEDIUM, HIGH 5 | } -------------------------------------------------------------------------------- /common/src/main/java/util/window/SlidingWindowCallback.java: -------------------------------------------------------------------------------- 1 | package util.window; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Author: Thilina 7 | * Date: 11/22/14 8 | */ 9 | public interface SlidingWindowCallback { 10 | void remove(List entries); 11 | } -------------------------------------------------------------------------------- /common/src/main/java/util/window/SlidingWindowEntry.java: -------------------------------------------------------------------------------- 1 | package util.window; 2 | 3 | /** 4 | * Author: Thilina 5 | * Date: 11/22/14 6 | */ 7 | public interface SlidingWindowEntry { 8 | 9 | long getTime(); 10 | 11 | } -------------------------------------------------------------------------------- /common/src/main/resources/config/CrossTables.properties: -------------------------------------------------------------------------------- 1 | ct.spout.threads=1 2 | ct.sink.threads=1 3 | ct.spout.class=applications.spout.SIMPLETxnSpout 4 | ct.spout.parser=parser.StringParser 5 | ct.sink.class=applications.sink.CTSink 6 | ct.threads=1 7 | ct.kafka.zookeeper.host=localhost:2181 8 | ct.kafka.spout.topic=ct 9 | ct.kafka.consumer.id=ct 10 | ct.kafka.zookeeper.path=/Documents/zookeeper-3.5.2-alpha -------------------------------------------------------------------------------- /common/src/main/resources/config/FraudDetection.properties: -------------------------------------------------------------------------------- 1 | batch_size.fd.spout.class=applications.spout.BatchFDspout 2 | fd.spout.class=applications.spout.MemFileSpout 3 | fd.spout.parser=parser.TransactionParser 4 | split=false 5 | win.fd.spout.path=fd//credit-card.dat 6 | unix.fd.spout.path=fd/credit-card.dat 7 | unix.fd.test.spout.path=fd//credit-card.dat 8 | fd.sink.class=applications.sink.MeasureSink 9 | fd.spout.threads=1 10 | fd.sink.threads=1 11 | fd.predictor.model=mm 12 | win.fd.markov.model.key=fd//model.txt 13 | unix.fd.markov.model.key=fd/model.txt 14 | fd.local.predictor=true 15 | fd.state.seq.window.size=10 16 | fd.state.ordinal=1 17 | fd.detection.algorithm=missProbability 18 | fd.metric.threshold=0.8 19 | 20 | -------------------------------------------------------------------------------- /common/src/main/resources/config/FraudDetection_latency.properties: -------------------------------------------------------------------------------- 1 | batch_size.fd.spout.class=applications.spout.BatchFDspout 2 | fd.spout.class=applications.spout.MemFileSpout_latency 3 | fd.sink.class=applications.sink.MeasureSink_latency 4 | fd.spout.parser=parser.TransactionParser 5 | split=false 6 | win.fd.spout.path=fd//credit-card.dat 7 | unix.fd.spout.path=fd/credit-card.dat 8 | unix.fd.test.spout.path=fd//credit-card.dat 9 | fd.spout.threads=1 10 | fd.sink.threads=1 11 | fd.predictor.model=mm 12 | win.fd.markov.model.key=fd//model.txt 13 | unix.fd.markov.model.key=fd/model.txt 14 | fd.local.predictor=true 15 | fd.state.seq.window.size=10 16 | fd.state.ordinal=1 17 | fd.detection.algorithm=missProbability 18 | fd.metric.threshold=0.8 19 | 20 | -------------------------------------------------------------------------------- /common/src/main/resources/config/Leaderboard.properties: -------------------------------------------------------------------------------- 1 | lb.spout.class=applications.spout.PhoneCallGeneratorSpout 2 | lb.spout.parser=parser.StringParser 3 | lb.sink.class=applications.sink.MeasureSink 4 | count_number=1 -------------------------------------------------------------------------------- /common/src/main/resources/config/LinearRoad.properties: -------------------------------------------------------------------------------- 1 | lr.spout.class=applications.spout.MemFileSpout 2 | batch_size.lr.spout.class=applications.spout.BatchMemFileSpout 3 | lr.spout.parser=parser.StringParser 4 | #lr.sink.class=applications.sink.ConsoleSink 5 | lr.sink.class=applications.sink.MeasureSink 6 | unix.lr.spout.path=lr/cardatapoints-combine.out 7 | win.lr.spout.path=lr//cardatapoints-combine.out 8 | unix.lr.test.spout.path=lr//cardatapoints-combine1.out 9 | unix.test.linear-history-file=lr/historical-tolls-combine.out 10 | unix.linear-history-file=lr/historical-tolls-combine.out 11 | win.linear-history-file=lr//historical-tolls-combine.out 12 | lr.spout.threads=1 13 | lr.sink.threads=1 14 | end_index1=100 15 | end_index2=100 16 | end_index3=100 17 | end_index4=5600 18 | linear-db-host=localhost 19 | split=false -------------------------------------------------------------------------------- /common/src/main/resources/config/LinearRoad_latency.properties: -------------------------------------------------------------------------------- 1 | lr.spout.class=applications.spout.MemFileSpout_latency 2 | batch_size.lr.spout.class=applications.spout.BatchMemFileSpout 3 | lr.spout.parser=parser.StringParser 4 | #lr.sink.class=applications.sink.ConsoleSink 5 | lr.sink.class=applications.sink.MeasureSink_latency 6 | unix.lr.spout.path=lr/cardatapoints-combine.out 7 | win.lr.spout.path=lr//cardatapoints-combine.out 8 | unix.lr.test.spout.path=lr//cardatapoints-combine1.out 9 | unix.test.linear-history-file=lr/historical-tolls-combine.out 10 | unix.linear-history-file=lr/historical-tolls-combine.out 11 | win.linear-history-file=lr//historical-tolls-combine.out 12 | lr.spout.threads=1 13 | lr.sink.threads=1 14 | end_index1=100 15 | end_index2=100 16 | end_index3=100 17 | end_index4=5600 18 | linear-db-host=localhost -------------------------------------------------------------------------------- /common/src/main/resources/config/LogProcessing.properties: -------------------------------------------------------------------------------- 1 | split=false 2 | batch_size.lg.spout.class=applications.spout.BatchMemFileSpout 3 | lg.spout.class=applications.spout.MemFileSpout 4 | lg.sink.class=applications.sink.MeasureSink 5 | unix.lg.spout.path=lg/http-server.log 6 | unix.lg.test.spout.path=lg/http-server.log 7 | win.lg.spout.path=lg//http-server.log 8 | lg.spout.parser=parser.CommonLogParser 9 | lg.spout.threads=1 10 | lg.count.sink.threads=1 11 | lg.count.sink.class=applications.sink.ForwardSink 12 | lg.status.sink.threads=1 13 | lg.status.sink.class=applications.sink.ForwardSink 14 | lg.country.sink.threads=1 15 | lg.country.sink.class=applications.sink.ForwardSink 16 | lg.volume_counter.window=60 17 | geoip.instance=geoip2 18 | unix.geoip2.db=lg/GeoLite2-City.mmdb 19 | win.geoip2.db=lg//GeoLite2-City.mmdb 20 | -------------------------------------------------------------------------------- /common/src/main/resources/config/LogProcessing_latency.properties: -------------------------------------------------------------------------------- 1 | split=false 2 | batch_size.lg.spout.class=applications.spout.BatchMemFileSpout 3 | lg.spout.class=applications.spout.MemFileSpout_latency 4 | lg.sink.class=applications.sink.MeasureSink_latency 5 | unix.lg.spout.path=lg/http-server.log 6 | unix.lg.test.spout.path=lg/http-server.log 7 | win.lg.spout.path=lg//http-server.log 8 | lg.spout.parser=parser.CommonLogParser 9 | lg.spout.threads=1 10 | lg.count.sink.threads=1 11 | lg.count.sink.class=applications.sink.ForwardSink_latency 12 | lg.status.sink.threads=1 13 | lg.status.sink.class=applications.sink.ForwardSink_latency 14 | lg.country.sink.threads=1 15 | lg.country.sink.class=applications.sink.ForwardSink_latency 16 | lg.volume_counter.window=60 17 | lg.volume_counter.threads=1 18 | lg.status_counter.threads=1 19 | lg.geo_finder.threads=1 20 | lg.geo_stats.threads=1 21 | geoip.instance=geoip2 22 | unix.geoip2.db=lg/GeoLite2-City.mmdb 23 | win.geoip2.db=lg//GeoLite2-City.mmdb 24 | -------------------------------------------------------------------------------- /common/src/main/resources/config/MicroBenchmark.properties: -------------------------------------------------------------------------------- 1 | mb.spout.threads=1 2 | mb.sink.threads=1 3 | mb.spout.class=applications.spout.SIMPLETxnSpout 4 | unix.mb.spout.path=wc/Skew01.dat 5 | win.mb.spout.path=wc//Skew01.dat 6 | unix.mb.test.spout.path=wc//Skew01.dat 7 | mb.spout.parser=parser.StringParser 8 | mb.sink.class=applications.sink.MBSink 9 | mb.threads=1 10 | mb.kafka.zookeeper.host=localhost:2181 11 | mb.kafka.spout.topic=mb 12 | mb.kafka.consumer.id=mb 13 | mb.kafka.zookeeper.path=/Documents/zookeeper-3.5.2-alpha -------------------------------------------------------------------------------- /common/src/main/resources/config/OnlineBiding.properties: -------------------------------------------------------------------------------- 1 | ob.spout.threads=1 2 | ob.sink.threads=1 3 | ob.spout.class=applications.spout.SIMPLETxnSpout 4 | ob.spout.parser=parser.StringParser 5 | ob.sink.class=applications.sink.OBSink 6 | ob.threads=1 7 | ob.kafka.zookeeper.host=localhost:2181 8 | ob.kafka.spout.topic=ob 9 | ob.kafka.consumer.id=ob 10 | #ob.kafka.zookeeper.path=/Documents/zookeeper-3.5.2-alpha -------------------------------------------------------------------------------- /common/src/main/resources/config/PositionKeeping.properties: -------------------------------------------------------------------------------- 1 | pk.spout.class=applications.spout.PKSpout 2 | unix.pk.spout.path=sd/sensors.dat 3 | win.pk.spout.path=sd//sensors.dat 4 | unix.pk.test.spout.path=sd/sensors.dat 5 | pk.spout.parser=parser.SensorParser 6 | pk.spout.threads=1 7 | pk.sink.threads=1 8 | pk.sink.class=applications.sink.PKSink 9 | sd.parser.value_field=temp 10 | split=false -------------------------------------------------------------------------------- /common/src/main/resources/config/SpikeDetection.properties: -------------------------------------------------------------------------------- 1 | batch_size.sd.spout.class=applications.spout.BatchMemFileSpout 2 | sd.spout.class=applications.spout.MemFileSpout 3 | unix.sd.spout.path=sd/sensors.dat 4 | win.sd.spout.path=sd//sensors.dat 5 | unix.sd.test.spout.path=sd/sensors1.dat 6 | sd.spout.parser=parser.SensorParser 7 | sd.spout.threads=1 8 | sd.sink.threads=1 9 | sd.sink.class=applications.sink.MeasureSink 10 | sd.parser.value_field=temp 11 | sd.moving_average.window=1000 12 | split=false -------------------------------------------------------------------------------- /common/src/main/resources/config/SpikeDetection_latency.properties: -------------------------------------------------------------------------------- 1 | sd.spout.class=applications.spout.MemFileSpout_latency 2 | unix.sd.spout.path=sd/sensors.dat 3 | unix.sd.test.spout.path=sd/sensors1.dat 4 | win.sd.spout.path=sd//sensors.dat 5 | sd.spout.parser=parser.SensorParser 6 | sd.spout.threads=1 7 | sd.sink.threads=1 8 | sd.sink.class=applications.sink.MeasureSink_latency 9 | sd.parser.value_field=temp 10 | sd.moving_average.window=1000 11 | -------------------------------------------------------------------------------- /common/src/main/resources/config/StreamingAnalysis.properties: -------------------------------------------------------------------------------- 1 | sa.spout.threads=1 2 | sa.sink.threads=1 3 | sa.sink.class=applications.sink.MeasureSink 4 | #sa.sink.class=applications.sink.ReadSink 5 | #sa.spout.class=applications.spout.MemFileSpout 6 | unix.sa.spout.path=wc/Skew0.dat 7 | win.sa.spout.path=wc//Skew0.dat 8 | sa.spout.parser=parser.SAParser 9 | sa.spout.class=applications.spout.DataSource 10 | sa.spout.wrapper=helper.wrapper.StringStatesWrapper 11 | #sa.spout.wrapper=ValueStatesWrapper -------------------------------------------------------------------------------- /common/src/main/resources/config/TrafficMonitoring.properties: -------------------------------------------------------------------------------- 1 | tm.spout.threads=1 2 | split=false 3 | tm.spout.class=applications.spout.MemFileSpout 4 | batch_size.tm.spout.class=applications.spout.BatchMemFileSpout 5 | unix.tm.spout.path=tm/taxi-traces.csv 6 | win.tm.spout.path=tm//taxi-traces.csv 7 | tm.spout.parser=parser.TaxiTraceParser 8 | #tm.spout.generator= 9 | tm.road.feature.id_key=osm_id 10 | #tm.road.feature.width_key 11 | unix.tm.map_matcher.shapefile=tm/beijing/roads.shp 12 | win.tm.map_matcher.shapefile=tm//beijing//roads.shp 13 | # Beijing bounding box 14 | tm.map_matcher.lat.min=39.689602 15 | tm.map_matcher.lat.max=40.122410 16 | tm.map_matcher.lon.min=116.105789 17 | tm.map_matcher.lon.max=116.670021 18 | tm.sink.threads=1 19 | tm.sink.class=applications.sink.MeasureSink 20 | -------------------------------------------------------------------------------- /common/src/main/resources/config/WordCount.properties: -------------------------------------------------------------------------------- 1 | allocation.path=C:/config/wc_allocation.txt 2 | wc.spout.class=applications.spout.WCGeneratorSpout 3 | unix.wc.spout.path=wc//Skew01.dat 4 | win.wc.spout.path=wc//Skew01.dat 5 | unix.wc.test.spout.path=wc//Skew01.dat 6 | wc.spout.parser=parser.StringParser 7 | wc.sink.class=applications.sink.MeasureSink 8 | count_number=1 9 | split=false -------------------------------------------------------------------------------- /common/src/main/resources/config/WordCount_FT.properties: -------------------------------------------------------------------------------- 1 | wc_ft.spout.class=applications.spout.MemFileSpout_FT 2 | batch_size.wc_ft.spout.class=applications.spout.BatchMemFileSpout 3 | unix.wc_ft.spout.path=wc/Skew0.dat 4 | win.wc_ft.spout.path=wc//Skew0.dat 5 | unix.wc_ft.test.spout.path=wc//test.csv 6 | wc_ft.spout.parser=parser.StringParser 7 | wc_ft.sink.class=applications.sink.MeasureSink_FT 8 | #wc_ft.sink.class=applications.sink.ReadSink 9 | #n=4 10 | end_index=40000000 11 | #wc_ft.spout.threads=2 12 | #wc_ft.sink.threads=1 13 | ###TUNED CONFIG (1 socket): 14 | #wc_ft.splitter.threads=2 15 | #wc_ft.counter.threads=1 16 | count_number=1 17 | #split=false -------------------------------------------------------------------------------- /common/src/main/resources/config/WordCount_latency.properties: -------------------------------------------------------------------------------- 1 | ;=allocation.path=C:/config/wc_allocation.txt 2 | wc.spout.class=applications.spout.MemFileSpout_latency 3 | batch_size.wc.spout.class=applications.spout.BatchMemFileSpout 4 | unix.wc.spout.path=wc/Skew0.dat 5 | win.wc.spout.path=wc//Skew0.dat 6 | unix.wc.test.spout.path=wc//test.csv 7 | wc.spout.parser=parser.StringParser 8 | wc.sink.class=applications.sink.MeasureSink_latency 9 | #wc.sink.class=applications.sink.ReadSink 10 | #n=4 11 | end_index=40000000 12 | #wc.spout.threads=2 13 | #wc.sink.threads=1 14 | ###TUNED CONFIG (1 socket): 15 | #wc.splitter.threads=2 16 | #wc.counter.threads=1 17 | count_number=1 18 | #split=false -------------------------------------------------------------------------------- /common/src/main/resources/file.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=FATAL, file 3 | # Direct executionNode messages to stdout 4 | log4j.appender.file=org.apache.log4j.RollingFileAppender 5 | log4j.appender.file.File=${my.executionNode} 6 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 8 | -------------------------------------------------------------------------------- /common/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | # Direct executionNode messages to stdout 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.Target=System.out 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /common/src/main/resources/standout.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=FATAL, stdout 3 | # Direct executionNode messages to stdout 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.Target=System.out 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /common/src/test/java/TestCommonRunner.java: -------------------------------------------------------------------------------- 1 | import applications.utils; 2 | import org.junit.Test; 3 | 4 | /** 5 | * Created by I309939 on 8/3/2016. 6 | */ 7 | public class TestCommonRunner { 8 | 9 | @Test 10 | public void TestDataGenerator() { 11 | 12 | utils test = new utils(); 13 | 14 | String[] args = { 15 | "main", 16 | String.valueOf(1),//function 17 | String.valueOf(1),//Brisk.execution.runtime.tuple size per byte 18 | String.valueOf(2),//data skew factor 19 | "true", 20 | "false"};//whether print it out. 21 | utils.main(args); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /git_find_big.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x 3 | 4 | # Shows you the largest objects in your repo's pack file. 5 | # Written for osx. 6 | # 7 | # @see http://stubbisms.wordpress.com/2009/07/10/git-script-to-show-largest-pack-objects-and-trim-your-waist-line/ 8 | # @author Antony Stubbs 9 | 10 | # set the internal field spereator to line break, so that we can iterate easily over the verify-pack output 11 | IFS=$'\n'; 12 | 13 | # list all objects including their size, sort by size, take top 10 14 | objects=`git verify-pack -v .git/objects/pack/pack-*.idx | grep -v chain | sort -k3nr | head` 15 | 16 | echo "All sizes are in kB's. The pack column is the size of the object, compressed, inside the pack file." 17 | 18 | output="size,pack,SHA,location" 19 | for y in $objects 20 | do 21 | # extract the size in bytes 22 | size=$((`echo $y | cut -f 5 -d ' '`/1024)) 23 | # extract the compressed size in bytes 24 | compressedSize=$((`echo $y | cut -f 6 -d ' '`/1024)) 25 | # extract the SHA 26 | sha=`echo $y | cut -f 1 -d ' '` 27 | # find the objects location in the repository tree 28 | other=`git rev-list --all --objects | grep $sha` 29 | #lineBreak=`echo -e "\n"` 30 | output="${output}\n${size},${compressedSize},${other}" 31 | done 32 | 33 | echo -e $output | column -t -s ', ' 34 | -------------------------------------------------------------------------------- /list.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x 3 | 4 | # Shows you the largest objects in your repo's pack file. 5 | # Written for osx. 6 | # 7 | # @see http://stubbisms.wordpress.com/2009/07/10/git-script-to-show-largest-pack-objects-and-trim-your-waist-line/ 8 | # @author Antony Stubbs 9 | 10 | # set the internal field spereator to line break, so that we can iterate easily over the verify-pack output 11 | IFS=$'\n'; 12 | 13 | # list all objects including their size, sort by size, take top 10 14 | objects=`git verify-pack -v .git/objects/pack/pack-*.idx | grep -v chain | sort -k3nr | head` 15 | 16 | echo "All sizes are in kB. The pack column is the size of the object, compressed, inside the pack file." 17 | 18 | output="size,pack,SHA,location" 19 | for y in $objects 20 | do 21 | # extract the size in bytes 22 | size=$((`echo $y | cut -f 5 -d ' '`/1024)) 23 | # extract the compressed size in bytes 24 | compressedSize=$((`echo $y | cut -f 6 -d ' '`/1024)) 25 | # extract the SHA 26 | sha=`echo $y | cut -f 1 -d ' '` 27 | # find the objects location in the repository tree 28 | other=`git rev-list --all --objects | grep $sha` 29 | #lineBreak=`echo -e "\n"` 30 | output="${output}\n${size},${compressedSize},${other}" 31 | done 32 | 33 | echo -e $output | column -t -s ', ' 34 | -------------------------------------------------------------------------------- /overseer-master/examples/Makefile: -------------------------------------------------------------------------------- 1 | inchpc = ../src/hpcOverseer 2 | flags = -m64 -O2 -I$(inchpc) -L../src/hpcOverseer ../src/hpcOverseer/libhpcOverseer.so.1.0.0 3 | 4 | all : 5 | g++ $(flags) -o test test.cpp 6 | g++ $(flags) -o test_mt test_mt.cpp -lpthread 7 | g++ $(flags) -o test_arch test_arch.cpp 8 | javac -sourcepath ../src/java/overseer.jar java_hpc.java 9 | javac -sourcepath ../src/java/overseer.jar java_arch.java 10 | javac -sourcepath ../src/java/overseer.jar java_agent.java 11 | javac -sourcepath ../src/java/overseer.jar java_ipmi.java 12 | 13 | # Housekeeping: 14 | .PHONY: clean 15 | clean : 16 | -rm -f *.o 17 | -rm -f test 18 | -rm -f test_mt 19 | -rm -f test_arch 20 | -rm -f *.class 21 | 22 | -------------------------------------------------------------------------------- /overseer-master/examples/test_java_agent.sh: -------------------------------------------------------------------------------- 1 | java -agentpath:/usr/local/lib/liboverAgent.so -cp .:../src/java/overseer.jar java_agent 2 | -------------------------------------------------------------------------------- /overseer-master/examples/test_java_arch.sh: -------------------------------------------------------------------------------- 1 | java -cp .:../src/java/overseer.jar java_arch 2 | -------------------------------------------------------------------------------- /overseer-master/examples/test_java_hpc.sh: -------------------------------------------------------------------------------- 1 | java -cp .:../src/java/overseer.jar java_hpc $1 2 | -------------------------------------------------------------------------------- /overseer-master/examples/test_java_ipmi.sh: -------------------------------------------------------------------------------- 1 | java -cp .:../src/java/overseer.jar java_ipmi 2 | -------------------------------------------------------------------------------- /overseer-master/maven-install.sh: -------------------------------------------------------------------------------- 1 | mvn install:install-file \ 2 | -DgroupId=ch.usi.overseer \ 3 | -DartifactId=overseer-java-api \ 4 | -Dpackaging=jar \ 5 | -Dversion=1.0-SNAPSHOT \ 6 | -Dfile=src/java/overseer.jar \ 7 | -DgeneratePom=true -------------------------------------------------------------------------------- /overseer-master/src/hpcOverseer/Makefile: -------------------------------------------------------------------------------- 1 | # hpcOverseer 2 | # Achille Peternier (C) 2010 USI 3 | 4 | 5 | 6 | ######################################## 7 | # Don't touch anything after this line # 8 | ######################################## 9 | 10 | # Variables: 11 | flags = -m64 -O2 -fPIC -c 12 | 13 | # Creation of libhpcOverseer: 14 | libhpcOverseer.so : hpcOverseer.o hpcOverseer_cpuinfo.o hpcOverseer_rdtsc.o hpcOverseer_scheduler.o hpcOverseer_tools.o 15 | g++ -shared -m64 -Wl,-soname,libhpcOverseer.so.1.0.0 -o libhpcOverseer.so.1.0.0 hpcOverseer.o hpcOverseer_cpuinfo.o hpcOverseer_rdtsc.o hpcOverseer_scheduler.o hpcOverseer_tools.o -lc -lpfm -lhwloc 16 | 17 | hpcOverseer.o : hpcOverseer.cpp 18 | g++ $(flags) hpcOverseer.cpp 19 | 20 | hpcOverseer_cpuinfo.o : hpcOverseer_cpuinfo.cpp 21 | g++ $(flags) hpcOverseer_cpuinfo.cpp 22 | 23 | hpcOverseer_rdtsc.o : hpcOverseer_rdtsc.cpp 24 | g++ $(flags) hpcOverseer_rdtsc.cpp 25 | 26 | hpcOverseer_scheduler.o : hpcOverseer_scheduler.cpp 27 | g++ $(flags) hpcOverseer_scheduler.cpp 28 | 29 | hpcOverseer_tools.o : hpcOverseer_tools.cpp 30 | g++ $(flags) hpcOverseer_tools.cpp 31 | 32 | # Housekeeping: 33 | .PHONY: clean 34 | clean : 35 | -rm -f *.o *.so.* 36 | 37 | 38 | -------------------------------------------------------------------------------- /overseer-master/src/hpcOverseer/hpcOverseer_tools.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file hpcOverseer_tools.cpp 3 | * @brief additional commodity stuffs 4 | * 5 | * @author Achille Peternier (C) USI 2010, achille.peternier@gmail.com 6 | */ 7 | 8 | 9 | 10 | ////////////// 11 | // #INCLUDE // 12 | ////////////// 13 | #include "hpcOverseer.h" 14 | -------------------------------------------------------------------------------- /overseer-master/src/hpcOverseerWrapper/Makefile: -------------------------------------------------------------------------------- 1 | # hpcOverseerWrapper 2 | # Achille Peternier (C) 2010 USI 3 | 4 | ################################# 5 | # Edit this part (if necessary) # 6 | ################################# 7 | 8 | incjni = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux 9 | 10 | 11 | 12 | ######################################## 13 | # Don't touch anything after this line # 14 | ######################################## 15 | 16 | # Variables: 17 | flags = -m64 -O2 -fPIC -c $(incjni) -I../hpcOverseer 18 | 19 | # Creation of libhpcOverseerWrapper: 20 | libhpcOverseerWrapper.so.1.0.0 : hpcOverseerWrapper.o 21 | g++ -shared -L../hpcOverseer -m64 -Wl,-soname,libhpcOverseerWrapper.so.1.0.0 -o libhpcOverseerWrapper.so.1.0.0 hpcOverseerWrapper.o ../hpcOverseer/libhpcOverseer.so.1.0.0 -lc 22 | 23 | hpcOverseerWrapper.o : hpcOverseerWrapper.cpp ch_usi_overseerJNI_WrapperJNI.h 24 | g++ $(flags) hpcOverseerWrapper.cpp 25 | 26 | # Housekeeping: 27 | .PHONY: clean 28 | clean : 29 | -rm -f *.o *.so.* 30 | -------------------------------------------------------------------------------- /overseer-master/src/java/Makefile: -------------------------------------------------------------------------------- 1 | # java bindings (overseer.jar) 2 | # Achille Peternier (C) 2010 USI 3 | 4 | # 5 | # Make sure that the java/bin directory is in the path 6 | # Fix the javac and jar paths otherwise 7 | # 8 | 9 | # Creation of overseer.jar: 10 | all : 11 | javac ch/usi/overseer/OverHpc.java 12 | javac ch/usi/overseer/OverAgent.java 13 | javac ch/usi/overseer/OverIpmi.java 14 | jar cf overseer.jar ch/usi/overseer/* 15 | 16 | # Housekeeping: 17 | .PHONY: clean 18 | clean : 19 | -rm -f overseer.jar 20 | -rm -f ch/usi/overseer/*.class 21 | -------------------------------------------------------------------------------- /overseer-master/src/overAgent/Makefile: -------------------------------------------------------------------------------- 1 | # overAgent 2 | # Achille Peternier (C) 2010 USI 3 | 4 | ################################# 5 | # Edit this part (if necessary) # 6 | ################################# 7 | 8 | incjni = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux 9 | 10 | 11 | 12 | ######################################## 13 | # Don't touch anything after this line # 14 | ######################################## 15 | 16 | # Variables: 17 | flags = -m64 -O2 -fPIC -c $(incjni) -I../hpcOverseer 18 | 19 | # Creation of liboverAgent: 20 | liboverAgent.so.1.0.0 : overAgent.o threadList.o jniInterface.o 21 | g++ -shared -m64 -Wl,-soname,liboverAgent.so.1.0.0 -o liboverAgent.so.1.0.0 overAgent.o threadList.o jniInterface.o -lc 22 | 23 | overAgent.o : overAgent.cpp 24 | g++ $(flags) overAgent.cpp 25 | 26 | threadList.o : threadList.cpp 27 | g++ $(flags) threadList.cpp 28 | 29 | jniInterface.o : jniInterface.cpp 30 | g++ $(flags) jniInterface.cpp 31 | 32 | 33 | # Housekeeping: 34 | .PHONY: clean 35 | clean : 36 | -rm -f *.o *.so.* 37 | -------------------------------------------------------------------------------- /overseer-master/src/overAgent/ch_usi_overseerJNI_WrapperJNI.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class ch_usi_overseerJNI_WrapperJNI */ 4 | 5 | #ifndef _Included_ch_usi_overseer_OverAgent 6 | #define _Included_ch_usi_overseer_OverAgent 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | JNIEXPORT jfloat JNICALL Java_ch_usi_overseer_OverAgent_getThreadCpuUsage 12 | (JNIEnv *, jobject, jint); 13 | 14 | JNIEXPORT jfloat JNICALL Java_ch_usi_overseer_OverAgent_getThreadCpuUsageRelative 15 | (JNIEnv *, jobject, jint); 16 | 17 | JNIEXPORT void JNICALL Java_ch_usi_overseer_OverAgent_updateStats 18 | (JNIEnv *, jobject); 19 | 20 | JNIEXPORT jboolean JNICALL Java_ch_usi_overseer_OverAgent_isRunning 21 | (JNIEnv *, jobject); 22 | 23 | JNIEXPORT jint JNICALL Java_ch_usi_overseer_OverAgent_getVersion 24 | (JNIEnv *, jobject); 25 | 26 | JNIEXPORT jint JNICALL Java_ch_usi_overseer_OverAgent_getNumberOfThreads 27 | (JNIEnv *, jobject); 28 | 29 | JNIEXPORT void JNICALL Java_ch_usi_overseer_OverAgent_init 30 | (JNIEnv *, jobject); 31 | 32 | #ifdef __cplusplus 33 | } 34 | #endif 35 | #endif 36 | 37 | -------------------------------------------------------------------------------- /overseer-master/src/overAgent/smartLock.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file smartLock.h 3 | * @brief Handy mutex manager class 4 | * 5 | * @author Achille Peternier (C) USI 2010, achille.peternier@gmail.com 6 | */ 7 | #ifndef SMARTLOCK_H_INCLUDED 8 | #define SMARTLOCK_H_INCLUDED 9 | 10 | 11 | 12 | ////////////// 13 | // #INCLUDE // 14 | ////////////// 15 | #include 16 | #include 17 | 18 | 19 | ///////////// 20 | // #DEFINE // 21 | ///////////// 22 | 23 | // Macros for locking: 24 | #define MUTEX_SYNC(x) SmartLock smartlock(&x) 25 | //#define MUTEX_SYNC(x) 26 | #define MUTEX_BEGIN(x) do { SmartLock smartlock(&x) 27 | #define MUTEX_END(x) } while(0) 28 | 29 | 30 | 31 | ///////////////////// 32 | // CLASS SmartLock // 33 | ///////////////////// 34 | class SmartLock 35 | { 36 | ///////////// 37 | public: // 38 | ///////////// 39 | 40 | // Const/dest: 41 | SmartLock(pthread_mutex_t *m) : mutex(m) 42 | { pthread_mutex_lock(mutex); } 43 | 44 | ~SmartLock() 45 | { pthread_mutex_unlock(mutex); } 46 | 47 | 48 | ////////////// 49 | private: // 50 | ////////////// 51 | 52 | // Mutex reference: 53 | pthread_mutex_t *mutex; 54 | }; 55 | 56 | #endif // SMARTLOCK_H_INCLUDED 57 | -------------------------------------------------------------------------------- /overseer-master/src/overIpmi/Makefile: -------------------------------------------------------------------------------- 1 | # overIpmi 2 | # Achille Peternier (C) 2010 USI 3 | 4 | ################################# 5 | # Edit this part (if necessary) # 6 | ################################# 7 | 8 | incjni = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux 9 | 10 | 11 | 12 | ######################################## 13 | # Don't touch anything after this line # 14 | ######################################## 15 | 16 | # Variables: 17 | flags = -m64 -O2 -fPIC -c $(incjni) -I../hpcOverseer 18 | 19 | # Creation of liboverIpmi: 20 | liboverIpmi.so.1.0.0 : overIpmi.o 21 | g++ -shared -m64 -Wl,-soname,liboverIpmi.so.1.0.0 -o liboverIpmi.so.1.0.0 overIpmi.o -lipmimonitoring -lc 22 | 23 | overIpmi.o : overIpmi.c 24 | gcc $(flags) overIpmi.c 25 | 26 | # Housekeeping: 27 | .PHONY: clean 28 | clean : 29 | -rm -f *.o *.so.* 30 | -------------------------------------------------------------------------------- /overseer-master/src/overIpmi/overIpmi.h: -------------------------------------------------------------------------------- 1 | #ifndef OVERIPMI_H_INCLUDED 2 | #define OVERIPMI_H_INCLUDED 3 | 4 | 5 | 6 | ////////////// 7 | // #INCLUDE // 8 | ////////////// 9 | #include 10 | 11 | 12 | 13 | ///////////// 14 | // #DEFINE // 15 | ///////////// 16 | 17 | // Generic: 18 | #define OI_NAME "overIpmi v0.1a" 19 | #define OI_VERSION 0.1 20 | 21 | 22 | 23 | ///////////// 24 | // METHODS // 25 | ///////////// 26 | 27 | JNIEXPORT jboolean JNICALL Java_ch_usi_overseer_OverIpmi_init(JNIEnv *env, jobject obj); 28 | JNIEXPORT jboolean JNICALL Java_ch_usi_overseer_OverIpmi_free(JNIEnv *env, jobject obj); 29 | JNIEXPORT jint JNICALL Java_ch_usi_overseer_OverIpmi_getNumberOfSensors(JNIEnv *env, jobject obj); 30 | JNIEXPORT jstring JNICALL Java_ch_usi_overseer_OverIpmi_getSensorName(JNIEnv *env, jobject obj, jint sensorId); 31 | JNIEXPORT jdouble JNICALL Java_ch_usi_overseer_OverIpmi_getSensorValue(JNIEnv *env, jobject obj, jint sensorId); 32 | 33 | #endif // OVERIPMI_H_INCLUDED 34 | --------------------------------------------------------------------------------