├── README.md ├── src └── net-apps │ ├── InProcStreamProcessing.Shared │ ├── InProcStreamProcessing.Shared.csproj │ ├── BusMessage.cs │ ├── BusConfiguration │ │ ├── IConfigurationLoader.cs │ │ ├── SensorConfig.cs │ │ └── ConfigurationLoader.cs │ └── DataBusInterface.cs │ ├── InProcStreamProcessing.TplDataflow │ ├── DataBus │ │ ├── FlowControlMode.cs │ │ ├── IDataBusReader.cs │ │ └── DataBusReader.cs │ ├── Messages │ │ ├── RawBusMessage.cs │ │ └── DecodedMessage.cs │ ├── Feeds │ │ ├── IRealTimePublisher.cs │ │ ├── IStatsFeedPublisher.cs │ │ ├── RealTimePublisher.cs │ │ └── StatsFeedPublisher.cs │ ├── Persistence │ │ ├── IDbPersister.cs │ │ ├── IMessageFileWriter.cs │ │ ├── MessageFileWriter.cs │ │ └── DbPersister.cs │ ├── Decoders │ │ ├── IDecoder.cs │ │ └── Decoder.cs │ ├── InProcStreamProcessing.TplDataFlow.csproj │ ├── Program.cs │ └── ProcessingPipeline.cs │ ├── DistributedStreamProcessing.ApacheKafka.Decoder │ ├── IDecoder.cs │ ├── RawBusMessage.cs │ ├── DecodedMessage.cs │ ├── DistributedStreamProcessing.ApacheKafka.Decoder.csproj │ ├── Program.cs │ ├── Producer.cs │ ├── Decoder.cs │ └── Consumer.cs │ ├── InProcStreamProcessing.Rx │ ├── Feeds │ │ ├── IRealTimePublisher.cs │ │ ├── IStatsFeedPublisher.cs │ │ ├── RealTimePublisher.cs │ │ └── StatsFeedPublisher.cs │ ├── Messages │ │ ├── RawBusMessage.cs │ │ └── DecodedMessage.cs │ ├── Persistence │ │ ├── IDbPersister.cs │ │ ├── IMessageFileWriter.cs │ │ ├── MessageFileWriter.cs │ │ └── DbPersister.cs │ ├── Decoders │ │ ├── IDecoder.cs │ │ └── Decoder.cs │ ├── InProcStreamProcessing.Rx.csproj │ ├── DataBus │ │ ├── IDataBusReader.cs │ │ └── DataBusReader.cs │ ├── RxExtensions.cs │ ├── Program.cs │ └── ProcessingPipeline.cs │ ├── DistributedStreamProcessing.ApacheKafka │ ├── RawBusMessage.cs │ ├── DistributedStreamProcessing.ApacheKafka.Producer.csproj │ ├── Program.cs │ ├── RawMessageSerializer.cs │ └── DataBusReader.cs │ ├── DistributedStreamProcessing.ApacheKafka.DbPersister │ ├── DistributedStreamProcessing.ApacheKafka.DbPersister.csproj │ ├── Persistence │ │ ├── IDbPersister.cs │ │ └── DbPersistence.cs │ ├── DecodedMessage.cs │ ├── Program.cs │ └── Consumer.cs │ ├── DistributedStreamProcessing.ApacheKafka.WindowedStats │ ├── DistributedStreamProcessing.ApacheKafka.WindowedStats.csproj │ ├── DecodedMessage.cs │ ├── StatsWindowMessage.cs │ ├── Program.cs │ ├── Consumer.cs │ └── Producer.cs │ └── StreamProcessing.sln ├── environment └── kafka │ └── docker-compose.yml ├── LICENSE └── .gitignore /README.md: -------------------------------------------------------------------------------- 1 | # StreamProcessingSeries 2 | Source code for my data processing pipelines blog series. 3 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Shared/InProcStreamProcessing.Shared.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/DataBus/FlowControlMode.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.TplDataFlow.DataBus 6 | { 7 | public enum FlowControlMode 8 | { 9 | LoadShed, 10 | BackPressure 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Shared/BusMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.Shared 6 | { 7 | public class BusMessage 8 | { 9 | public byte[] Data { get; set; } 10 | public long Ticks { get; set; } 11 | public string XYZ { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/IDecoder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DistributedStreamProcessing.ApacheKafka.Decoder 6 | { 7 | public interface IDecoder 8 | { 9 | void LoadSensorConfigs(); 10 | IEnumerable Decode(RawBusMessage message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Feeds/IRealTimePublisher.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Rx.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.Rx.Feeds 8 | { 9 | public interface IRealTimePublisher 10 | { 11 | Task PublishAsync(DecodedMessage message); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Messages/RawBusMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.Rx.Messages 6 | { 7 | public class RawBusMessage 8 | { 9 | public int Counter { get; set; } 10 | public byte[] Data { get; set; } 11 | public DateTime ReadingTime { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Persistence/IDbPersister.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Rx.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.Rx.Persistence 8 | { 9 | public interface IDbPersister 10 | { 11 | Task PersistAsync(IList messages); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Messages/RawBusMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.TplDataFlow.Messages 6 | { 7 | public class RawBusMessage 8 | { 9 | public int Counter { get; set; } 10 | public byte[] Data { get; set; } 11 | public DateTime ReadingTime { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Feeds/IStatsFeedPublisher.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Rx.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.Rx.Feeds 8 | { 9 | public interface IStatsFeedPublisher 10 | { 11 | Task PublishAsync(IList messages, TimeSpan period); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Feeds/IRealTimePublisher.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.TplDataFlow.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.TplDataFlow.Feeds 8 | { 9 | public interface IRealTimePublisher 10 | { 11 | Task PublishAsync(DecodedMessage message); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Persistence/IDbPersister.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.TplDataFlow.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.TplDataFlow.Persistence 8 | { 9 | public interface IDbPersister 10 | { 11 | Task PersistAsync(IList messages); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Feeds/IStatsFeedPublisher.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.TplDataFlow.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.TplDataFlow.Feeds 8 | { 9 | public interface IStatsFeedPublisher 10 | { 11 | Task PublishAsync(IList messages, TimeSpan window); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Persistence/IMessageFileWriter.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Rx.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.Rx.Persistence 8 | { 9 | public interface IMessageFileWriter 10 | { 11 | void Open(); 12 | Task WriteAsync(RawBusMessage message); 13 | void Close(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka/RawBusMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DistributedStreamProcessing.ApacheKafka.Producer 6 | { 7 | public class RawBusMessage 8 | { 9 | public int Counter { get; set; } 10 | public byte[] Data { get; set; } 11 | public DateTime ReadingTime { get; set; } 12 | public string MachineId { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Decoders/IDecoder.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared.SensorConfiguration; 2 | using InProcStreamProcessing.Rx.Messages; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | 7 | namespace InProcStreamProcessing.Rx.Decoders 8 | { 9 | public interface IDecoder 10 | { 11 | void LoadSensorConfigs(); 12 | IEnumerable Decode(RawBusMessage message); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/RawBusMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DistributedStreamProcessing.ApacheKafka.Decoder 6 | { 7 | public class RawBusMessage 8 | { 9 | public int Counter { get; set; } 10 | public byte[] Data { get; set; } 11 | public DateTime ReadingTime { get; set; } 12 | public string MachineId { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Persistence/IMessageFileWriter.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.TplDataFlow.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace InProcStreamProcessing.TplDataFlow.Persistence 8 | { 9 | public interface IMessageFileWriter 10 | { 11 | void Open(); 12 | Task WriteAsync(RawBusMessage message); 13 | void Close(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Decoders/IDecoder.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared.SensorConfiguration; 2 | using InProcStreamProcessing.TplDataFlow.Messages; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | 7 | namespace InProcStreamProcessing.TplDataFlow.Decoders 8 | { 9 | public interface IDecoder 10 | { 11 | void LoadSensorConfigs(); 12 | IEnumerable Decode(RawBusMessage message); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Shared/BusConfiguration/IConfigurationLoader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.Shared.SensorConfiguration 6 | { 7 | public interface IConfigurationLoader 8 | { 9 | IEnumerable GetConfigs(); 10 | IEnumerable GetConfigs(string componentCode); 11 | SensorConfig GetConfig(string componentCode, string sensorCode); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.DbPersister/DistributedStreamProcessing.ApacheKafka.DbPersister.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.WindowedStats/DistributedStreamProcessing.ApacheKafka.WindowedStats.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/InProcStreamProcessing.Rx.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Messages/DecodedMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.Rx.Messages 6 | { 7 | public class DecodedMessage 8 | { 9 | public int Counter { get; set; } 10 | public string Unit { get; set; } 11 | public double Value { get; set; } 12 | public string Source { get; set; } 13 | public string Label { get; set; } 14 | public DateTime ReadingTime { get; set; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.DbPersister/Persistence/IDbPersister.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace DistributedStreamProcessing.ApacheKafka.DbPersister.Persistence 9 | { 10 | public interface IDbPersister 11 | { 12 | void StartPersisting(CancellationToken token, TimeSpan interval, BlockingCollection decodedMessages); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/DataBus/IDataBusReader.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Rx.Messages; 2 | using System; 3 | using System.Collections.Concurrent; 4 | using System.Collections.Generic; 5 | using System.Reactive.Subjects; 6 | using System.Text; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | 10 | namespace InProcStreamProcessing.Rx.DataBus 11 | { 12 | public interface IDataBusReader 13 | { 14 | IObservable StartConsuming(CancellationToken token, TimeSpan interval); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/InProcStreamProcessing.TplDataFlow.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Messages/DecodedMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.TplDataFlow.Messages 6 | { 7 | public class DecodedMessage 8 | { 9 | public string Unit { get; set; } 10 | public double Value { get; set; } 11 | public string Source { get; set; } 12 | public string Label { get; set; } 13 | public DateTime ReadingTime { get; set; } 14 | public double Counter { get; set; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/DataBus/IDataBusReader.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.TplDataFlow.Messages; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using System.Threading.Tasks.Dataflow; 8 | 9 | namespace InProcStreamProcessing.TplDataFlow.DataBus 10 | { 11 | public interface IDataBusReader 12 | { 13 | Task StartConsuming(ITargetBlock target, CancellationToken token, TimeSpan interval, FlowControlMode flowControlMode); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/DecodedMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DistributedStreamProcessing.ApacheKafka.Decoder 6 | { 7 | public class DecodedMessage 8 | { 9 | public string MachineId { get; set; } 10 | public string Source { get; set; } 11 | public string Label { get; set; } 12 | public DateTime ReadingTime { get; set; } 13 | public string Unit { get; set; } 14 | public double Value { get; set; } 15 | public double Counter { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.DbPersister/DecodedMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DistributedStreamProcessing.ApacheKafka.DbPersister 6 | { 7 | public class DecodedMessage 8 | { 9 | public string MachineId { get; set; } 10 | public string Source { get; set; } 11 | public string Label { get; set; } 12 | public DateTime ReadingTime { get; set; } 13 | public string Unit { get; set; } 14 | public double Value { get; set; } 15 | public double Counter { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.WindowedStats/DecodedMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DistributedStreamProcessing.ApacheKafka.WindowedStats 6 | { 7 | public class DecodedMessage 8 | { 9 | public string MachineId { get; set; } 10 | public string Source { get; set; } 11 | public string Label { get; set; } 12 | public DateTime ReadingTime { get; set; } 13 | public string Unit { get; set; } 14 | public double Value { get; set; } 15 | public double Counter { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka/DistributedStreamProcessing.ApacheKafka.Producer.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/DistributedStreamProcessing.ApacheKafka.Decoder.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Shared/BusConfiguration/SensorConfig.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace InProcStreamProcessing.Shared.SensorConfiguration 6 | { 7 | public class SensorConfig 8 | { 9 | public string ComponentCode { get; set; } 10 | public string SensorCode { get; set; } 11 | public string Unit { get; set; } 12 | public int ByteIndex { get; set; } 13 | public int ByteCount { get; set; } 14 | public double Precision { get; set; } 15 | public bool Signed { get; set; } 16 | public double UpperRange { get; set; } 17 | public double LowerRange { get; set; } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /environment/kafka/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | services: 4 | kafkaserver: 5 | image: "spotify/kafka:latest" 6 | container_name: apachekafka 7 | hostname: kafkaserver 8 | networks: 9 | - kafkanet 10 | ports: 11 | - 2181:2181 12 | - 9092:9092 13 | environment: 14 | ADVERTISED_HOST: kafkaserver 15 | ADVERTISED_PORT: 9092 16 | JMX_PORT: 9999 17 | kafka_manager: 18 | image: "mzagar/kafka-manager-docker:1.3.3.4" 19 | container_name: kafkamanager 20 | networks: 21 | - kafkanet 22 | ports: 23 | - 9000:9000 24 | links: 25 | - kafkaserver 26 | environment: 27 | ZK_HOSTS: "kafkaserver:2181" 28 | 29 | networks: 30 | kafkanet: 31 | driver: bridge -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.WindowedStats/StatsWindowMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace DistributedStreamProcessing.ApacheKafka.WindowedStats 6 | { 7 | public class StatsWindowMessage 8 | { 9 | public string MachineId { get; set; } 10 | public string Source { get; set; } 11 | public string Label { get; set; } 12 | public DateTime PeriodStart { get; set; } 13 | public TimeSpan Window { get; set; } 14 | public string Unit { get; set; } 15 | public double Count { get; set; } 16 | public double MinValue { get; set; } 17 | public double MaxValue { get; set; } 18 | public double AvgValue { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Feeds/RealTimePublisher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | using InProcStreamProcessing.Rx.Messages; 6 | using System.Threading; 7 | 8 | namespace InProcStreamProcessing.Rx.Feeds 9 | { 10 | public class RealTimePublisher : IRealTimePublisher 11 | { 12 | public async Task PublishAsync(DecodedMessage message) 13 | { 14 | // send over a network socket 15 | if(ShowMessages.PrintRealTimeFeed) 16 | Console.WriteLine($" Publish in real-time message {message.Counter} Sensor: { message.Label} { message.Value} { message.Unit} on thread {Thread.CurrentThread.ManagedThreadId}"); 17 | await Task.Delay(1); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Feeds/RealTimePublisher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | using InProcStreamProcessing.TplDataFlow.Messages; 6 | using System.Threading; 7 | 8 | namespace InProcStreamProcessing.TplDataFlow.Feeds 9 | { 10 | public class RealTimePublisher : IRealTimePublisher 11 | { 12 | public async Task PublishAsync(DecodedMessage message) 13 | { 14 | // send over a network socket 15 | if (ShowMessages.PrintRealTimeFeed) 16 | Console.WriteLine($" Publish in real-time message Sensor: { message.Label} { message.Value} { message.Unit} on thread {Thread.CurrentThread.ManagedThreadId}"); 17 | await Task.Yield(); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka/Program.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Serialization; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | using System.Threading; 7 | 8 | namespace DistributedStreamProcessing.ApacheKafka.Producer 9 | { 10 | class Program 11 | { 12 | static void Main(string[] args) 13 | { 14 | Console.Title = "Topology Producer"; 15 | 16 | var source = new CancellationTokenSource(); 17 | var producer = new DataBusReader(); 18 | var producerTask = producer.StartProducing(source.Token, TimeSpan.FromMilliseconds(100)); 19 | 20 | Console.WriteLine("Press any key to shutdown"); 21 | Console.ReadKey(); 22 | 23 | source.Cancel(); 24 | producerTask.Wait(); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Persistence/MessageFileWriter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using InProcStreamProcessing.Rx.Messages; 5 | using System.IO; 6 | using System.Threading.Tasks; 7 | using System.Threading; 8 | 9 | namespace InProcStreamProcessing.Rx.Persistence 10 | { 11 | public class MessageFileWriter : IMessageFileWriter 12 | { 13 | public void Open() 14 | { 15 | // open some kind of file io 16 | } 17 | 18 | public void Close() 19 | { 20 | // safely shutdown the file io 21 | } 22 | 23 | public async Task WriteAsync(RawBusMessage message) 24 | { 25 | if (ShowMessages.PrintFileWriter) 26 | Console.WriteLine($"Write to file message: {message.Counter} on thread {Thread.CurrentThread.ManagedThreadId}"); 27 | await Task.Yield(); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Persistence/MessageFileWriter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using InProcStreamProcessing.TplDataFlow.Messages; 5 | using System.IO; 6 | using System.Threading.Tasks; 7 | using System.Threading; 8 | 9 | namespace InProcStreamProcessing.TplDataFlow.Persistence 10 | { 11 | public class MessageFileWriter : IMessageFileWriter 12 | { 13 | public void Open() 14 | { 15 | // open some kind of file io 16 | } 17 | 18 | public void Close() 19 | { 20 | // safely shutdown the file io 21 | } 22 | 23 | public async Task WriteAsync(RawBusMessage message) 24 | { 25 | if (ShowMessages.PrintFileWriter) 26 | Console.WriteLine($"Write to file message: {message.Counter} on thread {Thread.CurrentThread.ManagedThreadId}"); 27 | await Task.Yield(); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Persistence/DbPersister.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | using InProcStreamProcessing.Rx.Messages; 6 | using System.Linq; 7 | using System.Threading; 8 | 9 | namespace InProcStreamProcessing.Rx.Persistence 10 | { 11 | public class DbPersister : IDbPersister 12 | { 13 | private int _inProgress = 0; 14 | 15 | public async Task PersistAsync(IList messages) 16 | { 17 | Interlocked.Increment(ref _inProgress); 18 | if (ShowMessages.PrintDbPersist) 19 | Console.WriteLine($" Persisted batch of {messages.Count} with Counters from {messages.First().Counter} to {messages.Last().Counter}. In Progress: {_inProgress} on thread {Thread.CurrentThread.ManagedThreadId}"); 20 | 21 | await Task.Delay(100); 22 | Interlocked.Decrement(ref _inProgress); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Persistence/DbPersister.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using InProcStreamProcessing.TplDataFlow.Messages; 6 | using System.Threading; 7 | 8 | namespace InProcStreamProcessing.TplDataFlow.Persistence 9 | { 10 | public class DbPersister : IDbPersister 11 | { 12 | private int _inProgress = 0; 13 | 14 | public async Task PersistAsync(IList messages) 15 | { 16 | Interlocked.Increment(ref _inProgress); 17 | if (ShowMessages.PrintDbPersist) 18 | Console.WriteLine($" Persisted batch of {messages.Count} with Counters from {messages.First().Counter} to {messages.Last().Counter}. In Progress: {_inProgress} on thread {Thread.CurrentThread.ManagedThreadId}"); 19 | 20 | await Task.Delay(10000); 21 | Interlocked.Decrement(ref _inProgress); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka/RawMessageSerializer.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka.Serialization; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace DistributedStreamProcessing.ApacheKafka.Producer 7 | { 8 | public class RawMessageSerializer : ISerializer 9 | { 10 | public IEnumerable> Configure(IEnumerable> config, bool isKey) 11 | { 12 | return config; 13 | } 14 | 15 | public void Dispose() 16 | { 17 | 18 | } 19 | 20 | public byte[] Serialize(string topic, RawBusMessage data) 21 | { 22 | int len = data.Data.Length + 12; 23 | var bytes = new Byte[len]; 24 | Array.Copy(BitConverter.GetBytes(data.Counter), 0, bytes, 0, 4); 25 | Array.Copy(BitConverter.GetBytes(data.ReadingTime.Ticks), 0, bytes, 4, 8); 26 | Array.Copy(data.Data, 0, bytes, 12, len); 27 | 28 | return bytes; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Feeds/StatsFeedPublisher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using InProcStreamProcessing.Rx.Messages; 5 | using System.Threading.Tasks; 6 | using System.Threading; 7 | 8 | namespace InProcStreamProcessing.Rx.Feeds 9 | { 10 | public class StatsFeedPublisher : IStatsFeedPublisher 11 | { 12 | private int _inProgress = 0; 13 | 14 | public async Task PublishAsync(IList messages, TimeSpan period) 15 | { 16 | Interlocked.Increment(ref _inProgress); 17 | if (ShowMessages.PrintStatsFeed) 18 | Console.WriteLine($" Publish stats of {(int)period.TotalSeconds} second batch of {messages.Count} messages with Counters from {messages.First().Counter} to {messages.Last().Counter}. In Progress: {_inProgress} on thread {Thread.CurrentThread.ManagedThreadId}"); 19 | 20 | // simulate the work here 21 | await Task.Delay(1); 22 | Interlocked.Decrement(ref _inProgress); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/RxExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Reactive.Linq; 2 | using System.Reactive.Subjects; 3 | using System.Reactive.Concurrency; 4 | using System.Reactive.Disposables; 5 | 6 | namespace System.Reactive 7 | { 8 | public static class RxExtensions 9 | { 10 | public static IDisposable SubscribeToLatest(this IObservable source, Action action, IScheduler scheduler = null) 11 | { 12 | var sampler = new Subject(); 13 | scheduler = scheduler ?? Scheduler.Default; 14 | var p = source.Publish(); 15 | var connection = p.Connect(); 16 | 17 | var subscription = sampler.Select(x => p.Take(1)) 18 | .Switch() 19 | .ObserveOn(scheduler) 20 | .Subscribe(l => 21 | { 22 | action(l); 23 | sampler.OnNext(Unit.Default); 24 | }); 25 | 26 | sampler.OnNext(Unit.Default); 27 | 28 | return new CompositeDisposable(connection, subscription); 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Feeds/StatsFeedPublisher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using InProcStreamProcessing.TplDataFlow.Messages; 5 | using System.Threading.Tasks; 6 | using System.Linq; 7 | using System.Threading; 8 | 9 | namespace InProcStreamProcessing.TplDataFlow.Feeds 10 | { 11 | public class StatsFeedPublisher : IStatsFeedPublisher 12 | { 13 | private int _inProgress = 0; 14 | 15 | public async Task PublishAsync(IList messages, TimeSpan window) 16 | { 17 | Interlocked.Increment(ref _inProgress); 18 | 19 | if (ShowMessages.PrintStatsFeed) 20 | Console.WriteLine($" Publish {(int)window.TotalSeconds} seconds of stats of batch of {messages.Count} with Counters from {messages.First().Counter} to {messages.Last().Counter}. In Progress: {_inProgress} on thread {Thread.CurrentThread.ManagedThreadId}"); 21 | await Task.Delay(1); 22 | 23 | Interlocked.Decrement(ref _inProgress); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Jack Vanlightly 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.DbPersister/Program.cs: -------------------------------------------------------------------------------- 1 | using DistributedStreamProcessing.ApacheKafka.DbPersister.Persistence; 2 | using System; 3 | using System.Collections.Concurrent; 4 | using System.Diagnostics; 5 | using System.Linq; 6 | using System.Threading; 7 | 8 | namespace DistributedStreamProcessing.ApacheKafka.DbPersister 9 | { 10 | class Program 11 | { 12 | static void Main(string[] args) 13 | { 14 | Console.Title = "DB Persister"; 15 | var decodedMessages = new BlockingCollection(); 16 | 17 | var consumerCts = new CancellationTokenSource(); 18 | var consumer = new Consumer(); 19 | consumer.StartConsuming(consumerCts.Token, decodedMessages); 20 | 21 | var dbPersisterCts = new CancellationTokenSource(); 22 | var dbPersister = new DbPersistence(); 23 | dbPersister.StartPersisting(dbPersisterCts.Token, TimeSpan.FromSeconds(10), decodedMessages); 24 | 25 | Console.WriteLine("Press any key to shutdown"); 26 | Console.ReadKey(); 27 | consumerCts.Cancel(); 28 | 29 | // perform a shutdown, allowing for 60 seconds to process current items 30 | // this is just an example, it could end with message loss 31 | var sw = new Stopwatch(); 32 | sw.Start(); 33 | while (decodedMessages.Any()) 34 | { 35 | if (sw.ElapsedMilliseconds > 60000) 36 | dbPersisterCts.Cancel(); // force close (could lose messages) 37 | else 38 | Thread.Sleep(100); 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Shared/BusConfiguration/ConfigurationLoader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace InProcStreamProcessing.Shared.SensorConfiguration 7 | { 8 | public class ConfigurationLoader : IConfigurationLoader 9 | { 10 | private List _sensorConfigs; 11 | 12 | public ConfigurationLoader() 13 | { 14 | _sensorConfigs = new List(); 15 | _sensorConfigs.Add(new SensorConfig() { ComponentCode = "001", SensorCode = "TEMP", Unit = "DEG", ByteCount = 2, ByteIndex = 0, Precision = 0.01, Signed = true, LowerRange = -200, UpperRange = 400 }); 16 | _sensorConfigs.Add(new SensorConfig() { ComponentCode = "001", SensorCode = "PRESSURE", Unit = "Pa", ByteCount = 2, ByteIndex = 2, Precision = 15, Signed = false, LowerRange = 0, UpperRange = 1013250 }); 17 | _sensorConfigs.Add(new SensorConfig() { ComponentCode = "002", SensorCode = "VIB", Unit = "Hz", ByteCount = 1, ByteIndex = 4, Precision = 1.0, Signed = false, LowerRange = 0, UpperRange = 200 }); 18 | } 19 | 20 | public IEnumerable GetConfigs() 21 | { 22 | return _sensorConfigs; 23 | } 24 | 25 | public IEnumerable GetConfigs(string componentCode) 26 | { 27 | return _sensorConfigs.Where(x => x.ComponentCode.Equals(componentCode)); 28 | } 29 | 30 | public SensorConfig GetConfig(string componentCode, string sensorCode) 31 | { 32 | return _sensorConfigs.Single(x => x.ComponentCode.Equals(componentCode) && x.SensorCode.Equals(sensorCode)); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Program.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Rx.DataBus; 2 | using InProcStreamProcessing.Rx.Decoders; 3 | using InProcStreamProcessing.Rx.Feeds; 4 | using InProcStreamProcessing.Rx.Persistence; 5 | using InProcStreamProcessing.Shared.SensorConfiguration; 6 | using System; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | 10 | namespace InProcStreamProcessing.Rx 11 | { 12 | class ShowMessages 13 | { 14 | public static bool PrintFullMessageCollection = true; 15 | public static bool PrintReader = true; 16 | public static bool PrintFileWriter = false; 17 | public static bool PrintDecoder = false; 18 | public static bool PrintRealTimeFeed = true; 19 | public static bool PrintDbPersist = true; 20 | public static bool PrintStatsFeed = false; 21 | } 22 | 23 | class Program 24 | { 25 | static void Main(string[] args) 26 | { 27 | var cts = new CancellationTokenSource(); 28 | var pipeline = new ProcessingPipeline(new DataBusReader(), 29 | new MessageFileWriter(), 30 | new Decoder(new ConfigurationLoader()), 31 | new RealTimePublisher(), 32 | new StatsFeedPublisher(), 33 | new DbPersister()); 34 | var pipelineTask = Task.Run(async () => pipeline.StartPipelineAsync(cts.Token)).Unwrap(); 35 | 36 | Console.WriteLine("Press any key to shutdown the pipeline"); 37 | Console.ReadKey(); 38 | 39 | cts.Cancel(); 40 | pipelineTask.Wait(); 41 | 42 | 43 | Task.Delay(2000).Wait(); 44 | Console.WriteLine("Pipeline shutdown"); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.DbPersister/Persistence/DbPersistence.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using System.Threading; 6 | using System.Collections.Concurrent; 7 | using System.Diagnostics; 8 | 9 | namespace DistributedStreamProcessing.ApacheKafka.DbPersister.Persistence 10 | { 11 | public class DbPersistence : IDbPersister 12 | { 13 | private int _inProgress = 0; 14 | 15 | public void StartPersisting(CancellationToken token, TimeSpan interval, BlockingCollection decodedMessages) 16 | { 17 | Task.Run(async () => await StartPeriodicPersistenceAsync(token, interval, decodedMessages)); 18 | } 19 | 20 | private async Task StartPeriodicPersistenceAsync(CancellationToken token, TimeSpan interval, BlockingCollection decodedMessages) 21 | { 22 | while(!token.IsCancellationRequested && !decodedMessages.IsAddingCompleted) 23 | { 24 | var batch = new List(); 25 | 26 | DecodedMessage message = null; 27 | while (decodedMessages.TryTake(out message)) 28 | batch.Add(message); 29 | 30 | if(batch.Any()) 31 | await PersistAsync(batch); 32 | 33 | await Task.Delay(interval); 34 | } 35 | } 36 | 37 | private async Task PersistAsync(IList messages) 38 | { 39 | Interlocked.Increment(ref _inProgress); 40 | Console.WriteLine($"Persisted batch of {messages.Count} with Counters from {messages.First().Counter} to {messages.Last().Counter}. In Progress: {_inProgress} on thread {Thread.CurrentThread.ManagedThreadId}"); 41 | 42 | await Task.Delay(100); 43 | Interlocked.Decrement(ref _inProgress); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/Program.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Serialization; 3 | using InProcStreamProcessing.Shared.SensorConfiguration; 4 | using System; 5 | using System.Collections.Concurrent; 6 | using System.Collections.Generic; 7 | using System.Diagnostics; 8 | using System.Linq; 9 | using System.Text; 10 | using System.Threading; 11 | 12 | namespace DistributedStreamProcessing.ApacheKafka.Decoder 13 | { 14 | class Program 15 | { 16 | static void Main(string[] args) 17 | { 18 | Console.Title = "Decoder"; 19 | 20 | var producerCts = new CancellationTokenSource(); 21 | var decodedMessages = new BlockingCollection(boundedCapacity: 1000); 22 | var producer = new Producer(); 23 | producer.StartProducing(producerCts.Token, decodedMessages); 24 | 25 | var consumerCts = new CancellationTokenSource(); 26 | var consumer = GetConsumer(); 27 | consumer.StartConsuming(consumerCts.Token, decodedMessages); 28 | 29 | Console.WriteLine("Press any key to shutdown"); 30 | Console.ReadKey(); 31 | 32 | consumerCts.Cancel(); 33 | 34 | // perform a shutdown, allowing for 30 seconds to process current items 35 | // this is just an example, it could end with message loss 36 | var sw = new Stopwatch(); 37 | sw.Start(); 38 | while (decodedMessages.Any()) 39 | { 40 | if (sw.ElapsedMilliseconds > 60000) 41 | producerCts.Cancel(); // force close (could lose messages) 42 | else 43 | Thread.Sleep(100); 44 | } 45 | } 46 | 47 | private static Consumer GetConsumer() 48 | { 49 | // use your favourite IoC here 50 | return new Consumer(new Decoder(new ConfigurationLoader())); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/Producer.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Serialization; 3 | using Newtonsoft.Json; 4 | using System; 5 | using System.Collections.Concurrent; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace DistributedStreamProcessing.ApacheKafka.Decoder 12 | { 13 | public class Producer 14 | { 15 | public Task StartProducing(CancellationToken token, BlockingCollection decodedMessages) 16 | { 17 | return Task.Run(async () => await ProduceAsync(token, decodedMessages)); 18 | } 19 | 20 | private async Task ProduceAsync(CancellationToken token, BlockingCollection decodedMessages) 21 | { 22 | var config = new Dictionary 23 | { 24 | { "bootstrap.servers", "localhost:9092" } 25 | }; 26 | 27 | using (var producer = new Producer(config, new StringSerializer(Encoding.UTF8), new StringSerializer(Encoding.UTF8))) 28 | { 29 | while(!token.IsCancellationRequested && !decodedMessages.IsCompleted) 30 | { 31 | DecodedMessage decodedMessage = null; 32 | try 33 | { 34 | decodedMessage = decodedMessages.Take(); // blocks until message available or BlockingCollection completed 35 | } 36 | catch (InvalidOperationException) { } 37 | 38 | if (decodedMessage != null) 39 | { 40 | var jsonText = JsonConvert.SerializeObject(decodedMessage); 41 | var messageKey = $"{decodedMessage.MachineId}-{decodedMessage.Source}-{decodedMessage.Label}"; 42 | await producer.ProduceAsync("decoded", messageKey, jsonText); 43 | 44 | Console.WriteLine($"Decoded message sent. Message Key: {messageKey}"); 45 | } 46 | } 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.WindowedStats/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Threading; 6 | 7 | namespace DistributedStreamProcessing.ApacheKafka.WindowedStats 8 | { 9 | class Program 10 | { 11 | static void Main(string[] args) 12 | { 13 | Console.Title = "Statistics"; 14 | 15 | var producerCts = new CancellationTokenSource(); 16 | 17 | var oneSecondGroupMessages = new BlockingCollection(); 18 | var oneSecondProducer = new Producer(); 19 | oneSecondProducer.StartProducing(producerCts.Token, TimeSpan.FromSeconds(1), "one-second-stats", oneSecondGroupMessages); 20 | 21 | var thirtySecondGroupMessages = new BlockingCollection(); 22 | var thirtySecondProducer = new Producer(); 23 | thirtySecondProducer.StartProducing(producerCts.Token, TimeSpan.FromSeconds(30), "thirty-second-stats", thirtySecondGroupMessages); 24 | 25 | var consumerCts = new CancellationTokenSource(); 26 | var oneSecondConsumer = new Consumer(); 27 | oneSecondConsumer.StartConsuming(consumerCts.Token, "one-second-group", oneSecondGroupMessages); 28 | 29 | var thirtySecondConsumer = new Consumer(); 30 | thirtySecondConsumer.StartConsuming(consumerCts.Token, "thirty-second-group", thirtySecondGroupMessages); 31 | 32 | Console.WriteLine("Press any key to shutdown"); 33 | Console.ReadKey(); 34 | 35 | consumerCts.Cancel(); 36 | 37 | // perform a shutdown, allowing for 30 seconds to process current items 38 | // this is just an example, it could end with message loss 39 | var sw = new Stopwatch(); 40 | sw.Start(); 41 | while (oneSecondGroupMessages.Any()) 42 | { 43 | if (sw.ElapsedMilliseconds > 60000) 44 | producerCts.Cancel(); // force close (could lose messages) 45 | else 46 | Thread.Sleep(100); 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Program.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared.SensorConfiguration; 2 | using InProcStreamProcessing.TplDataFlow.DataBus; 3 | using InProcStreamProcessing.TplDataFlow.Decoders; 4 | using InProcStreamProcessing.TplDataFlow.Feeds; 5 | using InProcStreamProcessing.TplDataFlow.Persistence; 6 | using System; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | 10 | namespace InProcStreamProcessing.TplDataFlow 11 | { 12 | class ShowMessages 13 | { 14 | public static bool PrintReader = true; 15 | public static bool PrintFileWriter = true; 16 | public static bool PrintDecoder = true; 17 | public static bool PrintRealTimeFeed = true; 18 | public static bool PrintDbPersist = true; 19 | public static bool PrintStatsFeed = true; 20 | public static bool PrintFullBuffers = true; 21 | } 22 | 23 | class Program 24 | { 25 | static void Main(string[] args) 26 | { 27 | try 28 | { 29 | var cts = new CancellationTokenSource(); 30 | var pipeline = new ProcessingPipeline(new DataBusReader(), 31 | new MessageFileWriter(), 32 | new Decoder(new ConfigurationLoader()), 33 | new RealTimePublisher(), 34 | new StatsFeedPublisher(), 35 | new DbPersister()); 36 | 37 | var pipelineTask = Task.Run(async () => { 38 | try 39 | { 40 | await pipeline.StartPipelineWithBackPressureAsync(cts.Token); 41 | } 42 | catch (Exception ex) 43 | { 44 | Console.WriteLine($"Pipeline terminated due to error {ex}"); 45 | } 46 | }); 47 | 48 | Console.WriteLine("Press any key to shutdown the pipeline"); 49 | Console.ReadKey(); 50 | 51 | cts.Cancel(); 52 | pipelineTask.Wait(); 53 | 54 | Console.WriteLine("Pipeline shutdown"); 55 | } 56 | catch (Exception ex) 57 | { 58 | 59 | } 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/Decoders/Decoder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using InProcStreamProcessing.Rx.Messages; 5 | using InProcStreamProcessing.Shared.SensorConfiguration; 6 | 7 | namespace InProcStreamProcessing.Rx.Decoders 8 | { 9 | public class Decoder : IDecoder 10 | { 11 | private IConfigurationLoader _configurationLoader; 12 | private IEnumerable _sensorConfigs; 13 | 14 | public Decoder(IConfigurationLoader configurationLoader) 15 | { 16 | _configurationLoader = configurationLoader; 17 | } 18 | 19 | public void LoadSensorConfigs() 20 | { 21 | _sensorConfigs = _configurationLoader.GetConfigs(); 22 | } 23 | 24 | public IEnumerable Decode(RawBusMessage reading) 25 | { 26 | foreach (var sensorConfig in _sensorConfigs) 27 | { 28 | yield return Decode(reading, sensorConfig); 29 | } 30 | } 31 | 32 | public DecodedMessage Decode(RawBusMessage reading, SensorConfig sensorConfig) 33 | { 34 | var baseValue = GetDecodedBaseValue(reading.Data, sensorConfig); 35 | var finalValue = baseValue * sensorConfig.Precision; 36 | 37 | if(ShowMessages.PrintDecoder) 38 | Console.WriteLine($" Decoded Message: {reading.Counter} Sensor: {sensorConfig.SensorCode} {finalValue} {sensorConfig.Unit}"); 39 | 40 | return new DecodedMessage() 41 | { 42 | Label = sensorConfig.SensorCode, 43 | ReadingTime = reading.ReadingTime, 44 | Source = sensorConfig.ComponentCode, 45 | Unit = sensorConfig.Unit, 46 | Value = finalValue, 47 | Counter = reading.Counter 48 | }; 49 | } 50 | 51 | private int GetDecodedBaseValue(byte[] data, SensorConfig sensorConfig) 52 | { 53 | if (sensorConfig.ByteCount == 1) 54 | return data[sensorConfig.ByteIndex]; 55 | else if (sensorConfig.ByteCount == 2) 56 | return BitConverter.ToInt16(data, sensorConfig.ByteIndex); 57 | else if (sensorConfig.ByteCount == 4) 58 | return BitConverter.ToInt32(data, sensorConfig.ByteIndex); 59 | 60 | return 0; 61 | } 62 | } 63 | 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.DbPersister/Consumer.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Serialization; 3 | using DistributedStreamProcessing.ApacheKafka.DbPersister.Persistence; 4 | using Newtonsoft.Json; 5 | using System; 6 | using System.Collections.Concurrent; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | 12 | namespace DistributedStreamProcessing.ApacheKafka.DbPersister 13 | { 14 | public class Consumer 15 | { 16 | public Task StartConsuming(CancellationToken token, BlockingCollection decodedMessages) 17 | { 18 | return Task.Run(() => Consume(token, decodedMessages)); 19 | } 20 | 21 | private void Consume(CancellationToken token, BlockingCollection decodedMessages) 22 | { 23 | var conf = new Dictionary 24 | { 25 | { "group.id", "db-persister-consumer-group" }, 26 | { "bootstrap.servers", "localhost:9092" }, 27 | { "auto.commit.interval.ms", 5000 }, 28 | { "auto.offset.reset", "earliest" } 29 | }; 30 | 31 | using (var consumer = new Consumer(conf, new StringDeserializer(Encoding.UTF8), new StringDeserializer(Encoding.UTF8))) 32 | { 33 | consumer.OnMessage += (_, msg) => Persist(msg.Offset.Value, msg.Key, msg.Value, decodedMessages); 34 | 35 | consumer.OnError += (_, error) 36 | => Console.WriteLine($"Error: {error}"); 37 | 38 | consumer.OnConsumeError += (_, msg) 39 | => Console.WriteLine($"Consume error ({msg.TopicPartitionOffset}): {msg.Error}"); 40 | 41 | consumer.Subscribe("decoded"); 42 | 43 | while (!token.IsCancellationRequested) 44 | { 45 | consumer.Poll(TimeSpan.FromMilliseconds(100)); 46 | } 47 | } 48 | } 49 | 50 | private void Persist(long offset, string key, string jsonMessage, BlockingCollection decodedMessages) 51 | { 52 | var decodedMessage = JsonConvert.DeserializeObject(jsonMessage); 53 | Console.WriteLine($"Decoded message added to buffer with offset: {offset}, key: {key}"); 54 | decodedMessages.Add(decodedMessage); // blocks once bounded capacity reached 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.WindowedStats/Consumer.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Serialization; 3 | using Newtonsoft.Json; 4 | using System; 5 | using System.Collections.Concurrent; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace DistributedStreamProcessing.ApacheKafka.WindowedStats 12 | { 13 | public class Consumer 14 | { 15 | public Task StartConsuming(CancellationToken token, string consumerGroup, BlockingCollection decodedMessages) 16 | { 17 | return Task.Run(() => Consume(token, consumerGroup, decodedMessages)); 18 | } 19 | 20 | private void Consume(CancellationToken token, string consumerGroup, BlockingCollection decodedMessages) 21 | { 22 | var conf = new Dictionary 23 | { 24 | { "group.id", consumerGroup }, 25 | { "bootstrap.servers", "localhost:9092" }, 26 | { "auto.commit.interval.ms", 5000 }, 27 | { "auto.offset.reset", "earliest" } 28 | }; 29 | 30 | using (var consumer = new Consumer(conf, new StringDeserializer(Encoding.UTF8), new StringDeserializer(Encoding.UTF8))) 31 | { 32 | consumer.OnMessage += (_, msg) => Publish(msg.Offset.Value, msg.Key, msg.Value, decodedMessages, consumerGroup); 33 | 34 | consumer.OnError += (_, error) 35 | => Console.WriteLine($"Error: {error}"); 36 | 37 | consumer.OnConsumeError += (_, msg) 38 | => Console.WriteLine($"Consume error ({msg.TopicPartitionOffset}): {msg.Error}"); 39 | 40 | consumer.Subscribe("decoded"); 41 | 42 | while (!token.IsCancellationRequested) 43 | { 44 | consumer.Poll(TimeSpan.FromMilliseconds(100)); 45 | } 46 | } 47 | } 48 | 49 | private void Publish(long offset, string key, string jsonMessage, BlockingCollection decodedMessages, string consumerGroup) 50 | { 51 | var decodedMessage = JsonConvert.DeserializeObject(jsonMessage); 52 | Console.WriteLine($"Decoded message added to buffer of {consumerGroup} with offset: {offset}, key: {key}"); 53 | decodedMessages.Add(decodedMessage); // blocks once bounded capacity reached 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/Decoder.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared.SensorConfiguration; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace DistributedStreamProcessing.ApacheKafka.Decoder 7 | { 8 | public class Decoder : IDecoder 9 | { 10 | private IConfigurationLoader _configurationLoader; 11 | private IEnumerable _sensorConfigs; 12 | 13 | public Decoder(IConfigurationLoader configurationLoader) 14 | { 15 | _configurationLoader = configurationLoader; 16 | } 17 | 18 | public void LoadSensorConfigs() 19 | { 20 | _sensorConfigs = _configurationLoader.GetConfigs(); 21 | } 22 | 23 | public IEnumerable Decode(RawBusMessage reading) 24 | { 25 | int decodeCounter = 0; 26 | foreach (var sensorConfig in _sensorConfigs) 27 | { 28 | yield return Decode(reading, sensorConfig, decodeCounter); 29 | decodeCounter++; 30 | } 31 | } 32 | 33 | public DecodedMessage Decode(RawBusMessage reading, SensorConfig sensorConfig, int decodeCounter) 34 | { 35 | var baseValue = GetDecodedBaseValue(reading.Data, sensorConfig); 36 | var finalValue = baseValue * sensorConfig.Precision; 37 | 38 | Console.WriteLine($" Decoded Message: {reading.Counter} Sensor: {sensorConfig.SensorCode} {finalValue} {sensorConfig.Unit}"); 39 | 40 | return new DecodedMessage() 41 | { 42 | Label = sensorConfig.SensorCode, 43 | ReadingTime = reading.ReadingTime, 44 | Source = sensorConfig.ComponentCode, 45 | MachineId = reading.MachineId, 46 | Unit = sensorConfig.Unit, 47 | Value = finalValue, 48 | Counter = (double)reading.Counter + ((double)decodeCounter / 100.0) 49 | }; 50 | } 51 | 52 | private int GetDecodedBaseValue(byte[] data, SensorConfig sensorConfig) 53 | { 54 | if (sensorConfig.ByteCount == 1) 55 | return data[sensorConfig.ByteIndex]; 56 | else if (sensorConfig.ByteCount == 2) 57 | return BitConverter.ToInt16(data, sensorConfig.ByteIndex); 58 | else if (sensorConfig.ByteCount == 4) 59 | return BitConverter.ToInt32(data, sensorConfig.ByteIndex); 60 | 61 | return 0; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/Decoders/Decoder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using InProcStreamProcessing.TplDataFlow.Messages; 5 | using InProcStreamProcessing.Shared.SensorConfiguration; 6 | 7 | namespace InProcStreamProcessing.TplDataFlow.Decoders 8 | { 9 | public class Decoder : IDecoder 10 | { 11 | private IConfigurationLoader _configurationLoader; 12 | private IEnumerable _sensorConfigs; 13 | 14 | public Decoder(IConfigurationLoader configurationLoader) 15 | { 16 | _configurationLoader = configurationLoader; 17 | } 18 | 19 | public void LoadSensorConfigs() 20 | { 21 | _sensorConfigs = _configurationLoader.GetConfigs(); 22 | } 23 | 24 | public IEnumerable Decode(RawBusMessage reading) 25 | { 26 | int decodeCounter = 0; 27 | foreach (var sensorConfig in _sensorConfigs) 28 | { 29 | yield return Decode(reading, sensorConfig, decodeCounter); 30 | decodeCounter++; 31 | } 32 | } 33 | 34 | public DecodedMessage Decode(RawBusMessage reading, SensorConfig sensorConfig, int decodeCounter) 35 | { 36 | var baseValue = GetDecodedBaseValue(reading.Data, sensorConfig); 37 | var finalValue = baseValue * sensorConfig.Precision; 38 | 39 | if(ShowMessages.PrintDecoder) 40 | Console.WriteLine($" Decoded Message: {reading.Counter} Sensor: {sensorConfig.SensorCode} {finalValue} {sensorConfig.Unit}"); 41 | 42 | return new DecodedMessage() 43 | { 44 | Label = sensorConfig.SensorCode, 45 | ReadingTime = reading.ReadingTime, 46 | Source = sensorConfig.ComponentCode, 47 | Unit = sensorConfig.Unit, 48 | Value = finalValue, 49 | Counter = (double)reading.Counter + ((double)decodeCounter/100.0) 50 | }; 51 | } 52 | 53 | private int GetDecodedBaseValue(byte[] data, SensorConfig sensorConfig) 54 | { 55 | if (sensorConfig.ByteCount == 1) 56 | return data[sensorConfig.ByteIndex]; 57 | else if (sensorConfig.ByteCount == 2) 58 | return BitConverter.ToInt16(data, sensorConfig.ByteIndex); 59 | else if (sensorConfig.ByteCount == 4) 60 | return BitConverter.ToInt32(data, sensorConfig.ByteIndex); 61 | 62 | return 0; 63 | } 64 | } 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka/DataBusReader.cs: -------------------------------------------------------------------------------- 1 | 2 | using Confluent.Kafka; 3 | using Confluent.Kafka.Serialization; 4 | using InProcStreamProcessing.Shared; 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | 12 | namespace DistributedStreamProcessing.ApacheKafka.Producer 13 | { 14 | public class DataBusReader 15 | { 16 | private DataBusInterface _dataBus; 17 | private int _counter; 18 | 19 | public DataBusReader() 20 | { 21 | _dataBus = new DataBusInterface(); 22 | _dataBus.Initialize(); 23 | } 24 | 25 | public Task StartProducing(CancellationToken token, TimeSpan interval) 26 | { 27 | return Task.Factory.StartNew(() => Produce(token, interval), TaskCreationOptions.LongRunning); 28 | } 29 | 30 | private void Produce(CancellationToken token, TimeSpan interval) 31 | { 32 | var config = new Dictionary 33 | { 34 | { "bootstrap.servers", "localhost:9092" } 35 | }; 36 | 37 | var machineId = _dataBus.GetMachineId(); 38 | 39 | using (var producer = new Producer(config, new StringSerializer(Encoding.UTF8), new StringSerializer(Encoding.UTF8))) 40 | { 41 | long lastTicks = 0; 42 | while (!token.IsCancellationRequested) 43 | { 44 | _counter++; 45 | var reading = _dataBus.Read(); 46 | 47 | Console.WriteLine($"Read message {_counter} on thread {Thread.CurrentThread.ManagedThreadId}"); 48 | 49 | var message = new RawBusMessage(); 50 | message.Data = reading.Data; 51 | message.ReadingTime = new DateTime(reading.Ticks); 52 | message.Counter = _counter; 53 | message.MachineId = machineId; 54 | 55 | if (lastTicks < reading.Ticks) 56 | { 57 | var jsonText = JsonConvert.SerializeObject(message); 58 | var sendResult = producer.ProduceAsync(topic: "raw", key: machineId, val: jsonText, blockIfQueueFull: true).Result; 59 | if(sendResult.Error.HasError) 60 | Console.WriteLine("Could not send: " + sendResult.Error.Reason); 61 | } 62 | 63 | lastTicks = reading.Ticks; 64 | 65 | Thread.Sleep(interval); 66 | } 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.Decoder/Consumer.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Serialization; 3 | using Newtonsoft.Json; 4 | using System; 5 | using System.Collections.Concurrent; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace DistributedStreamProcessing.ApacheKafka.Decoder 12 | { 13 | public class Consumer 14 | { 15 | private readonly IDecoder _decoder; 16 | 17 | public Consumer(IDecoder decoder) 18 | { 19 | _decoder = decoder; 20 | } 21 | 22 | public Task StartConsuming(CancellationToken token, BlockingCollection decodedMessages) 23 | { 24 | _decoder.LoadSensorConfigs(); 25 | return Task.Run(() => Consume(token, decodedMessages)); 26 | } 27 | 28 | private void Consume(CancellationToken token, BlockingCollection decodedMessages) 29 | { 30 | var conf = new Dictionary 31 | { 32 | { "group.id", "decoder-consumer-group" }, 33 | { "bootstrap.servers", "localhost:9092" }, 34 | { "auto.commit.interval.ms", 5000 }, 35 | { "auto.offset.reset", "earliest" } 36 | }; 37 | 38 | using (var consumer = new Consumer(conf, new StringDeserializer(Encoding.UTF8), new StringDeserializer(Encoding.UTF8))) 39 | { 40 | consumer.OnMessage += (_, msg) => Decode(msg.Offset.Value, msg.Key, msg.Value, decodedMessages); 41 | 42 | consumer.OnError += (_, error) 43 | => Console.WriteLine($"Error: {error}"); 44 | 45 | consumer.OnConsumeError += (_, msg) 46 | => Console.WriteLine($"Consume error ({msg.TopicPartitionOffset}): {msg.Error}"); 47 | 48 | consumer.Subscribe("raw"); 49 | 50 | while (!token.IsCancellationRequested) 51 | { 52 | consumer.Poll(TimeSpan.FromMilliseconds(100)); 53 | } 54 | } 55 | 56 | decodedMessages.CompleteAdding(); // notifies consumers that no more messages will come 57 | } 58 | 59 | private void Decode(long offset, string key, string jsonMessage, BlockingCollection decodedMessages) 60 | { 61 | Console.WriteLine($"Raw message consumed with offset: {offset}, key: {key}"); 62 | var rawBusMessage = JsonConvert.DeserializeObject(jsonMessage); 63 | var decoded = _decoder.Decode(rawBusMessage); 64 | 65 | foreach (var decodedMessage in decoded) 66 | decodedMessages.Add(decodedMessage); // blocks once bounded capacity reached 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/net-apps/DistributedStreamProcessing.ApacheKafka.WindowedStats/Producer.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Serialization; 3 | using Newtonsoft.Json; 4 | using System; 5 | using System.Collections.Concurrent; 6 | using System.Collections.Generic; 7 | using System.Diagnostics; 8 | using System.Linq; 9 | using System.Text; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | 13 | namespace DistributedStreamProcessing.ApacheKafka.WindowedStats 14 | { 15 | public class Producer 16 | { 17 | public void StartProducing(CancellationToken token, TimeSpan windowSize, string topic, BlockingCollection decodedMessages) 18 | { 19 | Task.Run(async () => await ProduceAsync(token, windowSize, topic, decodedMessages)); 20 | } 21 | 22 | private async Task ProduceAsync(CancellationToken token, TimeSpan windowSize, string topic, BlockingCollection decodedMessages) 23 | { 24 | var config = new Dictionary 25 | { 26 | { "bootstrap.servers", "localhost:9092" } 27 | }; 28 | 29 | using (var producer = new Producer(config, new StringSerializer(Encoding.UTF8), new StringSerializer(Encoding.UTF8))) 30 | { 31 | var sw = new Stopwatch(); 32 | sw.Start(); 33 | 34 | while (!token.IsCancellationRequested && !decodedMessages.IsCompleted) 35 | { 36 | var batch = new List(); 37 | 38 | DecodedMessage message = null; 39 | while (sw.Elapsed < windowSize) 40 | { 41 | if(decodedMessages.TryTake(out message, TimeSpan.FromMilliseconds(100))) 42 | batch.Add(message); 43 | } 44 | 45 | sw.Restart(); 46 | if (batch.Any()) 47 | { 48 | var grouped = batch.GroupBy(x => new { x.MachineId, x.Source, x.Label }); 49 | foreach (var group in grouped) 50 | { 51 | var statsMessage = new StatsWindowMessage(); 52 | statsMessage.MachineId = group.Key.MachineId; 53 | statsMessage.Source = group.Key.Source; 54 | statsMessage.Label = group.Key.Label; 55 | statsMessage.Unit = group.First().Unit; 56 | statsMessage.PeriodStart = group.Min(x => x.ReadingTime); 57 | statsMessage.MinValue = group.Min(x => x.Value); 58 | statsMessage.MaxValue = group.Max(x => x.Value); 59 | statsMessage.AvgValue = group.Average(x => x.Value); 60 | statsMessage.Count = group.Count(); 61 | 62 | var jsonText = JsonConvert.SerializeObject(statsMessage); 63 | var messageKey = $"{statsMessage.MachineId}-{statsMessage.Source}-{statsMessage.Label}"; 64 | await producer.ProduceAsync(topic, messageKey, jsonText); 65 | Console.WriteLine($"Published to topic {topic} with key: {messageKey}"); 66 | } 67 | } 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/DataBus/DataBusReader.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared; 2 | using InProcStreamProcessing.TplDataFlow.Messages; 3 | using System; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | 8 | namespace InProcStreamProcessing.TplDataFlow.DataBus 9 | { 10 | public class DataBusReader : IDataBusReader 11 | { 12 | private DataBusInterface _dataBus; 13 | private int _counter; 14 | 15 | public DataBusReader() 16 | { 17 | _dataBus = new DataBusInterface(); 18 | _dataBus.Initialize(); 19 | } 20 | 21 | public Task StartConsuming(ITargetBlock target, CancellationToken token, TimeSpan interval, FlowControlMode flowControlMode) 22 | { 23 | if(flowControlMode == FlowControlMode.LoadShed) 24 | return Task.Factory.StartNew(() => ConsumeWithDiscard(target, token, interval), TaskCreationOptions.LongRunning); 25 | else 26 | return Task.Factory.StartNew(() => ConsumeWithBackPressure(target, token, interval), TaskCreationOptions.LongRunning); 27 | } 28 | 29 | private void ConsumeWithDiscard(ITargetBlock target, CancellationToken token, TimeSpan interval) 30 | { 31 | long lastTicks = 0; 32 | while (!token.IsCancellationRequested) 33 | { 34 | _counter++; 35 | var reading = _dataBus.Read(); 36 | 37 | if(ShowMessages.PrintReader) 38 | Console.WriteLine($"Read message {_counter} on thread {Thread.CurrentThread.ManagedThreadId}"); 39 | 40 | var message = new RawBusMessage(); 41 | message.Data = reading.Data; 42 | message.ReadingTime = new DateTime(reading.Ticks); 43 | message.Counter = _counter; 44 | 45 | if (lastTicks < reading.Ticks) 46 | { 47 | var posted = target.Post(message); 48 | if (!posted && ShowMessages.PrintFullBuffers) 49 | Console.WriteLine("Buffer full. Could not post"); 50 | } 51 | 52 | lastTicks = reading.Ticks; 53 | 54 | Thread.Sleep(interval); 55 | } 56 | } 57 | 58 | private void ConsumeWithBackPressure(ITargetBlock target, CancellationToken token, TimeSpan interval) 59 | { 60 | long lastTicks = 0; 61 | while (!token.IsCancellationRequested) 62 | { 63 | _counter++; 64 | var reading = _dataBus.Read(); 65 | 66 | if (ShowMessages.PrintReader) 67 | Console.WriteLine($"Read message {_counter} on thread {Thread.CurrentThread.ManagedThreadId}"); 68 | 69 | var message = new RawBusMessage(); 70 | message.Data = reading.Data; 71 | message.ReadingTime = new DateTime(reading.Ticks); 72 | message.Counter = _counter; 73 | 74 | if (lastTicks < reading.Ticks) 75 | { 76 | while (!target.Post(message)) 77 | Thread.Sleep((int)interval.TotalMilliseconds); 78 | } 79 | 80 | lastTicks = reading.Ticks; 81 | 82 | Thread.Sleep(interval); 83 | } 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Shared/DataBusInterface.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared.SensorConfiguration; 2 | using System; 3 | using System.Collections.Generic; 4 | 5 | namespace InProcStreamProcessing.Shared 6 | { 7 | public class DataBusInterface 8 | { 9 | private Dictionary _maxValues; 10 | private Dictionary _lastSensorValues; 11 | private Random _random; 12 | private ConfigurationLoader _configLoader; 13 | 14 | public void Initialize() 15 | { 16 | _configLoader = new ConfigurationLoader(); 17 | 18 | _maxValues = new Dictionary(); 19 | _maxValues.Add("TEMP", 30000); 20 | _maxValues.Add("PRESSURE", 50000); 21 | _maxValues.Add("VIB", 200); 22 | 23 | _lastSensorValues = new Dictionary(); 24 | _lastSensorValues.Add("TEMP", 0); 25 | _lastSensorValues.Add("PRESSURE", _maxValues["PRESSURE"] / 2); 26 | _lastSensorValues.Add("VIB", _maxValues["VIB"] / 2); 27 | 28 | _random = new Random(); 29 | } 30 | 31 | public string GetMachineId() 32 | { 33 | return "MD63FO1"; 34 | } 35 | 36 | public BusMessage Read() 37 | { 38 | // here we would interact with a C library to read the current value 39 | // but we'll just fake something 40 | 41 | var now = DateTime.Now; 42 | var reading = new BusMessage(); 43 | reading.XYZ = Guid.NewGuid().ToString(); 44 | reading.Ticks = now.Ticks; 45 | reading.Data = new byte[8]; 46 | 47 | var tempVal = GetTwoByteEncoding(GetNextValue("001", "TEMP")); 48 | var pressureVal = GetTwoByteEncoding(GetNextValue("001", "PRESSURE")); 49 | var vib = GetSingleByteEncoding(GetNextValue("002", "VIB")); 50 | 51 | reading.Data[0] = tempVal[0]; 52 | reading.Data[1] = tempVal[1]; 53 | reading.Data[2] = pressureVal[0]; 54 | reading.Data[3] = pressureVal[1]; 55 | reading.Data[4] = vib; 56 | 57 | return reading; 58 | } 59 | 60 | private byte GetSingleByteEncoding(int value) 61 | { 62 | return BitConverter.GetBytes(value)[0]; 63 | } 64 | 65 | private byte[] GetTwoByteEncoding(int value) 66 | { 67 | var bytes = BitConverter.GetBytes(value); 68 | return new byte[] { bytes[0], bytes[1] }; 69 | } 70 | 71 | private int GetNextValue(string componentCode, string sensorCode) 72 | { 73 | var sensorConfig = _configLoader.GetConfig(componentCode, sensorCode); 74 | var lastValue = _lastSensorValues[sensorCode]; 75 | double modifier = (_random.NextDouble() / 100.0); // up to 1% 76 | if (_random.Next(10) % 2 == 1) 77 | modifier = 1.0 + modifier; 78 | else 79 | modifier = 1.0 - modifier; 80 | 81 | double newValue = lastValue * modifier; 82 | 83 | var maxVal = _maxValues[sensorCode]; 84 | if (sensorConfig.Signed) 85 | { 86 | if (newValue >= maxVal) 87 | newValue -= maxVal * 0.1; 88 | else if (newValue <= -_maxValues[sensorCode]) 89 | newValue += maxVal * 0.1; 90 | } 91 | else 92 | { 93 | if (newValue >= _maxValues[sensorCode]) 94 | newValue = maxVal * 0.9; 95 | } 96 | 97 | _lastSensorValues[sensorCode] = newValue; 98 | 99 | return (int)newValue; 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/net-apps/StreamProcessing.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.27004.2009 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "InProcStreamProcessing.TplDataFlow", "InProcStreamProcessing.TplDataflow\InProcStreamProcessing.TplDataFlow.csproj", "{F789A381-A560-4425-8A71-4BFC6FE10F21}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "InProcStreamProcessing.Shared", "InProcStreamProcessing.Shared\InProcStreamProcessing.Shared.csproj", "{52E8EA22-DA63-4EE4-ABA9-7A0DF9C2B117}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "InProcStreamProcessing.Rx", "InProcStreamProcessing.Rx\InProcStreamProcessing.Rx.csproj", "{C0BA8EC0-35F2-4E86-9EE3-64583482E630}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistributedStreamProcessing.ApacheKafka.Producer", "DistributedStreamProcessing.ApacheKafka\DistributedStreamProcessing.ApacheKafka.Producer.csproj", "{AB519854-E873-41CD-B015-4AC35B5CA3B8}" 13 | EndProject 14 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Kafka", "Kafka", "{462B6AC2-B47F-4733-A9F7-96CAFAB28EB8}" 15 | EndProject 16 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistributedStreamProcessing.ApacheKafka.Decoder", "DistributedStreamProcessing.ApacheKafka.Decoder\DistributedStreamProcessing.ApacheKafka.Decoder.csproj", "{5FA54331-8350-422D-BA7D-ADC4AE368DDC}" 17 | EndProject 18 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistributedStreamProcessing.ApacheKafka.DbPersister", "DistributedStreamProcessing.ApacheKafka.DbPersister\DistributedStreamProcessing.ApacheKafka.DbPersister.csproj", "{CA7320F0-62ED-48B2-B19B-68CEA9F12B07}" 19 | EndProject 20 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistributedStreamProcessing.ApacheKafka.WindowedStats", "DistributedStreamProcessing.ApacheKafka.WindowedStats\DistributedStreamProcessing.ApacheKafka.WindowedStats.csproj", "{0DC6F722-3486-4225-88ED-F86F03F57545}" 21 | EndProject 22 | Global 23 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 24 | Debug|Any CPU = Debug|Any CPU 25 | Release|Any CPU = Release|Any CPU 26 | EndGlobalSection 27 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 28 | {F789A381-A560-4425-8A71-4BFC6FE10F21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {F789A381-A560-4425-8A71-4BFC6FE10F21}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {F789A381-A560-4425-8A71-4BFC6FE10F21}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {F789A381-A560-4425-8A71-4BFC6FE10F21}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {52E8EA22-DA63-4EE4-ABA9-7A0DF9C2B117}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {52E8EA22-DA63-4EE4-ABA9-7A0DF9C2B117}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {52E8EA22-DA63-4EE4-ABA9-7A0DF9C2B117}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {52E8EA22-DA63-4EE4-ABA9-7A0DF9C2B117}.Release|Any CPU.Build.0 = Release|Any CPU 36 | {C0BA8EC0-35F2-4E86-9EE3-64583482E630}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 37 | {C0BA8EC0-35F2-4E86-9EE3-64583482E630}.Debug|Any CPU.Build.0 = Debug|Any CPU 38 | {C0BA8EC0-35F2-4E86-9EE3-64583482E630}.Release|Any CPU.ActiveCfg = Release|Any CPU 39 | {C0BA8EC0-35F2-4E86-9EE3-64583482E630}.Release|Any CPU.Build.0 = Release|Any CPU 40 | {AB519854-E873-41CD-B015-4AC35B5CA3B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 41 | {AB519854-E873-41CD-B015-4AC35B5CA3B8}.Debug|Any CPU.Build.0 = Debug|Any CPU 42 | {AB519854-E873-41CD-B015-4AC35B5CA3B8}.Release|Any CPU.ActiveCfg = Release|Any CPU 43 | {AB519854-E873-41CD-B015-4AC35B5CA3B8}.Release|Any CPU.Build.0 = Release|Any CPU 44 | {5FA54331-8350-422D-BA7D-ADC4AE368DDC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 45 | {5FA54331-8350-422D-BA7D-ADC4AE368DDC}.Debug|Any CPU.Build.0 = Debug|Any CPU 46 | {5FA54331-8350-422D-BA7D-ADC4AE368DDC}.Release|Any CPU.ActiveCfg = Release|Any CPU 47 | {5FA54331-8350-422D-BA7D-ADC4AE368DDC}.Release|Any CPU.Build.0 = Release|Any CPU 48 | {CA7320F0-62ED-48B2-B19B-68CEA9F12B07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 49 | {CA7320F0-62ED-48B2-B19B-68CEA9F12B07}.Debug|Any CPU.Build.0 = Debug|Any CPU 50 | {CA7320F0-62ED-48B2-B19B-68CEA9F12B07}.Release|Any CPU.ActiveCfg = Release|Any CPU 51 | {CA7320F0-62ED-48B2-B19B-68CEA9F12B07}.Release|Any CPU.Build.0 = Release|Any CPU 52 | {0DC6F722-3486-4225-88ED-F86F03F57545}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 53 | {0DC6F722-3486-4225-88ED-F86F03F57545}.Debug|Any CPU.Build.0 = Debug|Any CPU 54 | {0DC6F722-3486-4225-88ED-F86F03F57545}.Release|Any CPU.ActiveCfg = Release|Any CPU 55 | {0DC6F722-3486-4225-88ED-F86F03F57545}.Release|Any CPU.Build.0 = Release|Any CPU 56 | EndGlobalSection 57 | GlobalSection(SolutionProperties) = preSolution 58 | HideSolutionNode = FALSE 59 | EndGlobalSection 60 | GlobalSection(NestedProjects) = preSolution 61 | {AB519854-E873-41CD-B015-4AC35B5CA3B8} = {462B6AC2-B47F-4733-A9F7-96CAFAB28EB8} 62 | {5FA54331-8350-422D-BA7D-ADC4AE368DDC} = {462B6AC2-B47F-4733-A9F7-96CAFAB28EB8} 63 | {CA7320F0-62ED-48B2-B19B-68CEA9F12B07} = {462B6AC2-B47F-4733-A9F7-96CAFAB28EB8} 64 | {0DC6F722-3486-4225-88ED-F86F03F57545} = {462B6AC2-B47F-4733-A9F7-96CAFAB28EB8} 65 | EndGlobalSection 66 | GlobalSection(ExtensibilityGlobals) = postSolution 67 | SolutionGuid = {D4544EF7-7C0E-4AC9-BA97-129D946F81CF} 68 | EndGlobalSection 69 | EndGlobal 70 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/DataBus/DataBusReader.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared; 2 | using System; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using System.Linq; 6 | using System.Reactive.Linq; 7 | using System.Reactive; 8 | using InProcStreamProcessing.Rx.Messages; 9 | using System.Reactive.Subjects; 10 | using System.Reactive.Disposables; 11 | using System.Reactive.Concurrency; 12 | using System.Collections.Concurrent; 13 | 14 | namespace InProcStreamProcessing.Rx.DataBus 15 | { 16 | public class DataBusReader : IDataBusReader 17 | { 18 | private DataBusInterface _dataBus; 19 | private int _counter = 0; 20 | 21 | public DataBusReader() 22 | { 23 | _dataBus = new DataBusInterface(); 24 | _dataBus.Initialize(); 25 | } 26 | 27 | public IObservable StartConsuming(CancellationToken token, TimeSpan interval) 28 | { 29 | //----------------------------------- 30 | //--------- Observable.Generate 31 | var scheduler = new NewThreadScheduler(ts => new Thread(ts) { Name = "DataBusPoller" }); 32 | var source = Observable.Generate(Read(), 33 | x => !token.IsCancellationRequested, 34 | x => Read(), 35 | x => x, 36 | x => interval, 37 | scheduler); 38 | 39 | return source; 40 | 41 | //----------------------------------- 42 | //--------- Observable.Create 43 | //IObservable source = Observable.Create(async (IObserver observer) => 44 | //{ 45 | // try 46 | // { 47 | // while (!token.IsCancellationRequested) 48 | // { 49 | // observer.OnNext(Read()); 50 | // await Task.Delay(interval); 51 | // } 52 | 53 | // observer.OnCompleted(); 54 | // } 55 | // catch (Exception ex) 56 | // { 57 | // observer.OnError(ex); 58 | // } 59 | 60 | // return Disposable.Empty; 61 | //}); 62 | 63 | //return source; 64 | 65 | //----------------------------------- 66 | //--------- Observable.Timer 67 | //var scheduler = new EventLoopScheduler(ts => new Thread(ts) { Name = "DataBusPoller" }); 68 | 69 | //var query = Observable.Timer(interval, scheduler) 70 | // .Select(_ => Read()) 71 | // .Repeat(); 72 | 73 | //return query; 74 | } 75 | 76 | // I didn't use this, but you can also produce to a BlockingCollection and turn that into an observable 77 | public BlockingCollection StartProducing(CancellationToken token, TimeSpan interval) 78 | { 79 | var messages = new BlockingCollection(100000); 80 | Task.Factory.StartNew(() => StartProducingToCollection(token, interval, messages), TaskCreationOptions.LongRunning); 81 | 82 | return messages; 83 | } 84 | 85 | // I didn't use this, but you can also produce to a BlockingCollection and turn that into an observable 86 | private void StartProducingToCollection(CancellationToken token, TimeSpan interval, BlockingCollection messages) 87 | { 88 | long lastTicks = 0; 89 | while (!token.IsCancellationRequested) 90 | { 91 | _counter++; 92 | var reading = _dataBus.Read(); 93 | 94 | if (ShowMessages.PrintReader) 95 | Console.WriteLine($"Read message {_counter} on thread {Thread.CurrentThread.ManagedThreadId}"); 96 | 97 | var message = new RawBusMessage(); 98 | message.Data = reading.Data; 99 | message.ReadingTime = new DateTime(reading.Ticks); 100 | message.Counter = _counter; 101 | 102 | if (_counter % 10000 == 0) 103 | Console.WriteLine("COLLECTION: " + messages.Count); 104 | 105 | if (lastTicks < reading.Ticks) 106 | { 107 | var posted = messages.TryAdd(message); 108 | if (!posted && ShowMessages.PrintFullMessageCollection) 109 | Console.WriteLine("Buffer full. Could not post"); 110 | } 111 | 112 | lastTicks = reading.Ticks; 113 | 114 | Thread.Sleep(interval); 115 | } 116 | } 117 | 118 | 119 | private RawBusMessage Read() 120 | { 121 | _counter++; 122 | var reading = _dataBus.Read(); 123 | if(ShowMessages.PrintReader) 124 | Console.WriteLine($"Read message {_counter} on thread {Thread.CurrentThread.ManagedThreadId}"); 125 | 126 | var message = new RawBusMessage(); 127 | message.Data = reading.Data; 128 | message.ReadingTime = new DateTime(reading.Ticks); 129 | message.Counter = _counter; 130 | 131 | return message; 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # MSTest test Results 33 | [Tt]est[Rr]esult*/ 34 | [Bb]uild[Ll]og.* 35 | 36 | # NUNIT 37 | *.VisualState.xml 38 | TestResult.xml 39 | 40 | # Build Results of an ATL Project 41 | [Dd]ebugPS/ 42 | [Rr]eleasePS/ 43 | dlldata.c 44 | 45 | # .NET Core 46 | project.lock.json 47 | project.fragment.lock.json 48 | artifacts/ 49 | **/Properties/launchSettings.json 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # Visual Studio code coverage results 117 | *.coverage 118 | *.coveragexml 119 | 120 | # NCrunch 121 | _NCrunch_* 122 | .*crunch*.local.xml 123 | nCrunchTemp_* 124 | 125 | # MightyMoose 126 | *.mm.* 127 | AutoTest.Net/ 128 | 129 | # Web workbench (sass) 130 | .sass-cache/ 131 | 132 | # Installshield output folder 133 | [Ee]xpress/ 134 | 135 | # DocProject is a documentation generator add-in 136 | DocProject/buildhelp/ 137 | DocProject/Help/*.HxT 138 | DocProject/Help/*.HxC 139 | DocProject/Help/*.hhc 140 | DocProject/Help/*.hhk 141 | DocProject/Help/*.hhp 142 | DocProject/Help/Html2 143 | DocProject/Help/html 144 | 145 | # Click-Once directory 146 | publish/ 147 | 148 | # Publish Web Output 149 | *.[Pp]ublish.xml 150 | *.azurePubxml 151 | # TODO: Comment the next line if you want to checkin your web deploy settings 152 | # but database connection strings (with potential passwords) will be unencrypted 153 | *.pubxml 154 | *.publishproj 155 | 156 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 157 | # checkin your Azure Web App publish settings, but sensitive information contained 158 | # in these scripts will be unencrypted 159 | PublishScripts/ 160 | 161 | # NuGet Packages 162 | *.nupkg 163 | # The packages folder can be ignored because of Package Restore 164 | **/packages/* 165 | # except build/, which is used as an MSBuild target. 166 | !**/packages/build/ 167 | # Uncomment if necessary however generally it will be regenerated when needed 168 | #!**/packages/repositories.config 169 | # NuGet v3's project.json files produces more ignorable files 170 | *.nuget.props 171 | *.nuget.targets 172 | 173 | # Microsoft Azure Build Output 174 | csx/ 175 | *.build.csdef 176 | 177 | # Microsoft Azure Emulator 178 | ecf/ 179 | rcf/ 180 | 181 | # Windows Store app package directories and files 182 | AppPackages/ 183 | BundleArtifacts/ 184 | Package.StoreAssociation.xml 185 | _pkginfo.txt 186 | 187 | # Visual Studio cache files 188 | # files ending in .cache can be ignored 189 | *.[Cc]ache 190 | # but keep track of directories ending in .cache 191 | !*.[Cc]ache/ 192 | 193 | # Others 194 | ClientBin/ 195 | ~$* 196 | *~ 197 | *.dbmdl 198 | *.dbproj.schemaview 199 | *.jfm 200 | *.pfx 201 | *.publishsettings 202 | orleans.codegen.cs 203 | 204 | # Since there are multiple workflows, uncomment next line to ignore bower_components 205 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 206 | #bower_components/ 207 | 208 | # RIA/Silverlight projects 209 | Generated_Code/ 210 | 211 | # Backup & report files from converting an old project file 212 | # to a newer Visual Studio version. Backup files are not needed, 213 | # because we have git ;-) 214 | _UpgradeReport_Files/ 215 | Backup*/ 216 | UpgradeLog*.XML 217 | UpgradeLog*.htm 218 | 219 | # SQL Server files 220 | *.mdf 221 | *.ldf 222 | *.ndf 223 | 224 | # Business Intelligence projects 225 | *.rdl.data 226 | *.bim.layout 227 | *.bim_*.settings 228 | 229 | # Microsoft Fakes 230 | FakesAssemblies/ 231 | 232 | # GhostDoc plugin setting file 233 | *.GhostDoc.xml 234 | 235 | # Node.js Tools for Visual Studio 236 | .ntvs_analysis.dat 237 | node_modules/ 238 | 239 | # Typescript v1 declaration files 240 | typings/ 241 | 242 | # Visual Studio 6 build log 243 | *.plg 244 | 245 | # Visual Studio 6 workspace options file 246 | *.opt 247 | 248 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 249 | *.vbw 250 | 251 | # Visual Studio LightSwitch build output 252 | **/*.HTMLClient/GeneratedArtifacts 253 | **/*.DesktopClient/GeneratedArtifacts 254 | **/*.DesktopClient/ModelManifest.xml 255 | **/*.Server/GeneratedArtifacts 256 | **/*.Server/ModelManifest.xml 257 | _Pvt_Extensions 258 | 259 | # Paket dependency manager 260 | .paket/paket.exe 261 | paket-files/ 262 | 263 | # FAKE - F# Make 264 | .fake/ 265 | 266 | # JetBrains Rider 267 | .idea/ 268 | *.sln.iml 269 | 270 | # CodeRush 271 | .cr/ 272 | 273 | # Python Tools for Visual Studio (PTVS) 274 | __pycache__/ 275 | *.pyc 276 | 277 | # Cake - Uncomment if you are using it 278 | # tools/** 279 | # !tools/packages.config 280 | 281 | # Telerik's JustMock configuration file 282 | *.jmconfig 283 | 284 | # BizTalk build output 285 | *.btp.cs 286 | *.btm.cs 287 | *.odx.cs 288 | *.xsd.cs 289 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.Rx/ProcessingPipeline.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Rx.DataBus; 2 | using InProcStreamProcessing.Rx.Messages; 3 | using System; 4 | using System.Reactive.Concurrency; 5 | using System.Reactive.PlatformServices; 6 | using System.Reactive.Linq; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | using System.Linq; 10 | using System.Collections.Generic; 11 | using InProcStreamProcessing.Rx.Persistence; 12 | using InProcStreamProcessing.Rx.Feeds; 13 | using InProcStreamProcessing.Rx.Decoders; 14 | using System.Reactive; 15 | 16 | namespace InProcStreamProcessing.Rx 17 | { 18 | public class ProcessingPipeline 19 | { 20 | private IDataBusReader _dataBusReader; 21 | private IMessageFileWriter _messageFileWriter; 22 | private IDecoder _decoder; 23 | private IRealTimePublisher _realTimeFeedPublisher; 24 | private IStatsFeedPublisher _statsFeedPublisher; 25 | private IDbPersister _dbPersister; 26 | 27 | public ProcessingPipeline(IDataBusReader dataBusReader, 28 | IMessageFileWriter messageFileWriter, 29 | IDecoder decoder, 30 | IRealTimePublisher realTimePublisher, 31 | IStatsFeedPublisher statsFeedPublisher, 32 | IDbPersister dbPersister) 33 | { 34 | _dataBusReader = dataBusReader; 35 | _messageFileWriter = messageFileWriter; 36 | _decoder = decoder; 37 | _realTimeFeedPublisher = realTimePublisher; 38 | _statsFeedPublisher = statsFeedPublisher; 39 | _dbPersister = dbPersister; 40 | } 41 | 42 | public async Task StartPipelineAsync(CancellationToken token) 43 | { 44 | _decoder.LoadSensorConfigs(); 45 | 46 | // Step 1 - Create our producer as a cold observable 47 | var source = _dataBusReader.StartConsuming(token, TimeSpan.FromMilliseconds(100)); 48 | 49 | // Step 2 - Add file writing and decoding stages to our cold observable pipeline 50 | var writeStream = source.ObserveOn(ThreadPoolScheduler.Instance) 51 | .Select(x => Observable.FromAsync(async () => 52 | { 53 | await _messageFileWriter.WriteAsync(x); 54 | return x; 55 | })).Concat(); 56 | 57 | var decodedStream = writeStream.Select(x => 58 | { 59 | return _decoder.Decode(x).ToObservable(); 60 | }).Concat(); 61 | 62 | // Step 3 - Create a hot observable that acts as a broadcast 63 | // and allows multiple subscribers without duplicating the work of the producer 64 | var multiCastStream = Observable.Publish(decodedStream); 65 | 66 | // Step 4 - Create our subscriptions 67 | // create a subscription to the hot obeservable that buffers in 1 second periods and performs up to 4 concurrent db writes 68 | var dbPersistenceComplete = false; 69 | var dbPersistenceSub = multiCastStream 70 | .Buffer(TimeSpan.FromSeconds(1)) 71 | .Where(messages => messages.Any()) 72 | .Select(messages => Observable.FromAsync(async () => await _dbPersister.PersistAsync(messages))) 73 | .Merge(4) // up to 4 concurrent executions of PersistAsync 74 | .Subscribe( 75 | (Unit u) => { }, 76 | (Exception ex) => { Console.WriteLine("DB Persistence error: " + ex); }, 77 | () => 78 | { 79 | dbPersistenceComplete = true; 80 | Console.WriteLine("DB Persistence complete!"); 81 | }); 82 | 83 | // create a subscription to the hot obeservable that buffers in 1 second periods and performs sequential processing of each batch 84 | bool statsFeed1Complete = false; 85 | var oneSecondStatsFeedSub = multiCastStream 86 | .Buffer(TimeSpan.FromSeconds(1)) 87 | .Where(messages => messages.Any()) 88 | .Select(messages => Observable.FromAsync(async () => await _statsFeedPublisher.PublishAsync(messages, TimeSpan.FromSeconds(1)))) 89 | .Concat() // one batch at a time 90 | .Subscribe( 91 | (Unit u) => { }, 92 | (Exception ex) => { Console.WriteLine("1 Second Stats Feed Error: " + ex); }, 93 | () => 94 | { 95 | statsFeed1Complete = true; 96 | Console.WriteLine("1 Second Stats Feed Complete!"); 97 | }); 98 | 99 | // create a subscription to the hot obeservable that buffers in 30 second periods and performs sequential processing of each batch 100 | bool statsFeed30Complete = false; 101 | var thirtySecondStatsFeedSub = multiCastStream 102 | .Buffer(TimeSpan.FromSeconds(30)) 103 | .Where(messages => messages.Any()) 104 | .Select(messages => Observable.FromAsync(async () => await _statsFeedPublisher.PublishAsync(messages, TimeSpan.FromSeconds(30)))) 105 | .Concat() // one batch at a time 106 | .Subscribe( 107 | (Unit u) => { }, 108 | (Exception ex) => { Console.WriteLine("30 Second Stats Feed Error: " + ex); }, 109 | () => 110 | { 111 | statsFeed30Complete = true; 112 | Console.WriteLine("30 Second Stats Feed Error Complete!"); 113 | }); 114 | 115 | // create a subscription to the hot obeservable that sequentially processes one message at a time in order 116 | bool realTimePubComplete = false; 117 | var realTimePubSub = multiCastStream 118 | .Select(messages => Observable.FromAsync(async () => await _realTimeFeedPublisher.PublishAsync(messages))) 119 | .Concat() // one message at a time 120 | .Subscribe( 121 | (Unit u) => { }, 122 | (Exception ex) => { Console.WriteLine("Real-time Pub Error: " + ex); }, 123 | () => 124 | { 125 | realTimePubComplete = true; 126 | Console.WriteLine("Real-time Pub Complete!"); 127 | }); 128 | 129 | // Step 6. Start the producer 130 | multiCastStream.Connect(); 131 | 132 | // Step 7. Keep things going until the CancellationToken gets cancelled 133 | while (!token.IsCancellationRequested) 134 | await Task.Delay(500); 135 | 136 | // Step 8. Safe shutdown of the pipeline 137 | // Wait for all subscriptions to complete their work 138 | while (!realTimePubComplete || !dbPersistenceComplete || !statsFeed1Complete || !statsFeed30Complete) 139 | await Task.Delay(500); 140 | 141 | Console.WriteLine("All subscribers complete!"); 142 | 143 | // dispose of all subscriptions 144 | dbPersistenceSub.Dispose(); 145 | oneSecondStatsFeedSub.Dispose(); 146 | thirtySecondStatsFeedSub.Dispose(); 147 | realTimePubSub.Dispose(); 148 | 149 | // safely clean up any other resources, for example, ZeroMQ 150 | } 151 | 152 | 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/net-apps/InProcStreamProcessing.TplDataflow/ProcessingPipeline.cs: -------------------------------------------------------------------------------- 1 | using InProcStreamProcessing.Shared.SensorConfiguration; 2 | using InProcStreamProcessing.TplDataFlow.DataBus; 3 | using InProcStreamProcessing.TplDataFlow.Decoders; 4 | using InProcStreamProcessing.TplDataFlow.Feeds; 5 | using InProcStreamProcessing.TplDataFlow.Messages; 6 | using InProcStreamProcessing.TplDataFlow.Persistence; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.Linq; 10 | using System.Text; 11 | using System.Threading; 12 | using System.Threading.Tasks; 13 | using System.Threading.Tasks.Dataflow; 14 | 15 | namespace InProcStreamProcessing.TplDataFlow 16 | { 17 | public class RoutedMessage 18 | { 19 | public RoutedMessage(int routeKey, DecodedMessage message) 20 | { 21 | RouteKey = routeKey; 22 | Message = message; 23 | } 24 | 25 | public int RouteKey { get; set; } 26 | public DecodedMessage Message { get; set; } 27 | } 28 | 29 | public class RoutedBatch 30 | { 31 | public RoutedBatch(int routeKey, IEnumerable messages) 32 | { 33 | RouteKey = routeKey; 34 | Messages = messages; 35 | } 36 | 37 | public int RouteKey { get; set; } 38 | public IEnumerable Messages { get; set; } 39 | } 40 | 41 | public class ProcessingPipeline 42 | { 43 | private IDataBusReader _dataBusReader; 44 | private IMessageFileWriter _messageFileWriter; 45 | private IDecoder _decoder; 46 | private IRealTimePublisher _realTimeFeedPublisher; 47 | private IStatsFeedPublisher _statsFeedPublisher; 48 | private IDbPersister _dbPersister; 49 | 50 | public ProcessingPipeline(IDataBusReader dataBusReader, 51 | IMessageFileWriter messageFileWriter, 52 | IDecoder decoder, 53 | IRealTimePublisher realTimePublisher, 54 | IStatsFeedPublisher s3Uploader, 55 | IDbPersister dbPersister) 56 | { 57 | _dataBusReader = dataBusReader; 58 | _messageFileWriter = messageFileWriter; 59 | _decoder = decoder; 60 | _realTimeFeedPublisher = realTimePublisher; 61 | _statsFeedPublisher = s3Uploader; 62 | _dbPersister = dbPersister; 63 | } 64 | 65 | public async Task StartPipelineAsync(CancellationToken token) 66 | { 67 | _decoder.LoadSensorConfigs(); 68 | 69 | // Step 1 - Configure the pipeline 70 | 71 | // make sure our complete call gets propagated throughout the whole pipeline 72 | var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; 73 | 74 | // create our block configurations 75 | var largeBufferOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 600000 }; 76 | var smallBufferOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 1000 }; 77 | var realTimeBufferOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 6000 }; 78 | var parallelizedOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 1000, MaxDegreeOfParallelism = 4 }; 79 | var batchOptions = new GroupingDataflowBlockOptions() { BoundedCapacity = 1000 }; 80 | 81 | // define each block 82 | var writeRawMessageBlock = new TransformBlock(async (RawBusMessage msg) => 83 | { 84 | await _messageFileWriter.WriteAsync(msg); 85 | return msg; 86 | }, largeBufferOptions); 87 | 88 | var decoderBlock = new TransformManyBlock( 89 | (RawBusMessage msg) => _decoder.Decode(msg), largeBufferOptions); 90 | 91 | var broadcast = new BroadcastBlock(msg => msg); 92 | 93 | var realTimeFeedBlock = new ActionBlock(async 94 | (DecodedMessage msg) => await _realTimeFeedPublisher.PublishAsync(msg), realTimeBufferOptions); 95 | 96 | var oneSecondBatchBlock = new BatchBlock(3000); 97 | var thirtySecondBatchBlock = new BatchBlock(90000); 98 | var batchBroadcastBlock = new BroadcastBlock(msg => msg); 99 | 100 | var oneSecondStatsFeedBlock = new ActionBlock(async 101 | (DecodedMessage[] messages) => await _statsFeedPublisher.PublishAsync(messages.ToList(), TimeSpan.FromSeconds(1)), smallBufferOptions); 102 | 103 | var dbPersistenceBlock = new ActionBlock(async 104 | (DecodedMessage[] messages) => await _dbPersister.PersistAsync(messages.ToList()), smallBufferOptions); 105 | var thirtySecondStatsFeedBlock = new ActionBlock(async 106 | (DecodedMessage[] messages) => await _statsFeedPublisher.PublishAsync(messages.ToList(), TimeSpan.FromSeconds(30)), smallBufferOptions); 107 | 108 | // link the blocks to together 109 | writeRawMessageBlock.LinkTo(decoderBlock, linkOptions); 110 | decoderBlock.LinkTo(broadcast, linkOptions); 111 | broadcast.LinkTo(realTimeFeedBlock, linkOptions); 112 | broadcast.LinkTo(oneSecondBatchBlock, linkOptions); 113 | broadcast.LinkTo(thirtySecondBatchBlock, linkOptions); 114 | oneSecondBatchBlock.LinkTo(batchBroadcastBlock, linkOptions); 115 | batchBroadcastBlock.LinkTo(oneSecondStatsFeedBlock, linkOptions); 116 | batchBroadcastBlock.LinkTo(dbPersistenceBlock, linkOptions); 117 | thirtySecondBatchBlock.LinkTo(thirtySecondStatsFeedBlock, linkOptions); 118 | 119 | // Step 2 - Start consuming the machine bus interface (the producer) 120 | var consumerTask = _dataBusReader.StartConsuming(writeRawMessageBlock, token, TimeSpan.FromMilliseconds(1000), FlowControlMode.LoadShed); 121 | 122 | // Step 3 - Keep going until the CancellationToken is cancelled or a leaf block is the the completed state either due to a fault or the completion of the pipeline. 123 | while (!token.IsCancellationRequested 124 | && !realTimeFeedBlock.Completion.IsCompleted 125 | && !oneSecondStatsFeedBlock.Completion.IsCompleted 126 | && !dbPersistenceBlock.Completion.IsCompleted 127 | && !thirtySecondStatsFeedBlock.Completion.IsCompleted) 128 | { 129 | await Task.Delay(500); 130 | } 131 | 132 | // Step 4 - the CancellationToken has been cancelled and our producer has stopped producing 133 | // call Complete on the first block, this will propagate down the pipeline 134 | writeRawMessageBlock.Complete(); 135 | 136 | // wait for all leaf blocks to finish processing their data 137 | await Task.WhenAll(realTimeFeedBlock.Completion, 138 | oneSecondStatsFeedBlock.Completion, 139 | dbPersistenceBlock.Completion, 140 | thirtySecondStatsFeedBlock.Completion, 141 | consumerTask); 142 | 143 | // clean up any other resources like ZeroMQ for example 144 | } 145 | 146 | public async Task StartPipelineWithBackPressureAsync(CancellationToken token) 147 | { 148 | _decoder.LoadSensorConfigs(); 149 | 150 | // Step 1 - Configure the pipeline 151 | 152 | // make sure our complete call gets propagated throughout the whole pipeline 153 | var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; 154 | 155 | // create our block configurations 156 | var largeBufferOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 60000 }; 157 | var smallBufferOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 1000 }; 158 | var parallelizedOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 1000, MaxDegreeOfParallelism = 4 }; 159 | var batchOptions = new GroupingDataflowBlockOptions() { BoundedCapacity = 200000 }; 160 | 161 | // create some branching functions for our TransformManyBlocks 162 | // DecodedMessage gets tranformed into 3 RoutedMessage for the first three way branch 163 | Func> messageBranchFunc = x => new List 164 | { 165 | new RoutedMessage(1, x), 166 | new RoutedMessage(2, x), 167 | new RoutedMessage(3, x) 168 | }; 169 | 170 | // DecodedMessage[] gets tranformed into a RoutedBatch for the final branch 171 | Func> batchBranchFunc = x => new List 172 | { 173 | new RoutedBatch(1, x.Select(c => c.Message).ToList()), 174 | new RoutedBatch(2, x.Select(c => c.Message).ToList()) 175 | }; 176 | 177 | // define each block 178 | var writeRawMessageBlock = new TransformBlock(async (RawBusMessage msg) => 179 | { 180 | await _messageFileWriter.WriteAsync(msg); 181 | return msg; 182 | }, largeBufferOptions); 183 | 184 | var decoderBlock = new TransformManyBlock( 185 | (RawBusMessage msg) => _decoder.Decode(msg), largeBufferOptions); 186 | 187 | 188 | var msgBranchBlock = new TransformManyBlock(messageBranchFunc, largeBufferOptions); 189 | 190 | var realTimeFeedBlock = new ActionBlock(async (RoutedMessage routedMsg) => 191 | await _realTimeFeedPublisher.PublishAsync(routedMsg.Message), largeBufferOptions); 192 | 193 | var thirtySecondBatchBlock = new BatchBlock(90000, batchOptions); 194 | var thirtySecondStatsFeedBlock = new ActionBlock(async (RoutedMessage[] batch) => 195 | await _statsFeedPublisher.PublishAsync(batch.Select(x => x.Message).ToList(), TimeSpan.FromSeconds(30)), smallBufferOptions); 196 | 197 | var oneSecondBatchBlock = new BatchBlock(3000, batchOptions); 198 | var batchBroadcastBlock = new TransformManyBlock(batchBranchFunc, smallBufferOptions); 199 | 200 | var oneSecondStatsFeedBlock = new ActionBlock(async (RoutedBatch batch) => 201 | await _statsFeedPublisher.PublishAsync(batch.Messages.ToList(), TimeSpan.FromSeconds(1)), smallBufferOptions); 202 | 203 | var dbPersistenceBlock = new ActionBlock(async (RoutedBatch batch) => 204 | await _dbPersister.PersistAsync(batch.Messages.ToList()), parallelizedOptions); 205 | 206 | // link the blocks to together 207 | writeRawMessageBlock.LinkTo(decoderBlock, linkOptions); 208 | decoderBlock.LinkTo(msgBranchBlock, linkOptions); 209 | msgBranchBlock.LinkTo(realTimeFeedBlock, linkOptions, routedMsg => routedMsg.RouteKey == 1); // route on the key 210 | msgBranchBlock.LinkTo(oneSecondBatchBlock, linkOptions, routedMsg => routedMsg.RouteKey == 2); // route on the key 211 | msgBranchBlock.LinkTo(thirtySecondBatchBlock, linkOptions, routedMsg => routedMsg.RouteKey == 3); // route on the key 212 | thirtySecondBatchBlock.LinkTo(thirtySecondStatsFeedBlock, linkOptions); 213 | oneSecondBatchBlock.LinkTo(batchBroadcastBlock, linkOptions); 214 | batchBroadcastBlock.LinkTo(oneSecondStatsFeedBlock, linkOptions, routedMsg => routedMsg.RouteKey == 1); // route on the key 215 | batchBroadcastBlock.LinkTo(dbPersistenceBlock, linkOptions, routedMsg => routedMsg.RouteKey == 2); // route on the key 216 | 217 | // Step 2 - Start consuming the machine bus interface (the producer) 218 | var consumerTask = _dataBusReader.StartConsuming(writeRawMessageBlock, token, TimeSpan.FromMilliseconds(1000), FlowControlMode.BackPressure); 219 | 220 | // Step 3 - Keep going until the CancellationToken is cancelled or a leaf block is the the completed state either due to a fault or the completion of the pipeline. 221 | while (!token.IsCancellationRequested 222 | || oneSecondStatsFeedBlock.Completion.IsCompleted 223 | || dbPersistenceBlock.Completion.IsCompleted 224 | || realTimeFeedBlock.Completion.IsCompleted 225 | || thirtySecondStatsFeedBlock.Completion.IsCompleted) 226 | { 227 | await Task.Delay(500); 228 | } 229 | 230 | // Step 4 - the CancellationToken has been cancelled and our producer has stopped producing 231 | // call Complete on the first block, this will propagate down the pipeline 232 | writeRawMessageBlock.Complete(); 233 | 234 | // wait for all leaf blocks to finish processing their data 235 | await Task.WhenAll(oneSecondStatsFeedBlock.Completion, 236 | thirtySecondStatsFeedBlock.Completion, 237 | dbPersistenceBlock.Completion, 238 | realTimeFeedBlock.Completion, 239 | consumerTask); 240 | 241 | // clean up any other resources like ZeroMQ for example 242 | } 243 | } 244 | } 245 | --------------------------------------------------------------------------------