├── .gitignore ├── .nuget └── packages.config ├── LICENSE ├── README.md ├── doc └── img │ ├── perfView-collect.png │ ├── perfView-dump.png │ ├── perfView-events.png │ └── perfView-params.png ├── errors-rework.txt ├── examples ├── ConsumerExample.cs ├── ProducerExample.cs ├── Program.cs ├── examples.csproj └── packages.config ├── kafka4net.sln ├── log4j.properties ├── nuget ├── nuget.exe.config └── publish.bat ├── src ├── BrokerException.cs ├── Cluster.cs ├── Compression │ ├── KafkaSnappyStream.cs │ ├── Lz4KafkaStream.cs │ └── StreamUtils.cs ├── CompressionType.cs ├── ConnState.cs ├── Connection.cs ├── Consumer.cs ├── ConsumerConfiguration.cs ├── ConsumerImpl │ ├── Fetcher.cs │ ├── PartitionFetchState.cs │ ├── PositionProviders.cs │ ├── TopicPartition.cs │ └── TopicPartitionOffsets.cs ├── ErrorCode.cs ├── FletcherHashedMessagePartitioner.cs ├── ILogger.cs ├── IMessagePartitioner.cs ├── Internal │ ├── PartitionRecoveryMonitor.cs │ └── PartitionStateChangeEvent.cs ├── Logger.cs ├── Message.cs ├── Metadata │ ├── BrokerMeta.cs │ ├── PartitionMeta.cs │ ├── PartitionOffsetInfo.cs │ └── TopicMeta.cs ├── PartitionFailedException.cs ├── Producer.cs ├── ProducerConfiguration.cs ├── Properties │ └── AssemblyInfo.cs ├── Protocols │ ├── CorrelationLoopException.cs │ ├── Protocol.cs │ ├── Requests │ │ ├── FetchRequest.cs │ │ ├── MessageData.cs │ │ ├── MessageSetItem.cs │ │ ├── OffsetRequest.cs │ │ ├── PartitionData.cs │ │ ├── ProduceRequest.cs │ │ ├── TopicData.cs │ │ └── TopicRequest.cs │ ├── ResponseCorrelation.cs │ ├── Responses │ │ ├── FetchResponse.cs │ │ ├── MetadataResponse.cs │ │ ├── OffsetResponse.cs │ │ └── ProducerResponse.cs │ └── Serializer.cs ├── ReceivedMessage.cs ├── Tracing │ └── EtwTrace.cs ├── Utils │ ├── AskEx.cs │ ├── BigEndianConverter.cs │ ├── CircularBuffer.cs │ ├── CountObservable.cs │ ├── Crc32.cs │ ├── DateTimeExtensions.cs │ ├── EnumerableEx.cs │ ├── LittleEndianConverter.cs │ ├── RxSyncContextFromScheduler.cs │ ├── TaskEx.cs │ └── WatchdogScheduler.cs ├── WorkingThreadHungException.cs ├── kafka4net.csproj ├── kafka4net.nuspec └── packages.config ├── tests ├── CompressionTests.cs ├── Properties │ └── AssemblyInfo.cs ├── RecoveryTest.cs ├── VagrantBrokerUtil.cs ├── packages.config └── tests.csproj ├── tools └── binary-console │ ├── build.gradle │ └── src │ └── main │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── ntent │ └── kafka │ └── main.scala └── vagrant ├── .gitignore ├── .hgignore ├── Vagrantfile ├── config ├── kafka.conf ├── zookeeper.conf └── zookeeper.properties ├── log4j.properties ├── scripts ├── broker.sh ├── env.sh ├── init.sh ├── kafka_version.txt └── zookeeper.sh └── server.properties /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | #ignore thumbnails created by windows 3 | Thumbs.db 4 | #Ignore files build by Visual Studio 5 | *.obj 6 | *.exe 7 | *.pdb 8 | *.user 9 | *.aps 10 | *.pch 11 | *.vspscc 12 | *_i.c 13 | *_p.c 14 | *.ncb 15 | *.suo 16 | *.tlb 17 | *.tlh 18 | *.bak 19 | *.cache 20 | *.ilk 21 | *.log 22 | [Bb]in 23 | [Dd]ebug*/ 24 | *.lib 25 | *.sbr 26 | obj/ 27 | [Rr]elease*/ 28 | _ReSharper*/ 29 | [Tt]est[Rr]esult* 30 | log.txt 31 | packages 32 | vagrant/files 33 | /Build.proj 34 | /src/Properties/AssemblyInfoGenerated.cs 35 | nuget/*.nupkg 36 | .gradle 37 | .idea 38 | *.iml 39 | build -------------------------------------------------------------------------------- /.nuget/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![NuGet version](https://badge.fury.io/nu/kafka4net.svg)](https://badge.fury.io/nu/kafka4net) 2 | kafka4net: kafka-0.8 client 3 | ========= 4 | 5 | #### Install: 6 | ```Install-Package kafka4net``` 7 | 8 | ##Features: 9 | * Event-driven architecture, all asynchronous 10 | * Automatic following changes of Leader Partition in case of broker failure 11 | * Connection sharing: one connection per kafka broker is used 12 | * Flow control: slow consumer will suspend fetching and prevent memory exhausting 13 | * Integration tests are part of the codebase. Use Vagrant to provision 1 zookeeper and 3 kafka virtual servers 14 | * Use RxExtensions library to expose API and for internal implementation 15 | * Support compression (gzip, lz4, snappy). Unit-tested to be interoperable with Java implementation 16 | 17 | ##Not implemented: 18 | * Offset Fetch/Commit API 19 | * Only protocol for 0.8 (aka v0) is implemented at the moment 20 | 21 | ## Documentation 22 | * [Design](https://github.com/ntent-ad/kafka4net/wiki/Design) 23 | * [Troubleshooting](https://github.com/ntent-ad/kafka4net/wiki/Troubleshooting) 24 | 25 | ## Usage 26 | ### Consumer 27 | 28 | ```C# 29 | using System; 30 | using System.Linq; 31 | using System.Text; 32 | using System.Text.RegularExpressions; 33 | using System.Threading.Tasks; 34 | using kafka4net; 35 | using kafka4net.ConsumerImpl; 36 | 37 | namespace examples 38 | { 39 | public static class ConsumerExample 40 | { 41 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092. 42 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094" 43 | readonly static string _connectionString = "kafka1, kafka2, kafka3"; 44 | readonly static string _topic = "some.topic"; 45 | 46 | /// Simplest consumer with termination example 47 | public static async Task TakeForOneMinute() 48 | { 49 | // If you want to consume all messages in the topic, use TopicPositionFactory.Start 50 | // TopicPositionFactory.End will start waiting for new messages starting from the moment of subscription 51 | var consumer = new Consumer(new ConsumerConfiguration(_connectionString, _topic, new StartPositionTopicEnd())); 52 | 53 | consumer.OnMessageArrived.Subscribe(msg => { 54 | // Perform your own deserialization here 55 | var text = Encoding.UTF8.GetString(msg.Value); 56 | Console.WriteLine($"Got message: '{text}' Partition: {msg.Partition} Offset: {msg.Offset} Lag: {msg.HighWaterMarkOffset - msg.Offset}"); 57 | }); 58 | 59 | // Connecting starts when subscribing to OnMessageArrived. If you need to know when connection is actually one, wait for IsConnected task completion 60 | await consumer.IsConnected; 61 | Console.WriteLine("Connected"); 62 | 63 | // Consume for one minute 64 | await Task.Delay(TimeSpan.FromMinutes(1)); 65 | 66 | await consumer.CloseAsync(); 67 | Console.WriteLine("Closed"); 68 | } 69 | } 70 | } 71 | ``` 72 | 73 | 74 | 75 | ### Consumer from multiple partitions 76 | 77 | ```C# 78 | using System; 79 | using System.Linq; 80 | using System.Text; 81 | using System.Text.RegularExpressions; 82 | using System.Threading.Tasks; 83 | using kafka4net; 84 | using kafka4net.ConsumerImpl; 85 | 86 | namespace examples 87 | { 88 | public static class ConsumerExample 89 | { 90 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092. 91 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094" 92 | readonly static string _connectionString = "kafka1, kafka2, kafka3"; 93 | readonly static string _topic = "some.topic"; 94 | 95 | public static async Task SubscribeToMultipleTopics() 96 | { 97 | var rx = new Regex("some\\.topic\\..+"); 98 | var cluster = new Cluster(_connectionString); 99 | await cluster.ConnectAsync(); 100 | var allTopics = await cluster.GetAllTopicsAsync(); 101 | var wantedTopics = allTopics.Where(topic => rx.IsMatch(topic)).ToArray(); 102 | 103 | var consumers = wantedTopics.Select(topic => new Consumer(new ConsumerConfiguration(_connectionString, topic, new StartPositionTopicEnd()))).ToArray(); 104 | consumers.AsParallel().ForAll(consumer => consumer.OnMessageArrived.Subscribe(msg => { 105 | var text = Encoding.UTF8.GetString(msg.Value); 106 | Console.WriteLine($"Got message '{text}' from topic '{msg.Topic}' partition {msg.Partition} offset {msg.Offset}"); 107 | })); 108 | 109 | await Task.Delay(TimeSpan.FromMinutes(1)); 110 | 111 | await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync())); 112 | } 113 | 114 | } 115 | } 116 | ``` 117 | 118 | ### Producer 119 | ```C# 120 | using System; 121 | using System.Linq; 122 | using System.Text; 123 | using System.Threading.Tasks; 124 | using kafka4net; 125 | 126 | namespace examples 127 | { 128 | public static class ProducerExample 129 | { 130 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092. 131 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094" 132 | readonly static string _connectionString = "kafka1, kafka2, kafka3"; 133 | readonly static string _topic = "some.topic"; 134 | 135 | public async static Task Produce100RandomMessages() 136 | { 137 | var rnd = new Random(); 138 | var randomNumbers = Enumerable.Range(1, 100).Select(_ => rnd.Next().ToString()); 139 | var producer = new Producer(_connectionString, new ProducerConfiguration(_topic)); 140 | 141 | // Technically not mandatory, but good idea to listen to possible errors and act accordingly to your application requirements 142 | producer.OnPermError += (exception, messages) => Console.WriteLine($"Failed to write {messages.Length} because of {exception.Message}"); 143 | 144 | // When message is confirmed by kafka broker to be persisted, OnSuccess is called. Can be used if upstream requires acknowlegement for extra reliability. 145 | producer.OnSuccess += messages => Console.WriteLine($"Sent {messages.Length} messages"); 146 | 147 | await producer.ConnectAsync(); 148 | 149 | foreach(var str in randomNumbers) 150 | { 151 | // Message.Key is optional. If not set, then driver will partition messages at random. In this case, for sake of example, partition by 1st character of the string 152 | var key = BitConverter.GetBytes(str[0]); 153 | // Implement your own serialization here. 154 | var msg = new Message { Value = Encoding.UTF8.GetBytes(str), Key = key}; 155 | producer.Send(msg); 156 | } 157 | 158 | // Await for every producer to complete. Note, that calling CloseAsync is safe: all bessages currently in buffers will be awaited until flushed. 159 | await producer.CloseAsync(TimeSpan.FromMinutes(1)); 160 | } 161 | } 162 | } 163 | ``` -------------------------------------------------------------------------------- /doc/img/perfView-collect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-collect.png -------------------------------------------------------------------------------- /doc/img/perfView-dump.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-dump.png -------------------------------------------------------------------------------- /doc/img/perfView-events.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-events.png -------------------------------------------------------------------------------- /doc/img/perfView-params.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-params.png -------------------------------------------------------------------------------- /errors-rework.txt: -------------------------------------------------------------------------------- 1 | Initial idea of implementing error recovery was to have single point, in tcp connection object where all error recovery would be made. 2 | But it turned out too low level, because depending on what this connection is doing, we might want different strategy for recovery. 3 | When RecoveryMonitor tests either broker is available, we do not want any recovery and want fail fast instead. 4 | Another issue is that currently only fetcher and producer are well protected, whereas earlier stages, such as connection, offset resolution, 5 | metadata fetching are more fragile, or have to implement their own recovery, thus polluting the code. 6 | 7 | Requirements: 8 | Fast reaction: consider partition failed immediately after tcp error, and not after many retries. 9 | Remember to fail all request tasks which are waiting for responses. 10 | While waiting for recovery, pay attention to changes in metadata 11 | Work nicely with shutdown and drain logic 12 | 13 | List of failure stages: 14 | In seed broker resolution 15 | In offset resolution 16 | In Metadata fetching 17 | In recovery monitor 18 | In produce message 19 | In fetch message -------------------------------------------------------------------------------- /examples/ConsumerExample.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Reactive.Linq; 4 | using System.Reactive.Threading.Tasks; 5 | using System.Text; 6 | using System.Text.RegularExpressions; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | using kafka4net; 10 | using kafka4net.ConsumerImpl; 11 | using log4net; 12 | 13 | namespace examples 14 | { 15 | public static class ConsumerExample 16 | { 17 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092. 18 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094" 19 | readonly static string _connectionString = "kafka1, kafka2, kafka3"; 20 | readonly static string _topic = "some.topic"; 21 | private static ILog _log = LogManager.GetLogger(typeof (ConsumerExample)); 22 | 23 | /// Simplest consumer with termination example 24 | public static async Task TakeForOneMinute() 25 | { 26 | // If you want to consume all messages in the topic, use TopicPositionFactory.Start 27 | // TopicPositionFactory.End will start waiting for new messages starting from the moment of subscription 28 | var consumer = new Consumer(new ConsumerConfiguration(_connectionString, _topic, new StartPositionTopicEnd())); 29 | 30 | consumer.OnMessageArrived.Subscribe(msg => { 31 | // Perform your own deserialization here 32 | var text = Encoding.UTF8.GetString(msg.Value); 33 | Console.WriteLine($"Got message: '{text}' Partition: {msg.Partition} Offset: {msg.Offset} Lag: {msg.HighWaterMarkOffset - msg.Offset}"); 34 | }); 35 | 36 | // Connecting starts when subscribing to OnMessageArrived. If you need to know when connection is actually one, wait for IsConnected task completion 37 | await consumer.IsConnected; 38 | Console.WriteLine("Connected"); 39 | 40 | // Consume for one minute 41 | await Task.Delay(TimeSpan.FromMinutes(1)); 42 | 43 | await consumer.CloseAsync(); 44 | Console.WriteLine("Closed"); 45 | } 46 | 47 | public static async Task SubscribeToMultipleTopics() 48 | { 49 | var rx = new Regex("some\\.topic\\..+"); 50 | var cluster = new Cluster(_connectionString); 51 | await cluster.ConnectAsync(); 52 | var allTopics = await cluster.GetAllTopicsAsync(); 53 | var wantedTopics = allTopics.Where(topic => rx.IsMatch(topic)).ToArray(); 54 | 55 | var consumers = wantedTopics.Select(topic => new Consumer(new ConsumerConfiguration(_connectionString, topic, new StartPositionTopicEnd()))).ToArray(); 56 | consumers.AsParallel().ForAll(consumer => consumer.OnMessageArrived.Subscribe(msg => { 57 | var text = Encoding.UTF8.GetString(msg.Value); 58 | Console.WriteLine($"Got message '{text}' from topic '{msg.Topic}' partition {msg.Partition} offset {msg.Offset}"); 59 | })); 60 | 61 | await Task.Delay(TimeSpan.FromMinutes(1)); 62 | 63 | await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync())); 64 | } 65 | 66 | public static async Task ConsumeMessages(string brokers, string topic, string file, CancellationToken cancel) 67 | { 68 | _log.Info("Conecting..."); 69 | var consumer = new Consumer(new ConsumerConfiguration(brokers, topic, new StartPositionTopicEnd())); 70 | 71 | consumer.OnMessageArrived.Subscribe(msg => { 72 | // Perform your own deserialization here 73 | var text = Encoding.UTF8.GetString(msg.Value); 74 | Console.WriteLine($"Received: '{text}' Partition: {msg.Partition} Offset: {msg.Offset} Lag: {msg.HighWaterMarkOffset - msg.Offset}"); 75 | }, 76 | e=> { _log.Error("Failed in Consumer", e); }, 77 | () => { _log.Info("Complete...");}, cancel); 78 | 79 | // Connecting starts when subscribing to OnMessageArrived. If you need to know when connection is actually one, wait for IsConnected task completion 80 | await consumer.IsConnected; 81 | _log.Info("Consuming..."); 82 | 83 | await cancel.WhenCanceled(); 84 | 85 | _log.Info("Closing..."); 86 | await consumer.CloseAsync(); 87 | Console.WriteLine("Completed..."); 88 | 89 | } 90 | 91 | public static Task WhenCanceled(this CancellationToken cancellationToken) 92 | { 93 | var tcs = new TaskCompletionSource(); 94 | cancellationToken.Register(s => ((TaskCompletionSource)s).SetResult(true), tcs); 95 | return tcs.Task; 96 | } 97 | 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /examples/ProducerExample.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Reactive.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using kafka4net; 8 | using log4net; 9 | using System.Reactive.Threading.Tasks; 10 | using System.Threading; 11 | 12 | namespace examples 13 | { 14 | public static class ProducerExample 15 | { 16 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092. 17 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094" 18 | private static readonly string _connectionString = "kafka1, kafka2, kafka3"; 19 | private static readonly string _topic = "some.topic"; 20 | private static ILog _log = LogManager.GetLogger(typeof (ProducerExample)); 21 | 22 | public static async Task Produce100RandomMessages() 23 | { 24 | var rnd = new Random(); 25 | var randomNumbers = Enumerable.Range(1, 100).Select(_ => rnd.Next().ToString()); 26 | var producer = new Producer(_connectionString, new ProducerConfiguration(_topic)); 27 | 28 | // Technically not mandatory, but good idea to listen to possible errors and act accordingly to your application requirements 29 | producer.OnPermError += 30 | (exception, messages) => 31 | Console.WriteLine($"Failed to write {messages.Length} because of {exception.Message}"); 32 | 33 | // When message is confirmed by kafka broker to be persisted, OnSuccess is called. Can be used if upstream requires acknowlegement for extra reliability. 34 | producer.OnSuccess += messages => Console.WriteLine($"Sent {messages.Length} messages"); 35 | 36 | await producer.ConnectAsync(); 37 | 38 | foreach (var str in randomNumbers) 39 | { 40 | // Message.Key is optional. If not set, then driver will partition messages at random. In this case, for sake of example, partition by 1st character of the string 41 | var key = BitConverter.GetBytes(str[0]); 42 | // Implement your own serialization here. 43 | var msg = new Message {Value = Encoding.UTF8.GetBytes(str), Key = key}; 44 | producer.Send(msg); 45 | } 46 | 47 | // Await for every producer to complete. Note, that calling CloseAsync is safe: all bessages currently in buffers will be awaited until flushed. 48 | await producer.CloseAsync(TimeSpan.FromMinutes(1)); 49 | } 50 | 51 | public static async Task ProduceMessages( 52 | string brokers, 53 | string topic, 54 | int produceDelayMs, 55 | int numMessages, 56 | string file, 57 | CancellationToken cancel) 58 | { 59 | var producer = new Producer(brokers, new ProducerConfiguration(topic)); 60 | 61 | // Technically not mandatory, but good idea to listen to possible errors and act accordingly to your application requirements 62 | producer.OnPermError += 63 | (exception, messages) => 64 | _log.Error($"Failed to write {messages.Length} because of {exception.Message}"); 65 | 66 | // When message is confirmed by kafka broker to be persisted, OnSuccess is called. Can be used if upstream requires acknowlegement for extra reliability. 67 | producer.OnSuccess += 68 | messages => { foreach (var msg in messages) _log.Info($"Sent {Encoding.UTF8.GetString(msg.Value)}"); }; 69 | 70 | _log.Info("Connecting..."); 71 | await producer.ConnectAsync(); 72 | 73 | _log.Info("Producing..."); 74 | Func dataFunc; 75 | if (file == null) 76 | dataFunc = i => i.ToString(); 77 | else 78 | { 79 | var fileData = File.ReadAllLines(file); 80 | dataFunc = i => fileData[i%fileData.Length]; 81 | } 82 | 83 | var data = Observable.Interval(TimeSpan.FromMilliseconds(produceDelayMs)).Select(dataFunc); 84 | if (numMessages > 0) 85 | data = data.Take(numMessages); 86 | 87 | var sendTask = data.Do(d => producer.Send(new Message {Value = Encoding.UTF8.GetBytes(d)})).ToTask(cancel); 88 | 89 | await sendTask; 90 | 91 | _log.Info("Closing..."); 92 | await producer.CloseAsync(TimeSpan.FromSeconds(10)); 93 | 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /examples/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Text; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | using log4net; 10 | using log4net.Config; 11 | 12 | namespace examples 13 | { 14 | class Program 15 | { 16 | #region staticLog4netAppender 17 | const string StaticConsoleAppender = @" 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | "; 46 | #endregion staticLog4netAppender 47 | 48 | private static ILog _log; 49 | 50 | static void Main(string[] args) 51 | { 52 | var log4NetConfig = ConfigurationManager.AppSettings.Get("LoggingConfigFile"); 53 | if (string.IsNullOrWhiteSpace(log4NetConfig)) 54 | log4NetConfig = "log4net.config"; 55 | 56 | if (File.Exists(log4NetConfig)) 57 | { 58 | XmlConfigurator.ConfigureAndWatch(new FileInfo(log4NetConfig)); 59 | } 60 | else 61 | { 62 | XmlConfigurator.Configure(new MemoryStream(Encoding.UTF8.GetBytes(StaticConsoleAppender))); 63 | } 64 | _log = LogManager.GetLogger(typeof(Program)); 65 | 66 | kafka4net.Logger.SetupLog4Net(); 67 | 68 | var showHelp = false; 69 | string brokers = null; 70 | string topic = null; 71 | bool isProducer = false, isConsumer = false; 72 | int produceDelayMs = 1000; 73 | int totalMessages = -1; 74 | string file = null; 75 | 76 | var opts = new NDesk.Options.OptionSet 77 | { 78 | {"?|h|help", "Show Help", v => showHelp = true}, 79 | {"p|producer", "Run Producer. Cannot be used with -c|-consumer.", v => isProducer = true }, 80 | {"c|consumer", "Run Consumer. Cannot be used with -p|-producer.", v => isConsumer = true }, 81 | {"d:|delay-ms:", "Producer delay between messages in ms (default 1000). Specify an integer number of milliseconds", (int v) => produceDelayMs = v }, 82 | {"m:|messages:", "Total number of messages to produce or consume before closing. Specify -1 (default) to send/receive indefinitely.", (int v) => totalMessages = v }, 83 | {"f:|file:", "File for producer to read from or consumer to write to. In producer mode, the file is read line by line, sending the content of each line as a message, and repeating as necessary. In Consumer mode, messages are written to this file line by line. If not specified, producer sends increasing numeric values, and consumer writes to console.", v => file = v }, 84 | {"b=|brokers=", "Comman delimited list of Broker(s) to connect to. Can be just name/IP to use default port of 9092, or :", v => brokers = v }, 85 | {"t=|topic=", "Topic to Produce or Consume from.", v => topic = v } 86 | }; 87 | var unusedArgs = opts.Parse(args); 88 | if (unusedArgs.Any()) 89 | ArgsFail($"Didn't recognize these command line options: {string.Join(", ", unusedArgs)}", opts); 90 | 91 | if (isProducer && isConsumer) 92 | ArgsFail("Cannot specify both producer and consumer",opts); 93 | 94 | if (!isProducer && !isConsumer) 95 | ArgsFail("Must specify either producer or consumer", opts); 96 | 97 | if (string.IsNullOrWhiteSpace(brokers)) 98 | ArgsFail("Must specify one or more brokers to connect to.",opts); 99 | 100 | if (string.IsNullOrWhiteSpace(topic)) 101 | ArgsFail("Must specify a topic name.",opts); 102 | 103 | if (!args.Any() || showHelp) 104 | { 105 | PrintHelp(opts); 106 | Environment.Exit(args.Any() ? 0 : 1); 107 | } 108 | 109 | if (isProducer) 110 | { 111 | var cancel = new CancellationTokenSource(); 112 | var produceTask = ProducerExample.ProduceMessages(brokers, topic, produceDelayMs, totalMessages, file, cancel.Token); 113 | 114 | _log.Debug("Starting..."); 115 | Task.WaitAny(produceTask, Task.Factory.StartNew(() => 116 | { 117 | Console.Out.WriteLine("Press 'Q' to cancel."); 118 | var key = ""; 119 | while (key != "q") 120 | { 121 | key = Console.ReadKey().KeyChar.ToString().ToLower(); 122 | } 123 | 124 | Console.WriteLine("Cancelling..."); 125 | cancel.Cancel(); 126 | }, cancel.Token)); 127 | 128 | } 129 | 130 | if (isConsumer) 131 | { 132 | var cancel = new CancellationTokenSource(); 133 | var consumeTask = ConsumerExample.ConsumeMessages(brokers, topic, file, cancel.Token); 134 | 135 | _log.Debug("Starting..."); 136 | Task.WaitAny(consumeTask, Task.Factory.StartNew(() => 137 | { 138 | Console.Out.WriteLine("Press 'Q' to cancel."); 139 | var key = ""; 140 | while (key != "q") 141 | { 142 | key = Console.ReadKey().KeyChar.ToString().ToLower(); 143 | } 144 | 145 | Console.WriteLine("Cancelling..."); 146 | cancel.Cancel(); 147 | }, cancel.Token)); 148 | 149 | } 150 | 151 | } 152 | 153 | private static void ArgsFail(string errMsg, NDesk.Options.OptionSet opts) 154 | { 155 | _log.Error(errMsg); 156 | Console.Error.WriteLine(errMsg); 157 | PrintHelp(opts); 158 | Environment.Exit(1); 159 | } 160 | 161 | private static void PrintHelp(NDesk.Options.OptionSet opts) 162 | { 163 | _log.InfoFormat("Showing options in console..."); 164 | 165 | Console.Out.WriteLine($"Usage: examples (--producer|--consumer) --brokers [:port][,[:port]...] --topic= [Options]+"); 166 | Console.Out.WriteLine("Options:"); 167 | Console.Out.WriteLine(); 168 | opts.WriteOptionDescriptions(Console.Out); 169 | Console.Out.WriteLine("Press any key to continue..."); 170 | Console.In.Read(); 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /examples/examples.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C} 8 | Exe 9 | Properties 10 | examples 11 | examples 12 | v4.5.2 13 | 512 14 | 15 | 16 | true 17 | full 18 | false 19 | bin\Debug\ 20 | DEBUG;TRACE 21 | prompt 22 | 4 23 | 24 | 25 | pdbonly 26 | true 27 | bin\Release\ 28 | TRACE 29 | prompt 30 | 4 31 | 32 | 33 | 34 | 35 | 36 | 37 | ..\packages\log4net.2.0.5\lib\net45-full\log4net.dll 38 | True 39 | 40 | 41 | ..\packages\NDesk.Options.0.2.1\lib\NDesk.Options.dll 42 | True 43 | 44 | 45 | 46 | 47 | 48 | ..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll 49 | True 50 | 51 | 52 | ..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll 53 | True 54 | 55 | 56 | ..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll 57 | True 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | {e3a39e66-db86-4e9c-b967-3aa71fa47c5d} 71 | kafka4net 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 88 | -------------------------------------------------------------------------------- /examples/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /kafka4net.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.25420.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kafka4net", "src\kafka4net.csproj", "{E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "tests", "tests\tests.csproj", "{8BAF9CFE-8324-4604-BCA1-96925E8B82AC}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{35FED764-3D31-4487-B0F4-837F2ADE072B}" 11 | ProjectSection(SolutionItems) = preProject 12 | .nuget\packages.config = .nuget\packages.config 13 | EndProjectSection 14 | EndProject 15 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "examples", "examples\examples.csproj", "{D76203FD-59DA-4FA2-8793-B8810DA44D9C}" 16 | EndProject 17 | Global 18 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 19 | Debug|Any CPU = Debug|Any CPU 20 | Release|Any CPU = Release|Any CPU 21 | EndGlobalSection 22 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 23 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 24 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Debug|Any CPU.Build.0 = Debug|Any CPU 25 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Release|Any CPU.ActiveCfg = Release|Any CPU 26 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Release|Any CPU.Build.0 = Release|Any CPU 27 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 28 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Debug|Any CPU.Build.0 = Debug|Any CPU 29 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Release|Any CPU.ActiveCfg = Release|Any CPU 30 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Release|Any CPU.Build.0 = Release|Any CPU 31 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 32 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Debug|Any CPU.Build.0 = Debug|Any CPU 33 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Release|Any CPU.ActiveCfg = Release|Any CPU 34 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Release|Any CPU.Build.0 = Release|Any CPU 35 | EndGlobalSection 36 | GlobalSection(SolutionProperties) = preSolution 37 | HideSolutionNode = FALSE 38 | EndGlobalSection 39 | EndGlobal 40 | -------------------------------------------------------------------------------- /log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /nuget/nuget.exe.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /nuget/publish.bat: -------------------------------------------------------------------------------- 1 | @rem \bin\nuget-2.8.5\NuGet.exe pack ..\src\kafka4net.csproj -Build -Symbols -Prop Configuration=Release 2 | nuget.exe pack ..\src\kafka4net.csproj -Build -Symbols -Prop Configuration=Release; -MSBuildVersion 14 -Version 2.0.1 3 | 4 | @rem -Verbosity detailed -------------------------------------------------------------------------------- /src/BrokerException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net 4 | { 5 | public class BrokerException : Exception 6 | { 7 | public BrokerException(string message) : base(message) {} 8 | public BrokerException(string message, Exception inner) : base(message, inner) { } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Compression/KafkaSnappyStream.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using kafka4net.Utils; 4 | using Snappy; 5 | 6 | namespace kafka4net.Compression 7 | { 8 | /// 9 | /// Kafka's snappy framing is non-standard. See https://github.com/xerial/snappy-java 10 | /// [magic header:16 bytes]([block size:int32][compressed data:byte array])* 11 | /// Version and min compatible version are both 1 12 | /// 13 | sealed class KafkaSnappyStream : Stream 14 | { 15 | static readonly byte[] _snappyMagic = { 0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0 }; 16 | const int SnappyHeaderLen = 16; // 8 bytes of magic and 2 ints for version compatibility 17 | 18 | CompressionStreamMode _mode; 19 | readonly Stream _base; 20 | byte[] _uncompressedBuffer; 21 | // Working buffer to store data from compressed stream and avoid allocations. Can be resized 22 | byte[] _compressedBuffer;// = new byte[10*1024]; 23 | int _bufferLen; 24 | int _bufferPtr; 25 | readonly byte[] _headerWorkingBuffer = new byte[SnappyHeaderLen]; 26 | static readonly byte[] _versionsHeader = { 0, 0, 0, 1, 0, 0, 0, 1}; 27 | 28 | /// 29 | /// 30 | /// 31 | /// 32 | /// 33 | /// Recommended size is 32Kb, as default in java xerces implementation 34 | public KafkaSnappyStream(Stream stream, CompressionStreamMode mode, byte[] uncompressedBuffer, byte[] compressedBuffer) 35 | { 36 | _mode = mode; 37 | _base = stream; 38 | _uncompressedBuffer = uncompressedBuffer; 39 | _compressedBuffer = compressedBuffer; 40 | 41 | if (mode == CompressionStreamMode.Decompress) 42 | { 43 | if (!ReadHeader()) 44 | throw new InvalidDataException("Failed to read snappy header"); 45 | ReadBlock(); 46 | } 47 | else 48 | { 49 | WriteHeader(); 50 | } 51 | } 52 | 53 | /// 54 | /// Allocate buffers of size, recommended for kafka usage. CompressedBuffer will be allocated of enough size, which will not require reallocation. 55 | /// 56 | /// 57 | /// 58 | public static void AllocateBuffers(out byte[] uncompressedBuffer, out byte[] compressedBuffer) 59 | { 60 | uncompressedBuffer = new byte[32 * 1024]; // 32K is default buffer size in xerces 61 | compressedBuffer = new byte[SnappyCodec.GetMaxCompressedLength(uncompressedBuffer.Length)]; 62 | } 63 | 64 | public override void Flush() 65 | { 66 | if(_bufferLen == 0) 67 | return; 68 | 69 | Flush(_uncompressedBuffer, 0, _bufferLen); 70 | _bufferLen = 0; 71 | } 72 | 73 | void Flush(byte[] buff, int offset, int count) 74 | { 75 | var compressedSize = SnappyCodec.Compress(buff, offset, count, _compressedBuffer, 0); 76 | 77 | BigEndianConverter.Write(_base, compressedSize); 78 | _base.Write(_compressedBuffer, 0, compressedSize); 79 | } 80 | 81 | public override long Seek(long offset, SeekOrigin origin) 82 | { 83 | throw new NotImplementedException(); 84 | } 85 | 86 | public override void SetLength(long value) 87 | { 88 | throw new NotImplementedException(); 89 | } 90 | 91 | public override int Read(byte[] buffer, int offset, int count) 92 | { 93 | if(!CanRead) 94 | throw new InvalidOperationException("Not a read stream"); 95 | 96 | if(_bufferPtr < _bufferLen) 97 | return ReadInternalBuffer(buffer, offset, count); 98 | 99 | if (!ReadBlock()) 100 | return 0; 101 | else 102 | return ReadInternalBuffer(buffer, offset, count); 103 | } 104 | 105 | public override void Write(byte[] buffer, int offset, int count) 106 | { 107 | if(!CanWrite) 108 | throw new InvalidOperationException("Not a write stream"); 109 | 110 | while(count > 0) 111 | { 112 | var size = Math.Min(count, _uncompressedBuffer.Length - _bufferLen); 113 | 114 | // Optimization: if full frame is going to be flushed, skip array copying and compress stright from input buffer 115 | if (_bufferLen == 0 && size == _uncompressedBuffer.Length) 116 | { 117 | Flush(buffer, offset, size); 118 | count -= size; 119 | offset += size; 120 | continue; 121 | } 122 | 123 | Buffer.BlockCopy(buffer, offset, _uncompressedBuffer, _bufferLen, size); 124 | _bufferLen += size; 125 | count -= size; 126 | offset += size; 127 | 128 | if(_bufferLen == _uncompressedBuffer.Length) 129 | Flush(); 130 | } 131 | } 132 | 133 | public override bool CanRead => _mode == CompressionStreamMode.Decompress; 134 | public override bool CanSeek => false; 135 | public override bool CanWrite => _mode == CompressionStreamMode.Compress; 136 | public override long Length { get { throw new NotImplementedException(); } } 137 | public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } } 138 | 139 | // 140 | // Implementation 141 | // 142 | 143 | bool ReadHeader() 144 | { 145 | // 146 | // Validate snappy header 147 | // 148 | if (!StreamUtils.ReadAll(_base, _headerWorkingBuffer, SnappyHeaderLen)) 149 | return false; 150 | 151 | for(int i=0; i<_snappyMagic.Length; i++) 152 | if(_headerWorkingBuffer[i] != _snappyMagic[i]) 153 | throw new InvalidDataException("Invalid snappy magic bytes"); 154 | 155 | // TODO: validate snappy min compatible version 156 | 157 | return true; 158 | } 159 | 160 | void WriteHeader() 161 | { 162 | _base.Write(_snappyMagic, 0, _snappyMagic.Length); 163 | _base.Write(_versionsHeader, 0, 8); 164 | } 165 | 166 | bool ReadBlock() 167 | { 168 | if (!StreamUtils.ReadAll(_base, _compressedBuffer, 4)) 169 | return false; 170 | 171 | var blockSize = BigEndianConverter.ReadInt32(_compressedBuffer); 172 | 173 | if (blockSize > _compressedBuffer.Length) 174 | _compressedBuffer = new byte[blockSize]; 175 | 176 | if(!StreamUtils.ReadAll(_base, _compressedBuffer, blockSize)) 177 | throw new InvalidDataException("Unexpected end of snappy data block"); 178 | 179 | // Reallocate compressed buffer if needed 180 | var uncompressedLen = Snappy.SnappyCodec.GetUncompressedLength(_compressedBuffer, 0, blockSize); 181 | if (uncompressedLen > _uncompressedBuffer.Length) 182 | _uncompressedBuffer = new byte[uncompressedLen]; 183 | 184 | var read = SnappyCodec.Uncompress(_compressedBuffer, 0, blockSize, _uncompressedBuffer, 0); 185 | 186 | _bufferLen = read; 187 | _bufferPtr = 0; 188 | 189 | return true; 190 | } 191 | 192 | int ReadInternalBuffer(byte[] buffer, int offset, int count) 193 | { 194 | var canRead = Math.Min(count, _bufferLen - _bufferPtr); 195 | Buffer.BlockCopy(_uncompressedBuffer, _bufferPtr, buffer, offset, canRead); 196 | _bufferPtr += canRead; 197 | return canRead; 198 | } 199 | 200 | protected override void Dispose(bool disposing) 201 | { 202 | if(_mode == CompressionStreamMode.Compress && _bufferLen != 0) 203 | Flush(); 204 | 205 | base.Dispose(disposing); 206 | } 207 | } 208 | 209 | enum CompressionStreamMode 210 | { 211 | Compress, 212 | Decompress 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/Compression/StreamUtils.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace kafka4net.Compression 4 | { 5 | static class StreamUtils 6 | { 7 | internal static bool ReadAll(Stream stream, byte[] dst, int count) 8 | { 9 | while (true) 10 | { 11 | var res = stream.Read(dst, 0, count); 12 | count -= res; 13 | if (count == 0) 14 | return true; 15 | if (res == 0) 16 | return false; 17 | } 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/CompressionType.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net 2 | { 3 | public enum CompressionType 4 | { 5 | None = 0, 6 | Gzip = 1, 7 | Snappy = 2, 8 | 9 | // Beaware of https://issues.apache.org/jira/browse/KAFKA-3160 10 | Lz4 = 3, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/ConnState.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net 2 | { 3 | enum ConnState 4 | { 5 | None, 6 | Connecting, 7 | Connected, 8 | Failed 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Connection.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Net.Sockets; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using kafka4net.Protocols; 7 | using kafka4net.Tracing; 8 | 9 | namespace kafka4net 10 | { 11 | internal class Connection 12 | { 13 | private static readonly ILogger _log = Logger.GetLogger(); 14 | 15 | private readonly string _host; 16 | private readonly int _port; 17 | private readonly Action _onError; 18 | private TcpClient _client; 19 | private readonly SemaphoreSlim _connectionLock = new SemaphoreSlim(1, 1); 20 | private bool _closed; 21 | private Task _loopTask; 22 | private CancellationTokenSource _loopTaskCancel; 23 | 24 | internal ResponseCorrelation Correlation; 25 | 26 | internal Connection(string host, int port, Action onError = null) 27 | { 28 | _host = host; 29 | _port = port; 30 | _onError = onError; 31 | } 32 | 33 | 34 | internal static Tuple[] ParseAddress(string seedConnections) 35 | { 36 | return seedConnections.Split(','). 37 | Select(_ => _.Trim()). 38 | Where(_ => _ != null). 39 | Select(s => 40 | { 41 | int port = 9092; 42 | string host = null; 43 | if (s.Contains(':')) 44 | { 45 | var parts = s.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries); 46 | if (parts.Length == 2) 47 | { 48 | host = parts[0]; 49 | port = int.Parse(parts[1]); 50 | } 51 | } 52 | else 53 | { 54 | host = s; 55 | } 56 | return Tuple.Create(host, port); 57 | }).ToArray(); 58 | } 59 | 60 | internal async Task GetClientAsync(bool noTransportErrors=false) 61 | { 62 | if (_closed) 63 | throw new ApplicationException("Attempt to reuse connection which is not supposed to be used anymore"); 64 | 65 | //EtwTrace.Log.ConnectionWaitingForLock(_host, _port); 66 | await _connectionLock.WaitAsync(); 67 | //EtwTrace.Log.ConnectionGotLock(_host, _port); 68 | 69 | try 70 | { 71 | if (_client != null && !_client.Connected) 72 | { 73 | _log.Debug("Replacing closed connection {0}:{1} with a new one", _host, _port); 74 | EtwTrace.Log.ConnectionReplaceClosedClient(_host, _port); 75 | _client = null; 76 | } 77 | 78 | if (_client == null) 79 | { 80 | _client = new TcpClient(); 81 | EtwTrace.Log.ConnectionConnecting(_host, _port); 82 | await _client.ConnectAsync(_host, _port); 83 | EtwTrace.Log.ConnectionConnected(_host, _port); 84 | 85 | var currentClient = _client; 86 | Correlation = new ResponseCorrelation(async e => 87 | { 88 | await MarkSocketAsFailed(currentClient); 89 | if(_onError != null && !noTransportErrors) 90 | _onError(e); 91 | }, _host+":"+_port+" conn object hash: " + GetHashCode()); 92 | 93 | // If there was a prior connection and loop task, cancel them before creating a new one. 94 | if (_loopTask != null && _loopTaskCancel != null) 95 | { 96 | _loopTaskCancel.Cancel(); 97 | _loopTask = null; 98 | } 99 | 100 | _loopTaskCancel = new CancellationTokenSource(); 101 | 102 | // Close connection in case of any exception. It is important, because in case of deserialization exception, 103 | // we are out of sync and can't continue. 104 | _loopTask = Correlation.CorrelateResponseLoop(_client, _loopTaskCancel.Token) 105 | .ContinueWith(async t => 106 | { 107 | _log.Debug("CorrelationLoop completed with status {0}. {1}", t.Status, t.Exception==null?"": string.Format("Closing connection because of error. {0}",t.Exception.Message)); 108 | await _connectionLock.WaitAsync();//.ConfigureAwait(false); 109 | try 110 | { 111 | if (_client != null) 112 | { 113 | _client.Close(); 114 | EtwTrace.Log.ConnectionDisconnected(_host, _port); 115 | } 116 | } 117 | // ReSharper disable once EmptyGeneralCatchClause 118 | catch {} 119 | finally 120 | { 121 | _client = null; 122 | _connectionLock.Release(); 123 | } 124 | }); 125 | 126 | return _client; 127 | } 128 | } 129 | catch(Exception e) 130 | { 131 | // some exception getting a connection, clear what we have for next time. 132 | _client = null; 133 | if (_onError != null && !noTransportErrors) 134 | _onError(e); 135 | throw; 136 | } 137 | finally 138 | { 139 | _connectionLock.Release(); 140 | //EtwTrace.Log.ConnectionLockRelease(_host, _port); 141 | } 142 | 143 | return _client; 144 | } 145 | 146 | public override string ToString() 147 | { 148 | return string.Format("Connection: {0}:{1}", _host, _port); 149 | } 150 | 151 | /// 152 | /// If correlation loop experienced an error, data in Tcp stream become unreliable because we've lost synchronization. 153 | /// So it is needed to close socket and reopen a new one later to recover from error. 154 | /// 155 | /// 156 | /// 157 | async Task MarkSocketAsFailed(TcpClient tcp) 158 | { 159 | await _connectionLock.WaitAsync(); 160 | try 161 | { 162 | if (_client != tcp) 163 | return; 164 | 165 | _log.Debug("Marking connection as failed. {0}", this); 166 | 167 | if (_client != null) 168 | { 169 | try 170 | { 171 | if (_loopTaskCancel != null) 172 | { 173 | EtwTrace.Log.Connection_MarkSocketAsFailed_CorrelationLoopCancelling(_host, _port); 174 | _loopTaskCancel.Cancel(); 175 | } 176 | EtwTrace.Log.Connection_MarkSocketAsFailed_TcpClosing(_host, _port); 177 | _client.Close(); 178 | } 179 | // ReSharper disable once EmptyGeneralCatchClause 180 | catch { /*empty*/ } 181 | finally { EtwTrace.Log.ConnectionDisconnected(_host, _port); } 182 | _client = null; 183 | } 184 | } 185 | finally 186 | { 187 | _connectionLock.Release(); 188 | } 189 | } 190 | 191 | /// 192 | /// Is called when Cluster is shut down, or when seed broker is replaced with a permanent one. 193 | /// 194 | public async Task ShutdownAsync() 195 | { 196 | _closed = true; 197 | _log.Debug("{0} Shutting down.", this); 198 | try 199 | { 200 | if (_loopTaskCancel != null) 201 | _loopTaskCancel.Cancel(); 202 | 203 | if (_loopTask != null) 204 | { 205 | try 206 | { 207 | _client.Close(); 208 | await _loopTask; 209 | _log.Debug("Loop task completed"); 210 | } 211 | catch (TaskCanceledException){} 212 | } 213 | } 214 | // ReSharper disable once EmptyGeneralCatchClause 215 | catch { } 216 | _log.Debug("{0} closed.", this); 217 | } 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/ConsumerConfiguration.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reactive.Concurrency; 3 | using kafka4net.ConsumerImpl; 4 | 5 | namespace kafka4net 6 | { 7 | /// 8 | /// Where to begin consuming. 9 | /// - TopicHead starts at the first message in each partition (beginning of the queue). 10 | /// - TopicTail starts at the most recent message of each partition (end of the queue) 11 | /// - SpecifiedLocations starts at the locations specified by the PartitionOffsetProvider function (see constructor paramters). 12 | /// 13 | public enum ConsumerLocation : long 14 | { 15 | TopicStart = -2L, 16 | TopicEnd = -1L, 17 | SpecifiedLocations 18 | } 19 | 20 | /// 21 | /// Interface to configuration to describe to the consumer where to start consuming (and which partitions to consume from) 22 | /// 23 | public interface IStartPositionProvider 24 | { 25 | /// 26 | /// Called to determine which partitions to consume from. Return true to consume from it or false not to. 27 | /// If this method returns true, it is expected that GetStartOffset will return a valid offset. 28 | /// 29 | /// The partition ID to check if we should consume from 30 | /// true to subscribe and consume from this partition, false to skip it 31 | bool ShouldConsumePartition(int partitionId); 32 | 33 | /// 34 | /// Gets the starting offset to use to consume at for this partition. 35 | /// 36 | /// The partition ID 37 | /// The next offset to consume. This will be the next offset consumed. 38 | long GetStartOffset(int partitionId); 39 | 40 | /// 41 | /// The logical start location represented by this IStartPositionProvider. 42 | /// If this returns one of TopicHead or TopicTail, then all partitions returned from ShouldConsumePartition 43 | /// Will be started from their current head or tail offset, and the GetStartOffset method will not be called. 44 | /// 45 | ConsumerLocation StartLocation { get; } 46 | } 47 | public interface IStopPositionProvider 48 | { 49 | /// 50 | /// Determines whether the consumer is finished consuming from a given partition based on the current message. 51 | /// Once this function returns false, the current message will be delivered but no further messages will be consumed from this partition. 52 | /// 53 | /// 54 | /// 55 | bool IsPartitionConsumingComplete(ReceivedMessage currentMessage); 56 | } 57 | 58 | public class ConsumerConfiguration 59 | { 60 | /// 61 | /// Subscription is performed asynchronously. 62 | /// 63 | /// Comma separated list of seed brokers. Port numbers are optional. 64 | /// 192.168.56.10,192.168.56.20:8081,broker3.local.net:8181 65 | /// 66 | /// 67 | /// 68 | /// 69 | /// 70 | /// 71 | /// 72 | /// 73 | /// If set to true, subscriber must call consumers Ack function to keep data flowing. 74 | /// Is used to prevent out of memory errors when subscriber is slow and data velocity is high 75 | /// (re-reading the log from beginning for example). 76 | /// Make sure that subscriber consumes more than lowWatermark, or data might stop flowing because driver would 77 | /// wait for subscriber to drain until lowWatermark and subscriber would wait for more data until continue processing 78 | /// (could happen when buffering is used). 79 | /// 80 | /// 81 | /// 82 | /// Driver will schedule outgoing messages events using this scheduler. By default it is 83 | /// EventLoopScheduler. Be careful if you want to redefine it. Concurrent scheduler can rearrange order of messages 84 | /// within the same partition! 85 | public ConsumerConfiguration( 86 | string seedBrokers, 87 | string topic, 88 | IStartPositionProvider startPosition, 89 | int maxWaitTimeMs=500, 90 | int minBytesPerFetch = 1, 91 | int maxBytesPerFetch=256*1024, 92 | int lowWatermark = 500, 93 | int highWatermark = 2000, 94 | bool useFlowControl = false, 95 | IStopPositionProvider stopPosition = null, 96 | IScheduler scheduler = null) 97 | { 98 | LowWatermark = lowWatermark; 99 | HighWatermark = highWatermark; 100 | UseFlowControl = useFlowControl; 101 | 102 | if(lowWatermark < 0) 103 | throw new ArgumentException("Can not be negative", "lowWatermark"); 104 | 105 | if (highWatermark < 0) 106 | throw new ArgumentException("Can not be negative", "highWatermark"); 107 | 108 | if(highWatermark < lowWatermark) 109 | throw new InvalidOperationException("highWatermark must be greater than lowWatermark"); 110 | 111 | SeedBrokers = seedBrokers; 112 | StartPosition = startPosition; 113 | Topic = topic; 114 | MaxWaitTimeMs = maxWaitTimeMs; 115 | MinBytesPerFetch = minBytesPerFetch; 116 | MaxBytesPerFetch = maxBytesPerFetch; 117 | StopPosition = stopPosition ?? new StopPositionNever(); 118 | OutgoingScheduler = scheduler ?? Scheduler.Default/*(ts => new Thread(ts) { IsBackground = true, Name = "Consumer outgoing scheduler" })*/; 119 | } 120 | 121 | public string SeedBrokers { get; private set; } 122 | public IStartPositionProvider StartPosition { get; private set; } 123 | public IStopPositionProvider StopPosition { get; private set; } 124 | public string Topic { get; private set; } 125 | public int MaxWaitTimeMs { get; private set; } 126 | public int MinBytesPerFetch { get; private set; } 127 | public int MaxBytesPerFetch { get; private set; } 128 | public readonly int LowWatermark; 129 | public readonly int HighWatermark; 130 | public readonly bool UseFlowControl; 131 | public readonly IScheduler OutgoingScheduler; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/ConsumerImpl/PartitionFetchState.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net.ConsumerImpl 2 | { 3 | class PartitionFetchState 4 | { 5 | public readonly int PartId; 6 | public readonly ConsumerLocation StartLocation; 7 | public long Offset; 8 | 9 | public PartitionFetchState(int partId, ConsumerLocation startLocation, long offset) 10 | { 11 | PartId = partId; 12 | StartLocation = startLocation; 13 | Offset = offset; 14 | } 15 | 16 | public override string ToString() 17 | { 18 | return string.Format("Part: {0}, Offset: {1} StartLocation: {2}", PartId, Offset, StartLocation); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/ConsumerImpl/PositionProviders.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace kafka4net.ConsumerImpl 5 | { 6 | public class StopPositionNever : IStopPositionProvider 7 | { 8 | public bool IsPartitionConsumingComplete(ReceivedMessage currentMessage) 9 | { 10 | return false; 11 | } 12 | } 13 | 14 | public class StartAndStopAtExplicitOffsets : IStartPositionProvider, IStopPositionProvider 15 | { 16 | private readonly TopicPartitionOffsets _startingOffsets; 17 | private readonly TopicPartitionOffsets _stoppingOffsets; 18 | 19 | public StartAndStopAtExplicitOffsets(Dictionary startingOffsets, Dictionary stoppingOffsets) 20 | { 21 | _startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets); 22 | _stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets); 23 | } 24 | public StartAndStopAtExplicitOffsets(IEnumerable> startingOffsets, IEnumerable> stoppingOffsets) 25 | { 26 | _startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets); 27 | _stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets); 28 | } 29 | public StartAndStopAtExplicitOffsets(IEnumerable> startingOffsets, IEnumerable> stoppingOffsets) 30 | { 31 | _startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets); 32 | _stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets); 33 | } 34 | public StartAndStopAtExplicitOffsets(TopicPartitionOffsets startingOffsets, TopicPartitionOffsets stoppingOffsets) 35 | { 36 | _startingOffsets = startingOffsets; 37 | _stoppingOffsets = stoppingOffsets; 38 | } 39 | 40 | public bool ShouldConsumePartition(int partitionId) 41 | { 42 | // we should only consume from this partition if we were told to by the start offsets and also if we are not stopping where we already are. 43 | return _startingOffsets.ShouldConsumePartition(partitionId) && _startingOffsets.NextOffset(partitionId) != _stoppingOffsets.NextOffset(partitionId); 44 | } 45 | 46 | public long GetStartOffset(int partitionId) 47 | { 48 | return _startingOffsets.GetStartOffset(partitionId); 49 | } 50 | 51 | public ConsumerLocation StartLocation 52 | { 53 | get { return ConsumerLocation.SpecifiedLocations; } 54 | } 55 | 56 | public bool IsPartitionConsumingComplete(ReceivedMessage currentMessage) 57 | { 58 | return _stoppingOffsets.IsPartitionConsumingComplete(currentMessage); 59 | } 60 | } 61 | 62 | public class StartPositionTopicEnd : StartPositionTopicTime 63 | { 64 | public override ConsumerLocation StartLocation { get { return ConsumerLocation.TopicEnd; } } 65 | } 66 | 67 | public class StartPositionTopicStart : StartPositionTopicTime 68 | { 69 | public override ConsumerLocation StartLocation { get { return ConsumerLocation.TopicStart; } } 70 | } 71 | 72 | public abstract class StartPositionTopicTime : IStartPositionProvider 73 | { 74 | private readonly ISet _partitionsToConsume; 75 | 76 | public StartPositionTopicTime() : this(null) { } 77 | public StartPositionTopicTime(IEnumerable partitionsToConsume) : this(new HashSet(partitionsToConsume)) {} 78 | public StartPositionTopicTime(ISet partitionsToConsume) 79 | { 80 | _partitionsToConsume = partitionsToConsume; 81 | } 82 | 83 | public bool ShouldConsumePartition(int partitionId) { return _partitionsToConsume == null || _partitionsToConsume.Contains(partitionId); } 84 | public long GetStartOffset(int partitionId) { throw new NotImplementedException(); } 85 | public abstract ConsumerLocation StartLocation { get; } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/ConsumerImpl/TopicPartition.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Reactive.Disposables; 4 | using System.Reactive.Linq; 5 | using kafka4net.Internal; 6 | 7 | namespace kafka4net.ConsumerImpl 8 | { 9 | class TopicPartition : IObserver 10 | { 11 | private static readonly ILogger _log = Logger.GetLogger(); 12 | private Consumer _subscribedConsumer; 13 | private readonly Cluster _cluster; 14 | private readonly string _topic; 15 | private readonly int _partitionId; 16 | private readonly PartitionFetchState _partitionFetchState; 17 | 18 | // subscription handles 19 | private IDisposable _fetcherChangesSubscription; 20 | private IDisposable _currentfetcherSubscription; 21 | 22 | public TopicPartition(Cluster cluster, string topic, int partitionId, long initialOffset) 23 | { 24 | _cluster = cluster; 25 | _topic = topic; 26 | _partitionId = partitionId; 27 | _partitionFetchState = new PartitionFetchState( 28 | PartitionId, 29 | ConsumerLocation.SpecifiedLocations, 30 | initialOffset); 31 | } 32 | 33 | public string Topic { get { return _topic; } } 34 | public int PartitionId { get { return _partitionId; } } 35 | 36 | public long CurrentOffset 37 | { 38 | get 39 | { 40 | // if we have not been initialized, something went wrong. 41 | if (_partitionFetchState == null) 42 | throw new Exception("Partition Fetch State is not initialized. Has a consumer subscribed?"); 43 | 44 | return _partitionFetchState.Offset; 45 | } 46 | } 47 | 48 | public IObservable FlowControl { get { return _subscribedConsumer.FlowControl; } } 49 | public bool FlowControlEnabled { get { return _subscribedConsumer.FlowControlEnabled; } } 50 | 51 | /// 52 | /// Subscribe a consumer to this topic partition. The act of subscribing 53 | /// will cause this partition to seek out and connect to the "correct" Fetcher. 54 | /// 55 | /// The consumer subscribing. This is not IObservable because we want to subscribe to the FlowControlState of the consumer. 56 | /// 57 | public IDisposable Subscribe(Consumer consumer) 58 | { 59 | if (_subscribedConsumer != null && consumer != _subscribedConsumer) 60 | throw new Exception(string.Format("TopicPartition {0} is already subscribed to by a consumer!", this)); 61 | 62 | _subscribedConsumer = consumer; 63 | 64 | // subscribe to fetcher changes for this partition. 65 | // We will immediately get a call with the "current" fetcher if it is available, and connect to it then. 66 | var fetchers = new List(); 67 | _fetcherChangesSubscription = _cluster 68 | .GetFetcherChanges(_topic, _partitionId, consumer.Configuration). 69 | Do(fetchers.Add). 70 | Subscribe(OnNewFetcher,OnFetcherChangesError,OnFetcherChangesComplete); 71 | 72 | _log.Debug("Starting {0} fetchers", fetchers.Count); 73 | fetchers.ForEach(fetcher => fetcher.PartitionsUpdated()); 74 | 75 | // give back a handle to close this topic partition. 76 | return Disposable.Create(DisposeImpl); 77 | 78 | } 79 | 80 | /// 81 | /// Handle the connection of a fetcher (potentially a new fetcher) for this partition 82 | /// 83 | /// The fetcher to use to fetch for this TopicPartition 84 | private void OnNewFetcher(Fetcher newFetcher) 85 | { 86 | // first close the subscription to the old fetcher if there is one. 87 | if (_currentfetcherSubscription != null) 88 | _currentfetcherSubscription.Dispose(); 89 | 90 | // now subscribe to the new fetcher. This will begin pumping messages through to the consumer. 91 | if (newFetcher != null) 92 | { 93 | _log.Debug("{0} Received new fetcher. Fetcher: {1}. Subscribing to this fetcher.", this, newFetcher); 94 | _currentfetcherSubscription = newFetcher.Subscribe(this); 95 | newFetcher.PartitionsUpdated(); 96 | } 97 | } 98 | 99 | private void OnFetcherChangesComplete() 100 | { 101 | // we aren't getting any more fetcher changes... shouldn't happen! 102 | _log.Warn("{0} Received FetcherChanges OnComplete event.... shouldn't happen unless we're shutting down.", this); 103 | DisposeImpl(); 104 | } 105 | 106 | private void OnFetcherChangesError(Exception ex) 107 | { 108 | // we aren't getting any more fetcher changes... shouldn't happen! 109 | _log.Fatal(ex, "{0} Received FetcherChanges OnError event.... shouldn't happen!", this); 110 | DisposeImpl(); 111 | } 112 | 113 | /// 114 | /// Called when there is a new message received. Pass it to the Consumer 115 | /// 116 | /// 117 | public void OnNext(ReceivedMessage value) 118 | { 119 | if (_subscribedConsumer == null) 120 | { 121 | _log.Error("{0} Recieved Message with no subscribed consumer. Discarding message.", this); 122 | } 123 | else 124 | { 125 | if (_partitionFetchState.Offset <= value.Offset) 126 | { 127 | // if we are sending back an offset greater than we asked for then we likely skipped an offset. 128 | // This happens when using Log Compaction (see https://kafka.apache.org/documentation.html#compaction ) 129 | if (_partitionFetchState.Offset < value.Offset) 130 | _log.Info("{0} was expecting offset {1} but received larger offset {2}",this,_partitionFetchState.Offset,value.Offset); 131 | 132 | _subscribedConsumer.OnMessageArrivedInput.OnNext(value); 133 | // track that we handled this offset so the next time around, we fetch the next message 134 | _partitionFetchState.Offset = value.Offset + 1; 135 | } 136 | else 137 | { 138 | // the returned message offset was less than the offset we asked for, just skip this message. 139 | _log.Debug("{0} Skipping message offset {1} as it is less than requested offset {2}",this,value.Offset,_partitionFetchState.Offset); 140 | } 141 | 142 | } 143 | } 144 | 145 | public void OnError(Exception error) 146 | { 147 | // don't pass this error up to the consumer, log it and wait for a new fetcher 148 | _log.Warn("{0} Recieved Error from Fetcher. Waiting for new or updated Fetcher. Message: {1}", this, error.Message); 149 | _currentfetcherSubscription.Dispose(); 150 | _currentfetcherSubscription = null; 151 | _cluster.NotifyPartitionStateChange(new PartitionStateChangeEvent(Topic, PartitionId, ErrorCode.FetcherException)); 152 | } 153 | 154 | public void OnCompleted() 155 | { 156 | // this shouldn't happen, but don't pass this up to the consumer, log it and wait for a new fetcher 157 | _log.Warn("{0} Recieved OnComplete from Fetcher. Fetcher may have errored out. Waiting for new or updated Fetcher.", this); 158 | _currentfetcherSubscription.Dispose(); 159 | _currentfetcherSubscription = null; 160 | } 161 | 162 | private void DisposeImpl() 163 | { 164 | // just end the subscription to the current fetcher and to the consumer. 165 | if (_subscribedConsumer != null) 166 | { 167 | _subscribedConsumer.OnTopicPartitionComplete(this); 168 | _subscribedConsumer = null; 169 | } 170 | 171 | if (_fetcherChangesSubscription != null) 172 | { 173 | _fetcherChangesSubscription.Dispose(); 174 | _fetcherChangesSubscription = null; 175 | } 176 | 177 | if (_currentfetcherSubscription != null) 178 | { 179 | _currentfetcherSubscription.Dispose(); 180 | _currentfetcherSubscription = null; 181 | } 182 | } 183 | 184 | public override string ToString() 185 | { 186 | return string.Format("'{0}' part: {1} offset: {2}",Topic, PartitionId, CurrentOffset); 187 | } 188 | 189 | public override bool Equals(object obj) 190 | { 191 | if (obj == null) 192 | return false; 193 | 194 | var objTopicPartition = obj as TopicPartition; 195 | if (objTopicPartition == null) 196 | return false; 197 | 198 | return _cluster == objTopicPartition._cluster && _topic == objTopicPartition._topic && _partitionId == objTopicPartition._partitionId; 199 | } 200 | 201 | public override int GetHashCode() 202 | { 203 | unchecked // disable overflow, for the unlikely possibility that you 204 | { // are compiling with overflow-checking enabled 205 | int hash = 27; 206 | hash = (13 * hash) + _topic.GetHashCode(); 207 | hash = (13 * hash) + _partitionId.GetHashCode(); 208 | return hash; 209 | } 210 | } 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/ConsumerImpl/TopicPartitionOffsets.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | 6 | namespace kafka4net.ConsumerImpl 7 | { 8 | /// 9 | /// Container Class to aid in storing offsets for all partitions in a topic. The class stores internally the NEXT offset to process for a partition. When the topic is empty, the offsets stored would be 0 (the next offset to fetch) 10 | /// When initialized to the TopicTail, it will contain the offset of the NEXT message to be written to the partition. 11 | /// 12 | public class TopicPartitionOffsets : IStartPositionProvider, IStopPositionProvider 13 | { 14 | public readonly string Topic; 15 | private readonly Dictionary _offsets; 16 | 17 | /// 18 | /// Initializes a TopicPartitionOffsets structure from a dictionary. The offsets in the dictionary must be the NEXT offset to process for each partition, not the *Current* offset. 19 | /// 20 | /// 21 | /// 22 | public TopicPartitionOffsets(string topic, Dictionary initialOffsets) 23 | { 24 | Topic = topic; 25 | _offsets = new Dictionary(initialOffsets); 26 | } 27 | public TopicPartitionOffsets(string topic, IEnumerable> startingOffsets) 28 | { 29 | Topic = topic; 30 | _offsets = startingOffsets.ToDictionary(kv => kv.Key, kv => kv.Value); 31 | } 32 | public TopicPartitionOffsets(string topic, IEnumerable> startingOffsets) 33 | { 34 | Topic = topic; 35 | _offsets = startingOffsets.ToDictionary(kv => kv.Item1, kv => kv.Item2); 36 | } 37 | public TopicPartitionOffsets(string topic) 38 | { 39 | Topic = topic; 40 | _offsets = new Dictionary(); 41 | } 42 | 43 | /// 44 | /// Construct a new set of offsets from the given stream. It is assumed that the stream contains serialized data created with WriteOffsets and is positioned at the start of this data. 45 | /// 46 | /// 47 | public TopicPartitionOffsets(byte[] serializedBytes) : this(new MemoryStream(serializedBytes)) 48 | { 49 | } 50 | 51 | /// 52 | /// Construct a new set of offsets from the given stream. It is assumed that the stream contains serialized data created with WriteOffsets and is positioned at the start of this data. 53 | /// 54 | /// 55 | public TopicPartitionOffsets(Stream inputStream) 56 | { 57 | var reader = new BinaryReader(inputStream); 58 | Topic = reader.ReadString(); 59 | 60 | // read from the input stream and initialize the offsets. 61 | _offsets = ReadOffsets(reader); 62 | } 63 | 64 | /// 65 | /// Get the next offset to request for the given partition ID (current offset + 1) 66 | /// 67 | /// 68 | /// the next offset for the given partition 69 | public long NextOffset(int partitionId) 70 | { 71 | long offset; 72 | if (_offsets.TryGetValue(partitionId, out offset)) 73 | return offset; 74 | 75 | throw new Exception(string.Format("Partition ID {0} does not exist in offsets for topic {1}", partitionId, Topic)); 76 | } 77 | 78 | public IEnumerable Partitions { get { return new List(_offsets.Keys); }} 79 | 80 | public Dictionary GetPartitionsOffset { get { return new Dictionary(_offsets); } } 81 | 82 | /// 83 | /// Update the offset for the given partition ID to include the given offset. This will cause offset+1 to be returned by the NextOffset function. 84 | /// Note that this function expects to be passed the *Current* offset that has been processed. 85 | /// 86 | /// 87 | /// 88 | public void UpdateOffset(int partitionId, long offset) 89 | { 90 | _offsets[partitionId] = offset+1; 91 | } 92 | 93 | /// 94 | /// Returns the total number of messages difference between this set of offsets, and the given set. 95 | /// 96 | /// The earlier offsets. If the passed offsets are larger than this instance offsets, the result will be negative. 97 | /// 98 | public long MessagesSince(TopicPartitionOffsets priorOffsets) 99 | { 100 | if (_offsets.Count != priorOffsets._offsets.Count || Topic != priorOffsets.Topic) 101 | throw new ArgumentException("priorOffsets does not match Topic or Number of Partitions of this set of offsets."); 102 | 103 | return _offsets.Keys.Select(p => _offsets[p] - priorOffsets._offsets[p]).Sum(); 104 | } 105 | 106 | /// 107 | /// Serialize all offsets in this object to the given stream 108 | /// 109 | /// 110 | public void WriteOffsets(Stream stream) 111 | { 112 | var writer = new BinaryWriter(stream); 113 | writer.Write(Topic); 114 | // number of offsets 115 | writer.Write(_offsets.Count); 116 | foreach (var partOffset in _offsets) 117 | { 118 | writer.Write(partOffset.Key); 119 | writer.Write(partOffset.Value); 120 | } 121 | } 122 | 123 | public byte[] WriteOffsets() 124 | { 125 | var memStream = new MemoryStream(800); 126 | WriteOffsets(memStream); 127 | return memStream.ToArray(); 128 | } 129 | 130 | /// 131 | /// Deserialize all offsets in this object from the given stream. 132 | /// 133 | /// 134 | /// 135 | private Dictionary ReadOffsets(BinaryReader reader) 136 | { 137 | // number of offsets 138 | var offsetCount = reader.ReadInt32(); 139 | var offsets = new Dictionary(offsetCount); 140 | for (var i = 0; i < offsetCount; i++) 141 | { 142 | offsets.Add(reader.ReadInt32(),reader.ReadInt64()); 143 | } 144 | return offsets; 145 | } 146 | 147 | public override string ToString() 148 | { 149 | return string.Format("{0} Offsets:[{1}]",Topic,string.Join(",", _offsets.Select(o=>o.Key + "|" + o.Value))); 150 | } 151 | 152 | #region IStopPositionProvider 153 | public bool IsPartitionConsumingComplete(ReceivedMessage currentMessage) 154 | { 155 | // "NextOffset" is the "Next" offset, so subtract one to get the current offset. 156 | var stopOffset = NextOffset(currentMessage.Partition) - 1; 157 | return currentMessage.Offset == stopOffset; 158 | } 159 | #endregion IStopPositionProvider 160 | 161 | #region IStartPositionProvider 162 | public bool ShouldConsumePartition(int partitionId) 163 | { 164 | return _offsets.ContainsKey(partitionId); 165 | } 166 | 167 | public long GetStartOffset(int partitionId) 168 | { 169 | return NextOffset(partitionId); 170 | } 171 | 172 | public ConsumerLocation StartLocation 173 | { 174 | get { return ConsumerLocation.SpecifiedLocations; } 175 | } 176 | #endregion IStartPositionProvider 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/ErrorCode.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net 2 | { 3 | public enum ErrorCode : short 4 | { 5 | ///No error--it worked! 6 | NoError = 0, 7 | ///An unexpected server error 8 | Unknown = -1, 9 | ///The requested offset is outside the range of offsets maintained by the server for the given 10 | /// topic/partition. 11 | OffsetOutOfRange = 1, 12 | ///This indicates that a message contents does not match its CRC 13 | InvalidMessage = 2, 14 | ///This request is for a topic or partition that does not exist on this broker. 15 | UnknownTopicOrPartition = 3, 16 | ///The message has a negative size 17 | InvalidMessageSize = 4, 18 | ///This error is thrown if we are in the middle of a leadership election and there is currently 19 | /// no leader for this partition and hence it is unavailable for writes. 20 | LeaderNotAvailable = 5, 21 | ///This error is thrown if the client attempts to send messages to a replica that is not 22 | /// the leader for some partition. It indicates that the clients metadata is out of date. 23 | NotLeaderForPartition = 6, 24 | ///This error is thrown if the request exceeds the user-specified time limit in the request 25 | RequestTimedOut = 7, 26 | ///This is not a client facing error and is used only internally by intra-cluster 27 | /// broker communication 28 | BrokerNotAvailable = 8, 29 | ///Unused 30 | ReplicaNotAvailable = 9, 31 | ///The server has a configurable maximum message size to avoid unbounded memory allocation. 32 | /// This error is thrown if the client attempt to produce a message larger than this maximum. 33 | MessageSizeTooLarge = 10, 34 | ///Internal error code for broker-to-broker communication 35 | StaleControllerEpoch = 11, 36 | ///If you specify a string larger than configured maximum for offset metadata 37 | OffsetMetadataTooLarge = 12, 38 | /// The broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition) 39 | OffsetsLoadInProgress = 14, 40 | /// The broker returns this error code for consumer metadata requests or offset commit requests 41 | /// if the offsets topic has not yet been created 42 | ConsumerCoordinatorNotAvailable = 15, 43 | /// The broker returns this error code if it receives an offset fetch or commit request for a consumer group that it is not a coordinator for 44 | NotCoordinatorForConsumer = 16, 45 | 46 | // New in 0.8.2 47 | InvalidTopic = 17, 48 | MessageSetSizeTooLarge = 18, 49 | NotEnoughReplicas = 19, 50 | NotEnoughReplicasAfterAppend = 20, 51 | 52 | // Below are driver's error codes. They are not returned by kafka server but are "virtual" states, such as transport error for example. 53 | FetcherException = 1001, 54 | TransportError 55 | } 56 | 57 | static class ErrorCodeClassification 58 | { 59 | /// 60 | /// Compare error codes, ignoring ReplicaNotAvailable 61 | /// 62 | public static bool IsDifferent(this ErrorCode c1, ErrorCode c2) 63 | { 64 | if (c1 == ErrorCode.ReplicaNotAvailable) 65 | c1 = ErrorCode.NoError; 66 | if (c2 == ErrorCode.ReplicaNotAvailable) 67 | c2 = ErrorCode.NoError; 68 | 69 | return c1 != c2; 70 | } 71 | 72 | public static bool IsSuccess(this ErrorCode errorCode) 73 | { 74 | return errorCode == ErrorCode.NoError || errorCode == ErrorCode.ReplicaNotAvailable; 75 | } 76 | 77 | public static bool IsFailure(this ErrorCode errorCode) 78 | { 79 | return !IsSuccess(errorCode); 80 | } 81 | 82 | public static bool IsPermanentFailure(this ErrorCode errorCode) 83 | { 84 | return errorCode == ErrorCode.MessageSetSizeTooLarge || 85 | errorCode == ErrorCode.MessageSizeTooLarge || 86 | errorCode == ErrorCode.InvalidMessageSize || 87 | errorCode == ErrorCode.InvalidMessage || 88 | errorCode == ErrorCode.OffsetOutOfRange; 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/FletcherHashedMessagePartitioner.cs: -------------------------------------------------------------------------------- 1 | using kafka4net.Metadata; 2 | using System; 3 | using System.Threading; 4 | 5 | namespace kafka4net 6 | { 7 | /// 8 | /// Default message partitioner. Uses a Fletcher hash to assign given message key to a partition. If no key, then randomly selects a partition. 9 | /// 10 | internal class FletcherHashedMessagePartitioner : IMessagePartitioner 11 | { 12 | private readonly ThreadLocal _rnd = new ThreadLocal(() => new Random()); 13 | 14 | public PartitionMeta GetMessagePartition(Message message, PartitionMeta[] allPartitions) 15 | { 16 | var index = message.Key == null ? 17 | _rnd.Value.Next(allPartitions.Length) : 18 | Fletcher32HashOptimized(message.Key) % allPartitions.Length; 19 | 20 | return allPartitions[index]; 21 | 22 | } 23 | 24 | /// Optimized Fletcher32 checksum implementation. 25 | /// 26 | private uint Fletcher32HashOptimized(byte[] msg) 27 | { 28 | if (msg == null) 29 | return 0; 30 | var words = msg.Length; 31 | int i = 0; 32 | uint sum1 = 0xffff, sum2 = 0xffff; 33 | 34 | while (words != 0) 35 | { 36 | var tlen = words > 359 ? 359 : words; 37 | words -= tlen; 38 | do 39 | { 40 | sum2 += sum1 += msg[i++]; 41 | } while (--tlen != 0); 42 | sum1 = (sum1 & 0xffff) + (sum1 >> 16); 43 | sum2 = (sum2 & 0xffff) + (sum2 >> 16); 44 | } 45 | /* Second reduction step to reduce sums to 16 bits */ 46 | sum1 = (sum1 & 0xffff) + (sum1 >> 16); 47 | sum2 = (sum2 & 0xffff) + (sum2 >> 16); 48 | return (sum2 << 16 | sum1); 49 | } 50 | 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/ILogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net 4 | { 5 | internal interface ILogger 6 | { 7 | void Debug(string msg); 8 | void Debug(string msg, params object[] args); 9 | void Debug(Exception e, string msg, params object[] args); 10 | void Info(string msg); 11 | void Info(string msg, params object[] args); 12 | void Info(Exception e, string msg, params object[] args); 13 | void Warn(string msg); 14 | void Warn(string msg, params object[] args); 15 | void Warn(Exception e, string msg, params object[] args); 16 | void Error(string msg); 17 | void Error(string msg, params object[] args); 18 | void Error(Exception e, string msg, params object[] args); 19 | void Fatal(string msg); 20 | void Fatal(string msg, params object[] args); 21 | void Fatal(Exception e, string msg, params object[] args); 22 | bool IsDebugEnabled { get; } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/IMessagePartitioner.cs: -------------------------------------------------------------------------------- 1 | using kafka4net.Metadata; 2 | 3 | namespace kafka4net 4 | { 5 | /// 6 | /// Assign message to certain partition. 7 | /// WARNING: this interface is called in caller (client) thread. Thus, if caller is multithreaded, 8 | /// IMessagePartitioner implementation must be thread-safe. 9 | /// 10 | /// It is possible to move call to IMessagePartitioner at later stage in the driver, where it would be thread-safe, but it would mean that 11 | /// message will get its partition at unpredictable time from client point of view, which could be confusing. 12 | /// 13 | public interface IMessagePartitioner 14 | { 15 | PartitionMeta GetMessagePartition(Message message, PartitionMeta[] allPartitions); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Internal/PartitionStateChangeEvent.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net.Internal 4 | { 5 | internal class PartitionStateChangeEvent 6 | { 7 | public PartitionStateChangeEvent(string topic, int partitionId, ErrorCode errorCode) 8 | { 9 | Topic = topic; 10 | PartitionId = partitionId; 11 | ErrorCode = errorCode; 12 | } 13 | 14 | public readonly string Topic; 15 | public readonly int PartitionId; 16 | public readonly ErrorCode ErrorCode; 17 | 18 | public override bool Equals(object obj) 19 | { 20 | if (obj == null) 21 | return false; 22 | 23 | var pse = obj as PartitionStateChangeEvent; 24 | if (pse == null) 25 | return false; 26 | 27 | return Topic == pse.Topic && PartitionId == pse.PartitionId && ErrorCode == pse.ErrorCode; 28 | } 29 | 30 | public override int GetHashCode() 31 | { 32 | unchecked // disable overflow, for the unlikely possibility that you 33 | { // are compiling with overflow-checking enabled 34 | int hash = 27; 35 | hash = (13 * hash) + Topic.GetHashCode(); 36 | hash = (13 * hash) + PartitionId.GetHashCode(); 37 | hash = (13 * hash) + ErrorCode.GetHashCode(); 38 | return hash; 39 | } 40 | } 41 | 42 | public override string ToString() 43 | { 44 | return string.Format("PartitionStateChangeEvent: '{0}'/{1}/{2}", Topic, PartitionId, ErrorCode); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Logger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.Reflection; 4 | using System.Runtime.CompilerServices; 5 | 6 | namespace kafka4net 7 | { 8 | /// 9 | /// Dynamic logging binder. Allows to log to log4net and NLog without linking kafka4net with those libraries. 10 | /// If you want to enable kafka4net logs, configure your logger as usual and call Logger.SetupLog4Net() or Logger.SetupNLog(). 11 | /// 12 | public class Logger 13 | { 14 | static Func LoggerFactory = t => new NullLogger(); 15 | 16 | public static void SetupNLog() 17 | { 18 | var logManagerType = Type.GetType("NLog.LogManager, NLog"); 19 | var method = logManagerType.GetMethod("GetLogger", BindingFlags.InvokeMethod | BindingFlags.Public | BindingFlags.Static, null, CallingConventions.Standard, new[] {typeof(string)}, null); 20 | 21 | var retType = method.ReturnType; 22 | var delegateTypeGeneric = typeof(Func<,>); 23 | var delegateType = delegateTypeGeneric.MakeGenericType(new[] {typeof(string), retType}); 24 | var del = Delegate.CreateDelegate(delegateType, method); 25 | var func = (Func)del; 26 | LoggerFactory = t => new NLogAdaprter(func(t.FullName)); 27 | } 28 | 29 | public static void SetupLog4Net() 30 | { 31 | var logManagerType = Type.GetType("log4net.LogManager, log4net"); 32 | var method = logManagerType.GetMethod("GetLogger", BindingFlags.InvokeMethod | BindingFlags.Public | BindingFlags.Static, null, CallingConventions.Standard, new[] { typeof(string) }, null); 33 | 34 | var retType = method.ReturnType; 35 | var delegateTypeGeneric = typeof(Func<,>); 36 | var delegateType = delegateTypeGeneric.MakeGenericType(new[] { typeof(string), retType }); 37 | var del = Delegate.CreateDelegate(delegateType, method); 38 | var func = (Func)del; 39 | LoggerFactory = t => new Log4NetAdaprter(func(t.FullName)); 40 | } 41 | 42 | [MethodImpl(MethodImplOptions.NoInlining)] 43 | internal static ILogger GetLogger() 44 | { 45 | var frame = new StackFrame(1, false); 46 | var klass = frame.GetMethod().DeclaringType; 47 | return LoggerFactory(klass); 48 | } 49 | 50 | class NullLogger : ILogger 51 | { 52 | public void Debug(string msg) { } 53 | public void Debug(Exception e, string msg, params object[] args) {} 54 | public void Debug(string msg, params object[] args) { } 55 | public void Info(string msg) {} 56 | public void Info(string msg, params object[] args) {} 57 | public void Info(Exception e, string msg, params object[] args) {} 58 | public void Warn(string msg) {} 59 | public void Warn(string msg, params object[] args) { } 60 | public void Warn(Exception e, string msg, params object[] args) { } 61 | public void Error(string msg) {} 62 | public void Error(string msg, params object[] args) {} 63 | public void Error(Exception e, string msg, params object[] args) {} 64 | public void Fatal(string msg) {} 65 | public void Fatal(string msg, params object[] args) {} 66 | public void Fatal(Exception e, string msg, params object[] args) {} 67 | public bool IsDebugEnabled { get { return false; } } 68 | } 69 | 70 | class NLogAdaprter : ILogger 71 | { 72 | readonly dynamic _actualLogger; 73 | 74 | public NLogAdaprter(object actualLogger) 75 | { 76 | _actualLogger = actualLogger; 77 | } 78 | 79 | public void Debug(string msg) 80 | { 81 | _actualLogger.Debug(msg); 82 | } 83 | 84 | public void Debug(string msg, params object[] args) 85 | { 86 | _actualLogger.Debug(msg, args); 87 | } 88 | 89 | public void Debug(Exception e, string msg, params object[] args) 90 | { 91 | _actualLogger.Debug(string.Format(msg, args), e); 92 | } 93 | 94 | public bool IsDebugEnabled { get { return _actualLogger.IsDebugEnabled; } } 95 | 96 | public void Info(string msg) 97 | { 98 | _actualLogger.Info(msg); 99 | } 100 | 101 | public void Info(string msg, params object[] args) 102 | { 103 | _actualLogger.Info(msg, args); 104 | } 105 | 106 | public void Info(Exception e, string msg, params object[] args) 107 | { 108 | _actualLogger.Info(string.Format(msg, args), e); 109 | } 110 | 111 | public void Warn(string msg) 112 | { 113 | _actualLogger.Warn(msg); 114 | } 115 | 116 | public void Warn(string msg, params object[] args) 117 | { 118 | _actualLogger.Warn(msg, args); 119 | } 120 | 121 | public void Warn(Exception e, string msg, params object[] args) 122 | { 123 | _actualLogger.Warn(string.Format(msg, args), e); 124 | } 125 | 126 | public void Error(string msg) 127 | { 128 | _actualLogger.Error(msg); 129 | } 130 | 131 | public void Error(string msg, params object[] args) 132 | { 133 | _actualLogger.Error(msg, args); 134 | } 135 | 136 | public void Error(Exception e, string msg, params object[] args) 137 | { 138 | _actualLogger.Error(string.Format(msg, args), e); 139 | } 140 | 141 | public void Fatal(string msg) 142 | { 143 | _actualLogger.Fatal(msg); 144 | } 145 | 146 | public void Fatal(string msg, params object[] args) 147 | { 148 | _actualLogger.Fatal(msg, args); 149 | } 150 | 151 | public void Fatal(Exception e, string msg, params object[] args) 152 | { 153 | _actualLogger.Fatal(string.Format(msg, args), e); 154 | } 155 | } 156 | 157 | class Log4NetAdaprter : ILogger 158 | { 159 | readonly dynamic _actualLogger; 160 | 161 | public Log4NetAdaprter(object actualLogger) 162 | { 163 | _actualLogger = actualLogger; 164 | } 165 | 166 | public void Debug(string msg) 167 | { 168 | _actualLogger.Debug(msg); 169 | } 170 | 171 | public void Debug(string msg, params object[] args) 172 | { 173 | _actualLogger.DebugFormat(msg, args); 174 | } 175 | 176 | public void Debug(Exception e, string msg, params object[] args) 177 | { 178 | _actualLogger.Debug(string.Format(msg, args), e); 179 | } 180 | 181 | public bool IsDebugEnabled { get { return _actualLogger.IsDebugEnabled; } } 182 | 183 | public void Info(string msg) 184 | { 185 | _actualLogger.Info(msg); 186 | } 187 | 188 | public void Info(string msg, params object[] args) 189 | { 190 | _actualLogger.InfoFormat(msg, args); 191 | } 192 | 193 | public void Info(Exception e, string msg, params object[] args) 194 | { 195 | _actualLogger.Info(string.Format(msg, args), e); 196 | } 197 | 198 | public void Warn(string msg) 199 | { 200 | _actualLogger.Warn(msg); 201 | } 202 | 203 | public void Warn(string msg, params object[] args) 204 | { 205 | _actualLogger.WarnFormat(msg, args); 206 | } 207 | 208 | public void Warn(Exception e, string msg, params object[] args) 209 | { 210 | _actualLogger.Warn(string.Format(msg, args), e); 211 | } 212 | 213 | public void Error(string msg) 214 | { 215 | _actualLogger.Error(msg); 216 | } 217 | 218 | public void Error(string msg, params object[] args) 219 | { 220 | _actualLogger.ErrorFormat(msg, args); 221 | } 222 | 223 | public void Error(Exception e, string msg, params object[] args) 224 | { 225 | _actualLogger.Error(string.Format(msg, args), e); 226 | } 227 | 228 | public void Fatal(string msg) 229 | { 230 | _actualLogger.Fatal(msg); 231 | } 232 | 233 | public void Fatal(string msg, params object[] args) 234 | { 235 | _actualLogger.FatalFormat(msg, args); 236 | } 237 | 238 | public void Fatal(Exception e, string msg, params object[] args) 239 | { 240 | _actualLogger.Fatal(string.Format(msg, args), e); 241 | } 242 | } 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /src/Message.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net 2 | { 3 | public class Message 4 | { 5 | public byte[] Key; 6 | public byte[] Value; 7 | 8 | internal long Offset; 9 | 10 | /// 11 | /// Not part of the protocol. Is used to group messages belonging to the same partition before sending. 12 | /// Is set by Publisher before enqueing to partition queue. 13 | /// 14 | internal int PartitionId { get; set; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Metadata/BrokerMeta.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace kafka4net.Metadata 5 | { 6 | class BrokerMeta 7 | { 8 | public int NodeId; 9 | public string Host; 10 | public int Port; 11 | 12 | // Not serialized, just a link to connection associated with this broker 13 | internal Connection Conn; 14 | 15 | public override string ToString() 16 | { 17 | return string.Format("{0}:{1} Id:{2}", Host, Port, NodeId); 18 | } 19 | 20 | #region comparer 21 | public static readonly IEqualityComparer NodeIdComparer = new ComparerImpl(); 22 | class ComparerImpl : IEqualityComparer 23 | { 24 | public bool Equals(BrokerMeta x, BrokerMeta y) 25 | { 26 | if(x.NodeId != -99 || y.NodeId != -99) 27 | return x.NodeId == y.NodeId; 28 | 29 | // If those are non-resolved seed brokers, do property comparison, because they all have NodeId==-99 30 | return string.Equals(x.Host, y.Host, StringComparison.OrdinalIgnoreCase) && x.Port == y.Port; 31 | } 32 | 33 | public int GetHashCode(BrokerMeta obj) 34 | { 35 | return obj.NodeId; 36 | } 37 | } 38 | #endregion 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Metadata/PartitionMeta.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | 3 | namespace kafka4net.Metadata 4 | { 5 | public class PartitionMeta 6 | { 7 | /// 8 | /// ReplicaNotAvailable is not really an error. Use IsSuccess() function instead of checking this value 9 | /// 10 | public ErrorCode ErrorCode; 11 | public int Id; 12 | 13 | /// 14 | /// The node id for the kafka broker currently acting as leader for this partition. 15 | /// If no leader exists because we are in the middle of a leader election this id will be -1. 16 | /// 17 | public int Leader; 18 | 19 | /// The set of alive nodes that currently acts as slaves for the leader for this partition 20 | public int[] Replicas; 21 | 22 | /// The set subset of the replicas that are "caught up" to the leader 23 | public int[] Isr; 24 | 25 | public override string ToString() 26 | { 27 | return string.Format("Id: {0} Leader: {1} Error: {2}, Replicas: {3}, Isr: {4}", Id, Leader, ErrorCode, IntToStringList(Replicas), IntToStringList(Isr)); 28 | } 29 | 30 | static string IntToStringList(int[] ints) 31 | { 32 | if (ints == null) 33 | return ""; 34 | return string.Join(",", ints.Select(i => i.ToString()).ToArray()); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/Metadata/PartitionOffsetInfo.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net.Metadata 2 | { 3 | public class PartitionOffsetInfo 4 | { 5 | public int Partition; 6 | /// If queue is empty than -1 7 | public long Head; 8 | public long Tail; 9 | 10 | public override string ToString() 11 | { 12 | return string.Format("Partition: {0} Head: {1} Tail {2}", Partition, Head, Tail); 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Metadata/TopicMeta.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace kafka4net.Metadata 5 | { 6 | public class TopicMeta 7 | { 8 | public ErrorCode ErrorCode; 9 | public string TopicName; 10 | public PartitionMeta[] Partitions; 11 | 12 | public override string ToString() 13 | { 14 | return string.Format("Topic '{0}' {1} Partitions [{2}]", TopicName, ErrorCode, string.Join(",", Partitions.AsEnumerable())); 15 | } 16 | 17 | #region name comparer 18 | static readonly NameCompareImpl NameComparerInstance = new NameCompareImpl(); 19 | public static IEqualityComparer NameComparer { get { return NameComparerInstance; } } 20 | class NameCompareImpl : IEqualityComparer 21 | { 22 | public bool Equals(TopicMeta x, TopicMeta y) 23 | { 24 | return x.TopicName == y.TopicName; 25 | } 26 | 27 | public int GetHashCode(TopicMeta obj) 28 | { 29 | return obj.TopicName.GetHashCode(); 30 | } 31 | } 32 | #endregion 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/PartitionFailedException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net 4 | { 5 | class PartitionFailedException : Exception 6 | { 7 | public readonly string Topic; 8 | public readonly int Partition; 9 | public readonly ErrorCode ErrorCode; 10 | 11 | public PartitionFailedException(string topic, int partition, ErrorCode errorCode) 12 | { 13 | Topic = topic; 14 | Partition = partition; 15 | ErrorCode = errorCode; 16 | } 17 | 18 | public override string Message 19 | { 20 | get { return string.Format("Topic '{0}' partition {1} failed with error code {2}", Topic, Partition, ErrorCode); } 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/ProducerConfiguration.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net 4 | { 5 | public class ProducerConfiguration 6 | { 7 | /// 8 | /// Full constructor for ProducerConfiguration. Specify all settings. 9 | /// Default values are: 10 | /// RequiredAcks: 1 11 | /// Timeout: 1s 12 | /// BatchFlushTime: 500ms 13 | /// BatchFlushSize: 1000 14 | /// 15 | /// 16 | /// Number of Kafka servers required to acknowledge the write for it to be considered successful. Default 1. 17 | /// Max time to wait before flushing messages. Default 500ms. 18 | /// Max number of messages to accumulate before flushing messages. Default 1000. 19 | /// Whether or not to automatically expand the send buffers (a buffer per partition of messages waiting to be sent). If set to false, an OnPermError will be triggered with messages that fail to get added to a full buffer. 20 | /// The initial size (in number of messages) of each send buffer. There is one send buffer per partition. If AutoGrowSendBuffers is true, the size will be expanded if necessary. 21 | /// The maximum size of MessageSet to send to the server. If exceed server's Segment Size, 22 | /// server will fail with MessageSetSizeTooLarge error. Default is 1Gb 23 | /// Type of compression 24 | public ProducerConfiguration( 25 | string topic, 26 | TimeSpan? batchFlushTime = null, 27 | int batchFlushSize = 1000, 28 | short requiredAcks = 1, 29 | bool autoGrowSendBuffers = true, 30 | int sendBuffersInitialSize = 200, 31 | int maxMessageSetSizeInBytes = 1024 * 1024 * 1024, 32 | TimeSpan? producerRequestTimeout = null, 33 | IMessagePartitioner partitioner = null, 34 | CompressionType compressionType = CompressionType.None) 35 | { 36 | Topic = topic; 37 | BatchFlushTime = batchFlushTime ?? TimeSpan.FromMilliseconds(500); 38 | BatchFlushSize = batchFlushSize; 39 | RequiredAcks = requiredAcks; 40 | AutoGrowSendBuffers = autoGrowSendBuffers; 41 | SendBuffersInitialSize = sendBuffersInitialSize; 42 | MaxMessageSetSizeInBytes = maxMessageSetSizeInBytes; 43 | ProduceRequestTimeout = producerRequestTimeout ?? TimeSpan.FromSeconds(1); 44 | Partitioner = partitioner ?? new FletcherHashedMessagePartitioner(); 45 | CompressionType = compressionType; 46 | } 47 | 48 | public string Topic { get; private set; } 49 | public short RequiredAcks { get; private set; } 50 | public TimeSpan BatchFlushTime { get; private set; } 51 | public int BatchFlushSize { get; private set; } 52 | public TimeSpan ProduceRequestTimeout { get; private set; } 53 | public int ProduceRequestTimeoutMs { get { return (int)Math.Floor(ProduceRequestTimeout.TotalMilliseconds); } } 54 | public IMessagePartitioner Partitioner { get; private set; } 55 | public bool AutoGrowSendBuffers { get; private set; } 56 | public int SendBuffersInitialSize { get; private set; } 57 | public int MaxMessageSetSizeInBytes { get; private set; } 58 | public CompressionType CompressionType { get; set; } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | [assembly: AssemblyTitle("kafka4net")] 6 | [assembly: AssemblyDescription("C# Kafka driver")] 7 | [assembly: AssemblyCompany("NTent")] 8 | [assembly: AssemblyProduct("kafka4net")] 9 | [assembly: AssemblyCopyright("Copyright © NTent 2014")] 10 | 11 | [assembly: ComVisible(false)] 12 | [assembly: Guid("b47c3611-9b2d-4124-9a87-4a094a5dc928")] 13 | 14 | [assembly: InternalsVisibleTo("kafka4net-tests")] 15 | 16 | -------------------------------------------------------------------------------- /src/Protocols/CorrelationLoopException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net.Protocols 4 | { 5 | class CorrelationLoopException : Exception 6 | { 7 | public bool IsRequestedClose; 8 | public CorrelationLoopException(string message) : base(message) 9 | { 10 | 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Protocols/Protocol.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net.Sockets; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using kafka4net.Metadata; 6 | using kafka4net.Protocols.Requests; 7 | using kafka4net.Protocols.Responses; 8 | using kafka4net.Tracing; 9 | 10 | namespace kafka4net.Protocols 11 | { 12 | // TODO: can this class be made static? 13 | internal class Protocol 14 | { 15 | private static readonly ILogger _log = Logger.GetLogger(); 16 | private readonly Cluster _cluster; 17 | static readonly EtwTrace _etw = EtwTrace.Log; 18 | 19 | internal Protocol(Cluster cluster) 20 | { 21 | _cluster = cluster; 22 | } 23 | 24 | internal async Task Produce(ProduceRequest request) 25 | { 26 | var conn = request.Broker.Conn; 27 | var client = await conn.GetClientAsync(); 28 | _log.Debug("Sending ProduceRequest to {0}, Request: {1}", conn, request); 29 | if(_etw.IsEnabled()) 30 | _etw.ProtocolProduceRequest(request.ToString(), request.Broker.NodeId); 31 | 32 | var response = await conn.Correlation.SendAndCorrelateAsync( 33 | id => Serializer.Serialize(request, id), 34 | Serializer.GetProducerResponse, 35 | client, 36 | CancellationToken.None 37 | ); 38 | _log.Debug("Got ProduceResponse: {0}", response); 39 | if (_etw.IsEnabled()) 40 | _etw.ProtocolProduceResponse(response.ToString(), request.Broker.NodeId); 41 | 42 | return response; 43 | } 44 | 45 | internal async Task MetadataRequest(TopicRequest request, BrokerMeta broker = null, bool noTransportErrors = false) 46 | { 47 | TcpClient tcp; 48 | Connection conn; 49 | 50 | if (broker != null) 51 | { 52 | conn = broker.Conn; 53 | tcp = await conn.GetClientAsync(noTransportErrors); 54 | } 55 | else 56 | { 57 | var clientAndConnection = await _cluster.GetAnyClientAsync(); 58 | conn = clientAndConnection.Item1; 59 | tcp = clientAndConnection.Item2; 60 | } 61 | 62 | //var tcp = await (broker != null ? broker.Conn.GetClientAsync() : _cluster.GetAnyClientAsync()); 63 | _log.Debug("Sending MetadataRequest to {0}", tcp.Client.RemoteEndPoint); 64 | if (_etw.IsEnabled()) 65 | { 66 | _etw.ProtocolMetadataRequest(request.ToString()); 67 | } 68 | 69 | var response = await conn.Correlation.SendAndCorrelateAsync( 70 | id => Serializer.Serialize(request, id), 71 | Serializer.DeserializeMetadataResponse, 72 | tcp, CancellationToken.None); 73 | 74 | if (_etw.IsEnabled()) 75 | { 76 | _etw.ProtocolMetadataResponse(response.ToString(), 77 | broker != null ? broker.Host : "", 78 | broker != null ? broker.Port : -1, 79 | broker != null ? broker.NodeId : -1); 80 | } 81 | 82 | 83 | return response; 84 | } 85 | 86 | internal async Task GetOffsets(OffsetRequest req, Connection conn) 87 | { 88 | var tcp = await conn.GetClientAsync(); 89 | 90 | if(_etw.IsEnabled()) 91 | _etw.ProtocolOffsetRequest(req.ToString()); 92 | 93 | var response = await conn.Correlation.SendAndCorrelateAsync( 94 | id => Serializer.Serialize(req, id), 95 | Serializer.DeserializeOffsetResponse, 96 | tcp, CancellationToken.None); 97 | 98 | _log.Debug("Got OffsetResponse {0}", response); 99 | if (_etw.IsEnabled()) 100 | _etw.ProtocolOffsetResponse(response.ToString()); 101 | 102 | return response; 103 | } 104 | 105 | internal async Task Fetch(FetchRequest req, Connection conn) 106 | { 107 | _log.Debug("Sending FetchRequest to broker {1}. Request: {0}", req, conn); 108 | if (_etw.IsEnabled()) 109 | _etw.ProtocolFetchRequest(req.ToString()); 110 | 111 | // Detect disconnected server. Wait no less than 5sec. 112 | // If wait time exceed wait time + 3sec, consider it a timeout too 113 | //var timeout = Math.Max(5000, req.MaxWaitTime + 3000); 114 | //var cancel = new CancellationTokenSource(timeout); 115 | 116 | var tcp = await conn.GetClientAsync(); 117 | var response = await conn.Correlation.SendAndCorrelateAsync( 118 | id => Serializer.Serialize(req, id), 119 | Serializer.DeserializeFetchResponse, 120 | tcp, /*cancel.Token*/ CancellationToken.None); 121 | 122 | if (_etw.IsEnabled()) 123 | _etw.ProtocolFetchResponse(response.ToString()); 124 | 125 | return response; 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/Protocols/Requests/FetchRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | 3 | namespace kafka4net.Protocols.Requests 4 | { 5 | class FetchRequest 6 | { 7 | /// 8 | /// The max wait time is the maximum amount of time in milliseconds to block waiting if 9 | /// insufficient data is available at the time the request is issued. 10 | /// 11 | public int MaxWaitTime; 12 | 13 | /// 14 | /// This is the minimum number of bytes of messages that must be available to give a response. 15 | /// If the client sets this to 0 the server will always respond immediately, however if there is 16 | /// no new data since their last request they will just get back empty message sets. 17 | /// If this is set to 1, the server will respond as soon as at least one partition has 18 | /// at least 1 byte of data or the specified timeout occurs. By setting higher values in combination 19 | /// with the timeout the consumer can tune for throughput and trade a little additional latency 20 | /// for reading only large chunks of data (e.g. setting MaxWaitTime to 100 ms and setting 21 | /// MinBytes to 64k would allow the server to wait up to 100ms to try to accumulate 64k 22 | /// of data before responding). 23 | /// 24 | public int MinBytes; 25 | 26 | public TopicData[] Topics; 27 | 28 | public class TopicData 29 | { 30 | public string Topic; 31 | public PartitionData[] Partitions; 32 | 33 | public override string ToString() 34 | { 35 | return string.Format("Topic: {0} Parts: [{1}]", Topic, Partitions == null ? "null" : string.Join(", ", Partitions.AsEnumerable())); 36 | } 37 | } 38 | 39 | public class PartitionData 40 | { 41 | public int Partition; 42 | public long FetchOffset; 43 | public int MaxBytes; 44 | 45 | public override string ToString() 46 | { 47 | return string.Format("Partition: {0} Offset: {1} MaxBytes: {2}", Partition, FetchOffset, MaxBytes); 48 | } 49 | } 50 | 51 | public override string ToString() 52 | { 53 | return string.Format("MaxTime: {0} MinBytes: {1} [{2}]", MaxWaitTime, MinBytes, Topics == null ? "null" : string.Join(",", Topics.AsEnumerable())); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Protocols/Requests/MessageData.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net.Protocols.Requests 4 | { 5 | class MessageData 6 | { 7 | /// This byte holds metadata attributes about the message. The lowest 2 bits contain 8 | /// the compression codec used for the message. The other bits should be set to 0 9 | //public byte Attributes; 10 | 11 | /// The key is an optional message key that was used for partition assignment. 12 | /// The key can be null 13 | public byte[] Key; 14 | /// The value is the actual message contents as an opaque byte array. 15 | /// Kafka supports recursive messages in which case this may itself contain a message set. 16 | /// The message can be null 17 | public ArraySegment Value; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Protocols/Requests/MessageSetItem.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net.Protocols.Requests 2 | { 3 | // TODO: delete this class, its unused 4 | class MessageSetItem 5 | { 6 | /// This is the offset used in kafka as the log sequence number. When the producer is sending 7 | /// messages it doesn't actually know the offset and can fill in any value here it likes 8 | public long Offset; 9 | public int MessageSize; 10 | public MessageData MessageData; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Protocols/Requests/OffsetRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | 3 | namespace kafka4net.Protocols.Requests 4 | { 5 | class OffsetRequest 6 | { 7 | public string TopicName; 8 | public PartitionData[] Partitions; 9 | /// -1 for last offset, -2 for first offset, otherwise time 10 | //public long Time; 11 | 12 | public class PartitionData 13 | { 14 | public int Id; 15 | public long Time; 16 | public int MaxNumOffsets; 17 | 18 | public override string ToString() 19 | { 20 | return string.Format("[{0}:{1}:{2}]", Id, Time, MaxNumOffsets); 21 | } 22 | } 23 | 24 | public override string ToString() 25 | { 26 | return string.Format("{0} Id:Time:MaxNumOffsets [{1}]", TopicName, Partitions == null ? "null" : string.Join("\n ", Partitions.AsEnumerable())); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/Protocols/Requests/PartitionData.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace kafka4net.Protocols.Requests 5 | { 6 | class PartitionData 7 | { 8 | public int Partition; 9 | // is calculated at serizlization time 10 | //public int MessageSetSize; 11 | public IEnumerable Messages; 12 | 13 | /// Is not serialized. Is carried through to send error/success notifications 14 | /// if herror happen 15 | public Producer Pub; 16 | 17 | /// 18 | /// Not serialized. 19 | /// Copy of origianl, application provided messages. Is needed when error happen 20 | /// and driver notifys app that those messages have failed. 21 | /// 22 | public Message[] OriginalMessages; 23 | 24 | /// 25 | /// Non serialazable. Carries compression type until serialization happen, where messages are re-packaged 26 | /// into compressed message set. 27 | /// 28 | public CompressionType CompressionType; 29 | 30 | public override string ToString() 31 | { 32 | return string.Format("Part: {0} Messages: {1}", Partition, Messages == null ? "null" : Messages.Count().ToString()); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Protocols/Requests/ProduceRequest.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using kafka4net.Metadata; 3 | 4 | namespace kafka4net.Protocols.Requests 5 | { 6 | class ProduceRequest 7 | { 8 | /// 9 | /// This field indicates how many acknowledgements the servers should receive before responding to the request. 10 | /// If it is 0 the server will not send any response (this is the only case where the server will not reply 11 | /// to a request). If it is 1, the server will wait the data is written to the local log before sending 12 | /// a response. If it is -1 the server will block until the message is committed by all in sync replicas 13 | /// before sending a response. For any number > 1 the server will block waiting for this number of 14 | /// acknowledgements to occur (but the server will never wait for more acknowledgements than there 15 | /// are in-sync replicas). 16 | /// 17 | public short RequiredAcks; 18 | 19 | /// 20 | /// This provides a maximum time in milliseconds the server can await the receipt of the number 21 | /// of acknowledgements in RequiredAcks. The timeout is not an exact limit on the request time 22 | /// for a few reasons: (1) it does not include network latency, (2) the timer begins at the 23 | /// beginning of the processing of this request so if many requests are queued due to server 24 | /// overload that wait time will not be included, (3) we will not terminate a local write so if 25 | /// the local write time exceeds this timeout it will not be respected. To get a hard timeout of 26 | /// this type the client should use the socket timeout. 27 | /// 28 | public int Timeout; 29 | 30 | public IEnumerable TopicData; 31 | 32 | /// This propert is not serialized and is used only to carry connection info from grouping 33 | /// by broker to sending phase 34 | public BrokerMeta Broker; 35 | 36 | public override string ToString() 37 | { 38 | var topicDataStr = TopicData == null ? "" : string.Join("\n ", TopicData); 39 | return string.Format("Broker: {0} Acks: {1} Timeout: {2} Topics: [{3}]", Broker, RequiredAcks, Timeout, topicDataStr); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Protocols/Requests/TopicData.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace kafka4net.Protocols.Requests 4 | { 5 | class TopicData 6 | { 7 | public string TopicName; 8 | public IEnumerable PartitionsData; 9 | 10 | public override string ToString() 11 | { 12 | var partDataStr = PartitionsData == null ? "" : string.Join(", ", PartitionsData); 13 | return string.Format("Topic: {0} Parts: [{1}]", TopicName, partDataStr); 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Protocols/Requests/TopicRequest.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net.Protocols.Requests 2 | { 3 | class TopicRequest 4 | { 5 | public string[] Topics; 6 | 7 | public override string ToString() 8 | { 9 | return string.Format("[{0}]", string.Join(",", Topics)); 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Protocols/ResponseCorrelation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Net.Sockets; 6 | using System.Text; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | using kafka4net.Tracing; 10 | using kafka4net.Utils; 11 | 12 | namespace kafka4net.Protocols 13 | { 14 | class ResponseCorrelation 15 | { 16 | // It is possible to have id per connection, but it's just simpler code and debug/tracing when it's global 17 | private static int _correlationId; 18 | private readonly ConcurrentDictionary> _corelationTable = new ConcurrentDictionary>(); 19 | private readonly Action _onError; 20 | private static readonly ILogger _log = Logger.GetLogger(); 21 | readonly string _id; 22 | private CancellationToken _cancellation; 23 | static readonly EtwTrace _etw = EtwTrace.Log; 24 | 25 | public ResponseCorrelation(Action onError, string id = "") 26 | { 27 | _onError = onError; 28 | _id = id; 29 | _etw.CorrelationCreate(); 30 | } 31 | 32 | static async Task ReadBuffer(TcpClient client, byte[] buff, int len, CancellationToken cancel) 33 | { 34 | var pos = 0; 35 | var left = len; 36 | do 37 | { 38 | _etw.Correlation_ReadingBodyChunk(left); 39 | var read = await client.GetStream().ReadAsync(buff, pos, left, cancel); 40 | 41 | if (read == 0) 42 | { 43 | _log.Info("Server closed connection"); 44 | _etw.CorrelationServerClosedConnection(); 45 | throw new IOException("Server closed connection. 0 bytes read."); 46 | } 47 | 48 | pos += read; 49 | left -= read; 50 | _etw.CorrelationReadBodyChunk(read, left); 51 | } while (left > 0); 52 | 53 | return pos; 54 | } 55 | 56 | internal async Task CorrelateResponseLoop(TcpClient client, CancellationToken cancel) 57 | { 58 | try 59 | { 60 | _cancellation = cancel; 61 | _log.Debug("Starting reading loop from socket. {0}", _id); 62 | _etw.CorrelationStart(); 63 | 64 | var buff = new byte[16 * 1024]; 65 | 66 | while (client.Connected && !cancel.IsCancellationRequested) 67 | { 68 | try 69 | { 70 | // read message size 71 | //var buff = new byte[4]; 72 | _etw.CorrelationReadingMessageSize(); 73 | await ReadBuffer(client, buff, 4, cancel); 74 | if (cancel.IsCancellationRequested) 75 | { 76 | _log.Debug("Stopped reading from {0} because cancell requested", _id); 77 | return; 78 | } 79 | 80 | var size = BigEndianConverter.ToInt32(buff); 81 | _etw.CorrelationReadMessageSize(size); 82 | 83 | // TODO: size sanity check. What is the reasonable max size? 84 | //var body = new byte[size]; 85 | if(size > buff.Length) 86 | buff = new byte[size]; 87 | await ReadBuffer(client, buff, size, cancel); 88 | _etw.CorrelationReadBody(size); 89 | 90 | try 91 | { 92 | int correlationId = -1; 93 | // TODO: check read==size && read > 4 94 | correlationId = BigEndianConverter.ToInt32(buff); 95 | _etw.CorrelationReceivedCorrelationId(correlationId); 96 | 97 | // find correlated action 98 | Action handler; 99 | // TODO: if correlation id is not found, there is a chance of corrupt 100 | // connection. Maybe recycle the connection? 101 | if (!_corelationTable.TryRemove(correlationId, out handler)) 102 | { 103 | _log.Error("Unknown correlationId: " + correlationId); 104 | continue; 105 | } 106 | _etw.CorrelationExecutingHandler(); 107 | handler(buff, size, null); 108 | _etw.CorrelationExecutedHandler(); 109 | } 110 | catch (Exception ex) 111 | { 112 | var error = string.Format("Error with handling message. Message bytes:\n{0}\n", FormatBytes(buff, size)); 113 | _etw.CorrelationError(ex.Message + " " + error); 114 | _log.Error(ex, error); 115 | throw; 116 | } 117 | } 118 | catch (SocketException e) 119 | { 120 | // shorter version of socket exception, without stack trace dump 121 | _log.Info("CorrelationLoop socket exception. {0}. {1}", e.Message, _id); 122 | throw; 123 | } 124 | catch (ObjectDisposedException) 125 | { 126 | _log.Debug("CorrelationLoop socket exception. Object disposed. {0}", _id); 127 | throw; 128 | } 129 | catch (IOException) 130 | { 131 | _log.Info("CorrelationLoop IO exception. {0}", _id); 132 | throw; 133 | } 134 | catch (Exception e) 135 | { 136 | _log.Error(e, "CorrelateResponseLoop error. {0}", _id); 137 | throw; 138 | } 139 | } 140 | 141 | _log.Debug("Finished reading loop from socket. {0}", _id); 142 | EtwTrace.Log.CorrelationComplete(); 143 | } 144 | catch (Exception e) 145 | { 146 | _corelationTable.Values.ForEach(c => c(null, 0, e)); 147 | if (_onError != null && !cancel.IsCancellationRequested) // don't call back OnError if we were told to cancel 148 | _onError(e); 149 | 150 | if (!cancel.IsCancellationRequested) 151 | throw; 152 | } 153 | finally 154 | { 155 | _log.Debug("Finishing CorrelationLoop. Calling back error to clear waiters."); 156 | _corelationTable.Values.ForEach(c => c(null, 0, new CorrelationLoopException("Correlation loop closed. Request will never get a response.") { IsRequestedClose = cancel.IsCancellationRequested })); 157 | _log.Debug("Finished CorrelationLoop."); 158 | } 159 | } 160 | 161 | internal async Task SendAndCorrelateAsync(Func serialize, Func deserialize, TcpClient tcp, CancellationToken cancel) 162 | { 163 | var correlationId = Interlocked.Increment(ref _correlationId); 164 | 165 | var callback = new TaskCompletionSource(); 166 | // TODO: configurable timeout 167 | // TODO: coordinate with Message's timeout 168 | // TODO: set exception in case of tcp socket exception and end of correlation loop 169 | //var timeout = new CancellationTokenSource(5 * 1000); 170 | try 171 | { 172 | var res = _corelationTable.TryAdd(correlationId, (body, size, ex) => 173 | { 174 | if (ex == null) 175 | callback.TrySetResult(deserialize(body, size)); 176 | else 177 | callback.TrySetException(ex); 178 | //timeout.Dispose(); 179 | }); 180 | if (!res) 181 | throw new ApplicationException("Failed to add correlationId: " + correlationId); 182 | var buff = serialize(correlationId); 183 | 184 | // TODO: think how buff can be reused. Would it be safe to declare buffer as a member? Is there a guarantee of one write at the time? 185 | _etw.CorrelationWritingMessage(correlationId, buff.Length); 186 | await tcp.GetStream().WriteAsync(buff, 0, buff.Length, cancel); 187 | } 188 | catch (Exception e) 189 | { 190 | callback.TrySetException(e); 191 | if (_onError != null) 192 | _onError(e); 193 | throw; 194 | } 195 | 196 | // TODO: is stream safe to use after timeout? 197 | cancel.Register(() => callback.TrySetCanceled(), useSynchronizationContext: false); 198 | try 199 | { 200 | return await callback.Task; 201 | } 202 | catch (TaskCanceledException e) 203 | { 204 | if (_onError != null) 205 | _onError(e); 206 | throw; 207 | } 208 | } 209 | 210 | private static string FormatBytes(byte[] buff, int len) 211 | { 212 | return buff.Take(Math.Min(256, len)).Aggregate(new StringBuilder(), (builder, b) => builder.Append(b.ToString("x2")), 213 | str => str.ToString()); 214 | } 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/Protocols/Responses/FetchResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | 3 | namespace kafka4net.Protocols.Responses 4 | { 5 | class FetchResponse 6 | { 7 | public TopicFetchData[] Topics; 8 | 9 | public class TopicFetchData 10 | { 11 | public string Topic; 12 | public PartitionFetchData[] Partitions; 13 | 14 | public override string ToString() 15 | { 16 | return ToString(false); 17 | } 18 | 19 | public string ToString(bool onlyPartitionsWithMessages) 20 | { 21 | return string.Format("'{0}' PartId:ErrorCode:HighWatermarkOffset:MessageCount [{1}]", Topic, Partitions == null ? "null" : string.Join("\n ", Partitions.Where(p => !onlyPartitionsWithMessages || p.Messages.Length > 0).AsEnumerable())); 22 | } 23 | } 24 | 25 | public class PartitionFetchData 26 | { 27 | public int Partition; 28 | public ErrorCode ErrorCode; 29 | public long HighWatermarkOffset; 30 | public Message[] Messages; 31 | 32 | public override string ToString() 33 | { 34 | return string.Format("{0}:{1}:{2}:{3}", Partition, ErrorCode, HighWatermarkOffset, Messages == null ? "null" : Messages.Length.ToString()); 35 | } 36 | } 37 | 38 | public override string ToString() 39 | { 40 | return ToString(false); 41 | } 42 | 43 | public string ToString(bool onlyPartitionsWithMessages) 44 | { 45 | if (Topics == null) 46 | return "null"; 47 | 48 | return string.Format("[{0}]", string.Join("\n ", Topics.Select(t=>t.ToString(onlyPartitionsWithMessages)))); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Protocols/Responses/MetadataResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using kafka4net.Metadata; 3 | 4 | namespace kafka4net.Protocols.Responses 5 | { 6 | class MetadataResponse 7 | { 8 | public BrokerMeta[] Brokers; 9 | public TopicMeta[] Topics; 10 | 11 | public override string ToString() 12 | { 13 | return string.Format("Brokers: [{0}], TopicMeta: [{1}]", 14 | Brokers == null ? "null" : string.Join(",", Brokers.AsEnumerable()), 15 | Topics == null ? "null" : string.Join("\n", Topics.AsEnumerable()) 16 | ); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Protocols/Responses/OffsetResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Globalization; 2 | using System.Linq; 3 | 4 | namespace kafka4net.Protocols.Responses 5 | { 6 | class OffsetResponse 7 | { 8 | public string TopicName; 9 | public PartitionOffsetData[] Partitions; 10 | 11 | public class PartitionOffsetData 12 | { 13 | public int Partition; 14 | public ErrorCode ErrorCode; 15 | public long[] Offsets; 16 | 17 | public override string ToString() 18 | { 19 | return string.Format("{0}:{1}:[{2}]", Partition, ErrorCode, 20 | Offsets == null ? "null" : string.Join(",", Offsets.Select(o => o.ToString(CultureInfo.InvariantCulture)))); 21 | } 22 | } 23 | 24 | public override string ToString() 25 | { 26 | return string.Format("'{0}' [{1}]", TopicName, Partitions == null ? "null" : string.Join(",", Partitions.Select(p => p.ToString()))); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/Protocols/Responses/ProducerResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | 3 | namespace kafka4net.Protocols.Responses 4 | { 5 | class ProducerResponse 6 | { 7 | internal TopicResponse[] Topics; 8 | 9 | internal class TopicResponse 10 | { 11 | public string TopicName; 12 | public PartitionResponse[] Partitions; 13 | 14 | public override string ToString() 15 | { 16 | return string.Format("{0} [{1}]", TopicName, Partitions == null ? "null" : string.Join("\n ", Partitions.AsEnumerable())); 17 | } 18 | } 19 | 20 | internal class PartitionResponse 21 | { 22 | public int Partition; 23 | public ErrorCode ErrorCode; 24 | public long Offset; 25 | 26 | public override string ToString() 27 | { 28 | return string.Format("Part: {0}, Err: {1} Offset: {2}", Partition, ErrorCode, Offset); 29 | } 30 | } 31 | 32 | public override string ToString() 33 | { 34 | return string.Format("Topics: [{0}]", Topics == null ? "null" : string.Join("\n ", Topics.AsEnumerable())); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/ReceivedMessage.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net 2 | { 3 | public class ReceivedMessage 4 | { 5 | public string Topic; 6 | public int Partition; 7 | public byte[] Key; 8 | public byte[] Value; 9 | public long Offset; 10 | public long HighWaterMarkOffset; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Utils/AskEx.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reactive.Concurrency; 3 | using System.Threading.Tasks; 4 | 5 | namespace kafka4net.Utils 6 | { 7 | static class AskEx 8 | { 9 | public static Task Ask(this IScheduler scheduler, Func action) 10 | { 11 | var src = new TaskCompletionSource(); 12 | scheduler.Schedule(() => 13 | { 14 | try 15 | { 16 | var res = action(); 17 | src.SetResult(res); 18 | } 19 | catch (Exception e) 20 | { 21 | src.SetException(e); 22 | } 23 | }); 24 | 25 | return src.Task; 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Utils/BigEndianConverter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace kafka4net.Utils 5 | { 6 | static class BigEndianConverter 7 | { 8 | public static int ReadInt32(Stream s) 9 | { 10 | if(s.CanSeek && s.Position + 4 > s.Length) 11 | throw new Exception(string.Format("ReadInt32 needs 4 bytes but got ony {0}", s.Length - s.Position)); 12 | return s.ReadByte() << 3 * 8 | s.ReadByte() << 2 * 8 | s.ReadByte() << 8 | s.ReadByte(); 13 | } 14 | 15 | public static int ReadInt32(byte[] buff, int offset=0) 16 | { 17 | return buff[offset] << 3 * 8 | buff[offset + 1] << 2 * 8 | buff[offset+2] << 8 | buff[offset+3]; 18 | } 19 | 20 | 21 | public static short ReadInt16(Stream s) 22 | { 23 | if (s.CanSeek && s.Position + 2 > s.Length) 24 | throw new Exception(string.Format("ReadInt16 needs 2 bytes but got ony {0}", s.Length - s.Position)); 25 | return (short)((s.ReadByte() << 8) | s.ReadByte()); 26 | } 27 | 28 | public static long ReadInt64(Stream stream) 29 | { 30 | if (stream.CanSeek && stream.Position + 8 > stream.Length) 31 | throw new Exception(string.Format("ReadInt64 needs 8 bytes but got ony {0}", stream.Length - stream.Position)); 32 | 33 | var res = 0L; 34 | for (int i = 0; i < 8; i++) 35 | res = res << 8 | stream.ReadByte(); 36 | return res; 37 | } 38 | 39 | public static void Write(Stream stream, long i) 40 | { 41 | ulong ui = (ulong)i; 42 | for (int j = 7; j >= 0; j--) 43 | stream.WriteByte((byte)(ui >> j * 8 & 0xff)); 44 | } 45 | 46 | public static void Write(Stream stream, int i) 47 | { 48 | WriteByte(stream, i >> 8 * 3); 49 | WriteByte(stream, i >> 8 * 2); 50 | WriteByte(stream, i >> 8); 51 | WriteByte(stream, i); 52 | } 53 | 54 | public static void Write(Stream stream, uint i) 55 | { 56 | Write(stream, (int)i); 57 | } 58 | 59 | public static void Write(Stream stream, short i) 60 | { 61 | WriteByte(stream, i >> 8); 62 | WriteByte(stream, i); 63 | } 64 | 65 | public static void WriteByte(Stream stream, int i) 66 | { 67 | stream.WriteByte((byte)(i & 0xff)); 68 | } 69 | 70 | public static void Write(byte[] buff, int i) 71 | { 72 | buff[0] = (byte)(i >> 8 * 3); 73 | buff[1] = (byte)((i & 0xff0000) >> 8 * 2); 74 | buff[2] = (byte)((i & 0xff00) >> 8); 75 | buff[3] = (byte)(i & 0xff); 76 | } 77 | 78 | public static int ToInt32(byte[] buff) 79 | { 80 | return (buff[0] << 8 * 3) | (buff[1] << 8 * 2) | (buff[2] << 8) | buff[3]; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/Utils/CountObservable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reactive.Linq; 3 | using System.Reactive.Subjects; 4 | using System.Threading; 5 | 6 | namespace kafka4net.Utils 7 | { 8 | public sealed class CountObservable : IObservable 9 | { 10 | readonly ISubject _subj; 11 | readonly IConnectableObservable _publishedCounter; 12 | int _count; 13 | 14 | public CountObservable() 15 | { 16 | _subj = new Subject(); 17 | _publishedCounter = _subj.Replay(1); 18 | _publishedCounter.Connect(); 19 | _subj.OnNext(0); 20 | } 21 | 22 | public void Incr() 23 | { 24 | _subj.OnNext(Interlocked.Increment(ref _count)); 25 | } 26 | 27 | public void Decr() 28 | { 29 | _subj.OnNext(Interlocked.Decrement(ref _count)); 30 | } 31 | 32 | public IDisposable Subscribe(IObserver observer) 33 | { 34 | return _publishedCounter.Subscribe(observer); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/Utils/Crc32.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net.Utils 4 | { 5 | /// Simplfied version of http://damieng.com/blog/2006/08/08/calculating_crc32_in_c_and_net 6 | static class Crc32 7 | { 8 | const UInt32 _polynomial = 0xEDB88320u; 9 | static readonly UInt32[] _table = InitializeTable(); 10 | 11 | public static UInt32 Update(byte[] buffer, UInt32 state = ~0U, int len = -1, int offset = 0) 12 | { 13 | if (len == -1) 14 | len = buffer.Length; 15 | 16 | for (int i = offset; i < offset + len; i++) 17 | state = (state >> 8) ^ _table[buffer[i] ^ state & 0xff]; 18 | 19 | return state; 20 | } 21 | 22 | public static UInt32 Update(ArraySegment buffer, UInt32 state = ~0U) 23 | { 24 | var array = buffer.Array; 25 | var end = buffer.Offset + buffer.Count; 26 | 27 | for (int i = buffer.Offset; i < end; i++) 28 | state = (state >> 8) ^ _table[array[i] ^ state & 0xff]; 29 | 30 | return state; 31 | } 32 | 33 | public static UInt32 Update(int i, UInt32 state) 34 | { 35 | state = (state >> 8) ^ _table[(i >> 8 * 3 & 0xff) ^ state & 0xff]; 36 | state = (state >> 8) ^ _table[(i >> 8 * 2 & 0xff) ^ state & 0xff]; 37 | state = (state >> 8) ^ _table[(i >> 8 & 0xff) ^ state & 0xff]; 38 | state = (state >> 8) ^ _table[(i & 0xff) ^ state & 0xff]; 39 | return state; 40 | } 41 | 42 | public static UInt32 Update(byte b, UInt32 state = ~0U) 43 | { 44 | state = (state >> 8) ^ _table[b ^ state & 0xff]; 45 | return state; 46 | } 47 | 48 | public static UInt32 GetHash(UInt32 state) 49 | { 50 | return ~state; 51 | } 52 | 53 | private static UInt32[] InitializeTable() 54 | { 55 | var createTable = new UInt32[256]; 56 | for (var i = 0; i < 256; i++) 57 | { 58 | var entry = (UInt32)i; 59 | for (var j = 0; j < 8; j++) 60 | if ((entry & 1) == 1) 61 | entry = (entry >> 1) ^ _polynomial; 62 | else 63 | entry = entry >> 1; 64 | createTable[i] = entry; 65 | } 66 | 67 | return createTable; 68 | } 69 | 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/Utils/DateTimeExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace kafka4net.Utils 4 | { 5 | public static class DateTimeExtensions 6 | { 7 | 8 | /// 9 | /// Unix epoch time from a datetime 10 | /// 11 | /// 12 | /// 13 | public static int DateTimeToEpochSeconds(this DateTimeOffset date) 14 | { 15 | var t = (date - new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero)); 16 | return (int)t.TotalSeconds; 17 | } 18 | 19 | /// 20 | /// Datetime from an int representing a unix epoch time 21 | /// 22 | /// 23 | /// 24 | public static DateTimeOffset EpochSecondsToDateTime(this int secondsSinceEpoch) 25 | { 26 | var date = new DateTimeOffset(1970, 1, 1,0,0,0,TimeSpan.Zero); 27 | return date.AddSeconds(secondsSinceEpoch); 28 | } 29 | 30 | /// 31 | /// Unix epoch time from a datetime 32 | /// 33 | /// 34 | /// 35 | public static long DateTimeToEpochMilliseconds(this DateTimeOffset date) 36 | { 37 | var t = (date - new DateTimeOffset(1970, 1, 1,0,0,0,TimeSpan.Zero)); 38 | return (long)t.TotalMilliseconds; 39 | } 40 | 41 | /// 42 | /// Datetime from an int representing a unix epoch time 43 | /// 44 | /// 45 | /// 46 | public static DateTimeOffset EpochMillisecondsToDateTime(this long msSinceEpoch) 47 | { 48 | var date = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero); 49 | return date.AddMilliseconds(msSinceEpoch); 50 | } 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/Utils/EnumerableEx.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | 5 | namespace kafka4net.Utils 6 | { 7 | static class EnumerableEx 8 | { 9 | [DebuggerStepThrough] 10 | public static void ForEach(this IEnumerable items, Action action) 11 | { 12 | foreach (T item in items) 13 | action(item); 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Utils/LittleEndianConverter.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace kafka4net.Utils 3 | { 4 | internal static class LittleEndianConverter 5 | { 6 | public static uint ReadUInt32(byte[] buff, int offset) 7 | { 8 | return (uint)(buff[offset+3] << 3 * 8 | buff[offset+2] << 2 * 8 | buff[offset+1] << 8 | buff[offset]); 9 | } 10 | 11 | public static void Write(uint i, byte[] buff, int offset) 12 | { 13 | buff[offset] = (byte)(i & 0xff); 14 | buff[offset + 1] = (byte)(i >> 8 & 0xff); 15 | buff[offset + 2] = (byte)(i >> 8*2 & 0xff); 16 | buff[offset + 3] = (byte)(i >> 8*3 & 0xff); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Utils/RxSyncContextFromScheduler.cs: -------------------------------------------------------------------------------- 1 | using System.Reactive.Concurrency; 2 | using System.Threading; 3 | 4 | namespace kafka4net.Utils 5 | { 6 | /// 7 | /// Implements SynchronizationContext which routes "async" keyword callbacks to Rx Scheduler 8 | /// 9 | class RxSyncContextFromScheduler : SynchronizationContext 10 | { 11 | private readonly IScheduler _scheduler; 12 | 13 | public RxSyncContextFromScheduler(IScheduler scheduler) 14 | { 15 | _scheduler = scheduler; 16 | } 17 | 18 | public override void Post(SendOrPostCallback d, object state) 19 | { 20 | _scheduler.Schedule(() => d(state)); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Utils/TaskEx.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace kafka4net.Utils 8 | { 9 | static class TaskEx 10 | { 11 | /// 12 | /// Not very performant, but good enough to use in shutdown routine. 13 | /// http://blogs.msdn.com/b/pfxteam/archive/2011/11/10/10235834.aspx 14 | /// 15 | public async static Task TimeoutAfter(this Task task, TimeSpan timeout) 16 | { 17 | var delay = Task.Delay(timeout); 18 | var res = await Task.WhenAny(task, delay).ConfigureAwait(false); 19 | return res == task; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Utils/WatchdogScheduler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.Reactive.Concurrency; 4 | using System.Reactive.Linq; 5 | 6 | namespace kafka4net.Utils 7 | { 8 | /// 9 | /// Wrapper around scheduler, which check execution duration every second and expose the duration as DelaySampler observable. 10 | /// The caller can use DelaySampler to set treshold and take an action if it is suspected that working thread is hang. 11 | /// Also it capture the last executed stack, which help finding offending code. 12 | /// 13 | class WatchdogScheduler : LocalScheduler, ISchedulerPeriodic, IDisposable 14 | { 15 | readonly IScheduler _scheduler; 16 | DateTime _actionStart = DateTime.MaxValue; 17 | public readonly IObservable DelaySampler; 18 | public StackTrace LastStack; 19 | bool _disposed; 20 | private static readonly ILogger _log = Logger.GetLogger(); 21 | 22 | 23 | public WatchdogScheduler(IScheduler scheduler) 24 | { 25 | _scheduler = scheduler; 26 | 27 | DelaySampler = Observable.Interval(TimeSpan.FromSeconds(1)). 28 | Select(_ => _actionStart). 29 | Where(_ => _ != DateTime.MaxValue). 30 | Select(_ => DateTime.Now - _); 31 | } 32 | 33 | public override IDisposable Schedule(TState state, TimeSpan dueTime, Func action) 34 | { 35 | var stack = new StackTrace(false); 36 | Func actionWrapper = (scheduler, state1) => 37 | { 38 | _actionStart = DateTime.Now; 39 | LastStack = stack; 40 | try 41 | { 42 | return action(scheduler, state1); 43 | } 44 | finally 45 | { 46 | LastStack = null; 47 | _actionStart = DateTime.MaxValue; 48 | } 49 | }; 50 | 51 | return _scheduler.Schedule(state, dueTime, actionWrapper); 52 | } 53 | 54 | public IDisposable SchedulePeriodic(TState state, TimeSpan period, Func action) 55 | { 56 | var stack = new StackTrace(false); 57 | Func actionWrapper = (s1) => 58 | { 59 | _actionStart = DateTime.Now; 60 | LastStack = stack; 61 | try 62 | { 63 | return action(s1); 64 | } 65 | finally 66 | { 67 | LastStack = null; 68 | _actionStart = DateTime.MaxValue; 69 | } 70 | }; 71 | 72 | return _scheduler.SchedulePeriodic(state, period, actionWrapper); 73 | } 74 | 75 | public void Dispose() 76 | { 77 | if(_disposed) 78 | return; 79 | _disposed = true; 80 | 81 | (_scheduler as IDisposable)?.Dispose(); 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /src/WorkingThreadHungException.cs: -------------------------------------------------------------------------------- 1 | namespace kafka4net 2 | { 3 | public class WorkingThreadHungException : BrokerException 4 | { 5 | public WorkingThreadHungException(string message) : base(message) {} 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/kafka4net.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D} 8 | Library 9 | Properties 10 | kafka4net 11 | kafka4net 12 | v4.5 13 | 512 14 | 15 | 16 | 17 | 18 | 19 | true 20 | full 21 | false 22 | bin\Debug\ 23 | DEBUG;TRACE 24 | prompt 25 | 4 26 | false 27 | 28 | 29 | pdbonly 30 | true 31 | bin\Release\ 32 | TRACE 33 | prompt 34 | 4 35 | 36 | 37 | 38 | ..\packages\Crc32C.NET.1.0.5.0\lib\net20\Crc32C.NET.dll 39 | True 40 | 41 | 42 | ..\packages\lz4net.1.0.10.93\lib\net4-client\LZ4.dll 43 | True 44 | 45 | 46 | ..\packages\Snappy.NET.1.1.1.8\lib\net45\Snappy.NET.dll 47 | True 48 | 49 | 50 | 51 | 52 | 53 | ..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll 54 | 55 | 56 | ..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll 57 | 58 | 59 | ..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll 60 | 61 | 62 | ..\packages\Rx-PlatformServices.2.2.5\lib\net45\System.Reactive.PlatformServices.dll 63 | 64 | 65 | False 66 | ..\packages\Microsoft.Tpl.Dataflow.4.5.24\lib\portable-net45+win8+wpa81\System.Threading.Tasks.Dataflow.dll 67 | 68 | 69 | ..\packages\xxHashSharp.1.0.0\lib\net45\xxHashSharp.dll 70 | True 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | Designer 139 | 140 | 141 | 142 | 143 | true 144 | 145 | 146 | 147 | 148 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 149 | 150 | 151 | 152 | 153 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /src/kafka4net.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | kafka4net 5 | $version$ 6 | NTent 7 | http://www.apache.org/licenses/ 8 | https://github.com/ntent-ad/kafka4net 9 | false 10 | kafka4net 11 | Event-driven async Kafka driver for .NET 12 | 13 | Kafka driver for .NET 14 | 15 | Event-driven architecture, all asynchronous. 16 | Automatic following changes of Leader Partition in case of broker failure. 17 | Integration tests are part of the codebase. Use Vagrant to provision 1 zookeeper and 3 kafka virtual servers. 18 | Use RxExtensions library to expose API and for internal implementation. 19 | Support compression (gzip, lz4, snappy). 20 | 21 | Bug fix release 22 | Copyright NTent 2015 23 | kafka kafka4net 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /tests/CompressionTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using kafka4net.Compression; 5 | using NUnit.Framework; 6 | 7 | namespace tests 8 | { 9 | [TestFixture] 10 | public class CompressionTests 11 | { 12 | [Test] 13 | [Category("Compression")] 14 | public void TestLz4Stream() 15 | { 16 | var buffers = Enumerable.Range(1, 256 * 1024). 17 | AsParallel(). 18 | Select(size => { 19 | var buff = new byte[size]; 20 | new Random().NextBytes(buff); 21 | return buff; 22 | }); 23 | 24 | buffers. 25 | ForAll(buffer => 26 | { 27 | var compressed = new MemoryStream(); 28 | var lz = new Lz4KafkaStream(compressed, CompressionStreamMode.Compress); 29 | lz.Write(buffer, 0, buffer.Length); 30 | lz.Close(); 31 | var compressedBuff = compressed.ToArray(); 32 | 33 | var lz2 = new Lz4KafkaStream(new MemoryStream(compressedBuff), CompressionStreamMode.Decompress); 34 | var uncompressed = new MemoryStream(); 35 | lz2.CopyTo(uncompressed); 36 | var uncompressedBuffer = uncompressed.ToArray(); 37 | 38 | var res = buffer.SequenceEqual(uncompressedBuffer); 39 | Assert.IsTrue(res, $"Buffer size {buffer.Length}"); 40 | }); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("tests")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("NTent")] 12 | [assembly: AssemblyProduct("tests")] 13 | [assembly: AssemblyCopyright("Copyright © NTent 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("d9726911-6f07-4eb3-a862-48ba69c23a3a")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("0.1.0.0")] 36 | [assembly: AssemblyFileVersion("0.1.0.0")] 37 | -------------------------------------------------------------------------------- /tests/VagrantBrokerUtil.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Net.Sockets; 7 | using System.Text; 8 | using System.Threading; 9 | using kafka4net; 10 | using kafka4net.Utils; 11 | 12 | namespace tests 13 | { 14 | internal static class VagrantBrokerUtil 15 | { 16 | static readonly NLog.Logger _log = NLog.LogManager.GetCurrentClassLogger(); 17 | 18 | public static Dictionary BrokerIpToName = new Dictionary 19 | { 20 | {"192.168.56.10","broker1"}, 21 | {"192.168.56.20","broker2"}, 22 | {"192.168.56.30","broker3"}, 23 | }; 24 | 25 | static readonly string _kafkaVersion = File.ReadLines( 26 | Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..\\..\\..\\vagrant\\scripts\\kafka_version.txt") 27 | ).First(); 28 | 29 | public static string GetBrokerNameFromIp(string ip) 30 | { 31 | if (!BrokerIpToName.ContainsKey(ip)) 32 | throw new Exception("Unknown ip: " + ip); 33 | return BrokerIpToName[ip]; 34 | } 35 | 36 | 37 | public static void CreateTopic(string topicName, int numPartitions, int replicationFactor) 38 | { 39 | _log.Info("Creating topic: '{0}'", topicName); 40 | string createTopicScript = "ssh -c '/opt/kafka_2.10-"+_kafkaVersion+"/bin/kafka-topics.sh --create --topic {0} --partitions {1} --replication-factor {2} --zookeeper 192.168.56.2' broker1"; 41 | Vagrant(string.Format(createTopicScript, topicName, numPartitions, replicationFactor)); 42 | _log.Info("Created topic: '{0}'", topicName); 43 | } 44 | 45 | public static void RebalanceLeadership() 46 | { 47 | _log.Info("Rebalancing Leadership"); 48 | string rebalanceScript = "ssh -c '/opt/kafka_2.10-"+_kafkaVersion+"/bin/kafka-preferred-replica-election.sh --zookeeper 192.168.56.2' broker1"; 49 | Vagrant(rebalanceScript); 50 | _log.Info("Rebalanced Leadership"); 51 | } 52 | 53 | public static void DescribeTopic(string topic) 54 | { 55 | _log.Info("Getting topic description"); 56 | string script = "ssh -c '/opt/kafka_2.10-" + _kafkaVersion + "/bin/kafka-topics.sh --zookeeper 192.168.56.2 --describe --topic " + topic + "' broker1"; 57 | Vagrant(script); 58 | _log.Info("Got topic description"); 59 | } 60 | 61 | public static void ReassignPartitions(Cluster cluster, string topic, int partition) 62 | { 63 | var brokerMeta = cluster.FindBrokerMetaForPartitionId(topic, partition); 64 | var brokerToMoveTo = brokerMeta.NodeId == 1 ? 2 : 1; 65 | 66 | var partitionsJson = string.Format("{{\"partitions\":[{{\"topic\":\"{0}\",\"partition\":{1},\"replicas\":[{2}]}}], \"version\":1}}", 67 | topic, partition, brokerToMoveTo); 68 | 69 | _log.Info(string.Format("Reassigning Partitions (topic {0}, partition {1}, from node {2} to node {3})", topic, partition, brokerMeta.NodeId, brokerToMoveTo)); 70 | 71 | var generateJson = "ssh -c \"printf '" + partitionsJson.Replace("\"", @"\\\""") + "' >partitions-to-move.json\" broker1"; 72 | Vagrant(generateJson); 73 | 74 | var reassignScript = "ssh -c '/opt/kafka_2.10-" + _kafkaVersion + "/bin/kafka-reassign-partitions.sh --zookeeper 192.168.56.2 --reassignment-json-file partitions-to-move.json --execute' broker1"; 75 | Vagrant(reassignScript); 76 | 77 | _log.Info("Reassigned Partitions"); 78 | } 79 | 80 | public static void RestartBrokers() 81 | { 82 | BrokerIpToName.Where(np=>!IsBrokerResponding(np.Key)).ForEach(np=>StartBroker(np.Value)); 83 | } 84 | 85 | public static void StopBroker(string broker) 86 | { 87 | _log.Info("Stopping broker {0}", broker); 88 | Vagrant("ssh -c 'sudo service kafka stop' " + broker); 89 | _log.Info("Stopped broker {0}", broker); 90 | } 91 | 92 | public static string StopBrokerLeaderForPartition(Cluster cluster, string topic, int partition) 93 | { 94 | var brokerMeta = cluster.FindBrokerMetaForPartitionId(topic, partition); 95 | var brokerName = GetBrokerNameFromIp(brokerMeta.Host); 96 | StopBroker(brokerName); 97 | return brokerName; 98 | } 99 | 100 | public static void StartBroker(string broker) 101 | { 102 | var brokerIp = BrokerIpToName.Single(b => b.Value == broker).Key; 103 | if (IsBrokerResponding(brokerIp)) 104 | { 105 | _log.Info("Broker already running: '{0}'", broker); 106 | return; 107 | } 108 | 109 | _log.Info("Starting broker: '{0}'", broker); 110 | Vagrant("ssh -c 'sudo service kafka start' " + broker); 111 | 112 | // await for tcp to become accessible, because process start does not mean that server has done initializing and start listening 113 | while(!IsBrokerResponding(brokerIp)) 114 | Thread.Sleep(200); 115 | 116 | _log.Info("Started broker: '{0}'", broker); 117 | } 118 | 119 | private static bool IsBrokerResponding(string ip) 120 | { 121 | try 122 | { 123 | var client = new TcpClient(); 124 | return client.ConnectAsync(ip, 9092).Wait(TimeSpan.FromSeconds(2)); 125 | } 126 | catch 127 | { 128 | return false; 129 | } 130 | } 131 | 132 | private static string Vagrant(string script) 133 | { 134 | // TODO: implement vagrant-control 135 | var dir = AppDomain.CurrentDomain.BaseDirectory; 136 | dir = Path.Combine(dir, @"..\..\..\vagrant"); 137 | dir = Path.GetFullPath(dir); 138 | var output = new StringBuilder(); 139 | var pi = new ProcessStartInfo(@"vagrant.exe", script) 140 | { 141 | CreateNoWindow = true, 142 | RedirectStandardError = true, 143 | RedirectStandardOutput = true, 144 | UseShellExecute = false, 145 | WorkingDirectory = dir 146 | }; 147 | 148 | var p = Process.Start(pi); 149 | p.OutputDataReceived += (sender, args) => 150 | { 151 | output.Append(args.Data); 152 | _log.Info(args.Data); 153 | } ; 154 | p.ErrorDataReceived += (sender, args) => 155 | { 156 | output.Append(args.Data); 157 | _log.Info(args.Data); 158 | }; 159 | p.BeginOutputReadLine(); 160 | p.BeginErrorReadLine(); 161 | p.WaitForExit(); 162 | 163 | if(p.ExitCode != 0) 164 | throw new Exception($"Vagrant failed with exit code {p.ExitCode}. Output: {output}"); 165 | 166 | return output.ToString(); 167 | } 168 | 169 | public static string GenerateMessagesWithJava(string codec, string topic) 170 | { 171 | return Vagrant($"ssh broker1 -c 'java -jar /vagrant/files/binary-console-all.jar {topic} {codec} produce'"); 172 | } 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /tests/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC} 7 | Library 8 | Properties 9 | tests 10 | kafka4net-tests 11 | v4.5 12 | 512 13 | {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 14 | 10.0 15 | $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) 16 | $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages 17 | False 18 | UnitTest 19 | 20 | 21 | true 22 | full 23 | false 24 | bin\Debug\ 25 | DEBUG;TRACE 26 | prompt 27 | 4 28 | 29 | 30 | pdbonly 31 | true 32 | bin\Release\ 33 | TRACE 34 | prompt 35 | 4 36 | 37 | 38 | 39 | ..\packages\log4net.2.0.5\lib\net45-full\log4net.dll 40 | True 41 | 42 | 43 | ..\packages\NLog.4.4.0\lib\net45\NLog.dll 44 | True 45 | 46 | 47 | ..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll 48 | True 49 | 50 | 51 | 52 | ..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll 53 | 54 | 55 | ..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll 56 | 57 | 58 | ..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll 59 | 60 | 61 | ..\packages\Rx-PlatformServices.2.2.5\lib\net45\System.Reactive.PlatformServices.dll 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | {e3a39e66-db86-4e9c-b967-3aa71fa47c5d} 88 | kafka4net 89 | 90 | 91 | 92 | 93 | 94 | 95 | False 96 | 97 | 98 | False 99 | 100 | 101 | False 102 | 103 | 104 | False 105 | 106 | 107 | 108 | 109 | 110 | 111 | 118 | -------------------------------------------------------------------------------- /tools/binary-console/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'scala' 3 | id 'java' 4 | id 'application' 5 | id 'com.github.johnrengelman.shadow' version '1.2.3' 6 | } 7 | /*apply plugin: 'scala' 8 | apply plugin: 'application'*/ 9 | 10 | mainClassName = 'com.ntent.kafka.main' 11 | 12 | repositories{mavenCentral()} 13 | 14 | dependencies { 15 | compile "org.scala-lang:scala-library:2.10.6" 16 | compile 'org.apache.kafka:kafka_2.10:0.8.2.2' 17 | } 18 | 19 | -------------------------------------------------------------------------------- /tools/binary-console/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /tools/binary-console/src/main/scala/com/ntent/kafka/main.scala: -------------------------------------------------------------------------------- 1 | package com.ntent.kafka 2 | 3 | import java.io._ 4 | import java.security.MessageDigest 5 | import java.util.Properties 6 | import java.util.concurrent.atomic.AtomicInteger 7 | import java.util.concurrent.{Callable, Executors} 8 | 9 | import kafka.consumer.{Consumer, ConsumerConfig} 10 | import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord} 11 | 12 | import scala.io.Source 13 | import scala.util.Random 14 | 15 | /** 16 | * Created by vchekan on 4/29/2016. 17 | */ 18 | object main extends App { 19 | val topic = args(0) 20 | val compression = args(1) 21 | val action = args(2) 22 | val md5 = MessageDigest.getInstance("MD5") 23 | 24 | action match { 25 | case "produce" => produce() 26 | case "consume" => consume() 27 | case _ => throw new RuntimeException("Unknown action. Should be 'produce' or 'consume'") 28 | } 29 | 30 | def produce() = { 31 | val hashFileName = "/vagrant/files/hashes.txt" 32 | val sizesFileName = "/vagrant/files/sizes.txt" 33 | 34 | val hashFile = new PrintWriter(new File(hashFileName)) 35 | val sizes = readSizes(sizesFileName) 36 | 37 | val producer = new KafkaProducer[Array[Byte], Array[Byte]](producerProps) 38 | println(s"Read ${sizes.length} lines") 39 | val futures = sizes.map(size => { 40 | producer.send(generateMessage(size, hashFile)) 41 | }) 42 | 43 | producer.close() 44 | 45 | futures.foreach(f => { 46 | val res = f.get() 47 | println(s"Sent part:${res.partition()} offset: ${res.offset()}") 48 | }) 49 | 50 | 51 | hashFile.close() 52 | } 53 | 54 | def consume(): Unit = { 55 | val hashFileName = "C:\\projects\\kafka4net\\vagrant\\files\\hashes.txt" 56 | val sizesFileName = "C:\\projects\\kafka4net\\vagrant\\files\\sizes.txt" 57 | 58 | val hashFile = new PrintWriter(new File(hashFileName)) 59 | val consumer = Consumer.create(new ConsumerConfig(consumerProps)) 60 | val consumerMap = consumer.createMessageStreams(Map(topic -> 3)) 61 | val streams = consumerMap.get(topic).get 62 | val expectedCount = readSizes(sizesFileName).length 63 | val count = new AtomicInteger() 64 | 65 | val threads = for(stream <- streams) yield { 66 | new Thread(new Runnable { 67 | override def run(): Unit = { 68 | try { 69 | println(s"Starting stream ${stream}") 70 | val it = stream.iterator() 71 | while (it.hasNext()) { 72 | try { 73 | val msg = it.next() 74 | val value = msg.message() 75 | count.incrementAndGet() 76 | //println(s"Got ${count.get()}/$expectedCount size: ${value.length} part: ${msg.partition} offset: ${msg.offset}") 77 | val hash = md5.digest(value).map("%02X".format(_)).mkString 78 | hashFile.println(hash) 79 | if (count.get() == expectedCount) { 80 | consumer.shutdown() 81 | } 82 | } catch { 83 | case e: Throwable => println(e.getMessage) 84 | } 85 | } 86 | } catch { 87 | case e: Throwable => println(s"Error: ${e.getMessage}") 88 | } 89 | } 90 | }) 91 | } 92 | 93 | threads.foreach(_.start()) 94 | 95 | threads.foreach(_.join(5*60*1000)) 96 | 97 | hashFile.close() 98 | } 99 | 100 | def generateMessage(size: Int, hashFile: PrintWriter): ProducerRecord[Array[Byte], Array[Byte]] = { 101 | val buff = new Array[Byte](size) 102 | Random.nextBytes(buff) 103 | val hash = md5.digest(buff).map("%02X".format(_)).mkString 104 | hashFile.println(hash) 105 | 106 | // put all messages into the same partition 107 | val key = Array[Byte](0) 108 | println(hash) 109 | new ProducerRecord(topic, key, buff) 110 | } 111 | 112 | def producerProps = { 113 | val props = new Properties() 114 | props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.56.10:9092") 115 | props.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, compression) 116 | props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "10000") 117 | props.setProperty(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG, "true") 118 | props.setProperty(ProducerConfig.BUFFER_MEMORY_CONFIG, (11*1024*1024).toString) // 11Mb 119 | props.setProperty(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, (8*1000*1000).toString) 120 | props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, (1024*1024).toString) 121 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer") 122 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer") 123 | props 124 | } 125 | 126 | def consumerProps: Properties = { 127 | val props = new Properties() 128 | 129 | //props.put("bootstrap.servers", "192.168.56.10:9092") 130 | props.setProperty("zookeeper.connect", "192.168.56.2:2181") 131 | props.setProperty("group.id", "kafka4net-testing") 132 | //props.setProperty("auto.offset.reset", "largest") 133 | props.setProperty("fetch.message.max.bytes", (500*1024*1024).toString) 134 | props.put("auto.commit.enable", "false") 135 | props 136 | } 137 | 138 | def readSizes(sizesFileName: String) = { 139 | Source.fromFile(sizesFileName).getLines().map(Integer.parseInt(_)).toArray 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /vagrant/.gitignore: -------------------------------------------------------------------------------- 1 | /.vagrant/ 2 | /rpm/ 3 | -------------------------------------------------------------------------------- /vagrant/.hgignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | .git 3 | .gitignore 4 | .vagrant 5 | files/ 6 | -------------------------------------------------------------------------------- /vagrant/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | VAGRANTFILE_API_VERSION = "2" 5 | 6 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 7 | k_count=3 8 | 9 | config.vm.box = "ubuntu/trusty64" 10 | config.vm.box_check_update = false 11 | 12 | # zookeeper 13 | config.vm.define "zookeeper1" do |s| 14 | s.vm.hostname = "zookeeper1" 15 | 16 | s.vm.network :private_network, ip: "192.168.56.2" 17 | # zookeeper1 is cache for packages 18 | s.vm.provision "shell", inline: "apt-get update" 19 | s.vm.provision "shell", inline: "apt-get install -y squid-deb-proxy avahi-utils" 20 | s.vm.provision "shell", inline: "apt-get install -y squid-deb-proxy-client" 21 | s.vm.provision "shell", path: "scripts/init.sh" 22 | s.vm.provision "shell", path: "scripts/zookeeper.sh" 23 | end 24 | 25 | # configure brokers 26 | (1..k_count).each do |i| 27 | config.vm.define "broker#{i}" do |s| 28 | s.vm.hostname = "broker#{i}" 29 | #s.vm.network "private_network", ip: "10.30.3.#{4-i}0", netmask: "255.255.255.0", virtualbox__intnet: "servidors", drop_nat_interface_default_route: true 30 | s.vm.network :private_network, ip: "192.168.56.#{i}0" 31 | s.vm.provision "shell", inline: "apt-get update" 32 | s.vm.provision "shell", inline: "apt-get install -y squid-deb-proxy-client" 33 | s.vm.provision "shell", path: "scripts/init.sh" 34 | s.vm.provision "shell", path: "scripts/broker.sh", args:"#{i}" 35 | end 36 | end 37 | 38 | config.vm.provider "virtualbox" do |v| 39 | #v.gui = true 40 | v.customize ["modifyvm", :id, "--cpuexecutioncap", "50", "--memory", "3072"] 41 | end 42 | end 43 | 44 | -------------------------------------------------------------------------------- /vagrant/config/kafka.conf: -------------------------------------------------------------------------------- 1 | description "Apache Kafka server" 2 | author "Vadim Chekan " 3 | 4 | start on runlevel [2345] 5 | stop on runlevel [016] 6 | kill signal SIGINT 7 | exec /bin/bash /opt/kafka_2.10-kafka_version/bin/kafka-server-start.sh /opt/kafka_2.10-kafka_version/config/server.properties > /tmp/kafka.log 2>&1 -------------------------------------------------------------------------------- /vagrant/config/zookeeper.conf: -------------------------------------------------------------------------------- 1 | description "Apache Zookeeper server" 2 | author "Vadim Chekan " 3 | 4 | start on runlevel [2345] 5 | stop on runlevel [016] 6 | kill signal SIGINT 7 | exec /bin/bash /opt/kafka_2.10-kafka_version/bin/zookeeper-server-start.sh /opt/kafka_2.10-kafka_version/config/zookeeper.properties > /tmp/zookeeper.log 2>&1 -------------------------------------------------------------------------------- /vagrant/config/zookeeper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # the directory where the snapshot is stored. 16 | dataDir=/tmp/zookeeper 17 | # the port at which the clients will connect 18 | clientPort=2181 19 | # clientPortAddress=192.168.56.2 20 | # disable the per-ip limit on the number of connections since this is a non-production config 21 | maxClientCnxns=0 22 | tickTime=2000 23 | initLimit=5 24 | syncLimit=2 25 | -------------------------------------------------------------------------------- /vagrant/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /vagrant/scripts/broker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | id=$1 4 | . /vagrant/scripts/env.sh 5 | 6 | sed -i -e "s/^broker\.id=.*/broker.id=$id/" "/opt/kafka_2.10-${kafka_version}/config/server.properties" 7 | sed -i -e "s|^#*listeners=.*|listeners=PLAINTEXT://192.168.56.${id}0:9092|" "/opt/kafka_2.10-${kafka_version}/config/server.properties" 8 | #sed -i -e "s/^log.segment.bytes=.*/log.segment.bytes=1000048576/" "/opt/kafka_2.10-${kafka_version}/config/server.properties" 9 | sed -i -e "s/^#*num\.io\.threads=.*/num.io.threads=2/" "/opt/kafka_2.10-${kafka_version}/config/server.properties" 10 | sed -i -e 's/^num\.partitions=.*/num\.partitions=6\ 11 | default.replication.factor=3/' "/opt/kafka_2.10-${kafka_version}/config/server.properties" 12 | sed -i -e "s/^#*zookeeper\.connect=.*/zookeeper.connect=192.168.56.2:2181/" \ 13 | "/opt/kafka_2.10-${kafka_version}/config/server.properties" 14 | sed -i -e 's/-Xmx1G/-Xmx1G/' "/opt/kafka_2.10-${kafka_version}/bin/kafka-server-start.sh" 15 | sed -i -e 's/-Xms1G/-Xms1G/' "/opt/kafka_2.10-${kafka_version}/bin/kafka-server-start.sh" 16 | 17 | echo 'message.max.bytes=50000000' >> "/opt/kafka_2.10-${kafka_version}/config/server.properties" 18 | echo 'replica.fetch.max.bytes=50000000' >> "/opt/kafka_2.10-${kafka_version}/config/server.properties" 19 | 20 | cp /vagrant/config/kafka.conf /etc/init/ 21 | chmod a-x /etc/init/kafka.conf 22 | sed -i -e "s/kafka_version/${kafka_version}/g" "/etc/init/kafka.conf" 23 | 24 | service kafka start 25 | -------------------------------------------------------------------------------- /vagrant/scripts/env.sh: -------------------------------------------------------------------------------- 1 | kafka_version=`cat /vagrant/scripts/kafka_version.txt` 2 | kafka="kafka_2.10-${kafka_version}" 3 | kafka_bin="$kafka.tgz" 4 | #kafka_url="http://www.eng.lsu.edu/mirrors/apache/kafka/${kafka_version}/${kafka_bin}" 5 | #kafka_url="https://people.apache.org/~junrao/kafka-${kafka_version}-candidate1/${kafka_bin}" 6 | kafka_url="http://apache.mirrors.lucidnetworks.net/kafka/${kafka_version}/${kafka_bin}" 7 | -------------------------------------------------------------------------------- /vagrant/scripts/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Executed as root 4 | 5 | . /vagrant/scripts/env.sh 6 | 7 | apt-get install -y openjdk-7-jre-headless 8 | 9 | if [ ! -d "/opt/$kafka" ]; then 10 | mkdir -p /vagrant/files 11 | cd /vagrant/files 12 | if [ ! -f "/vagrant/files/$kafka_bin" ]; then 13 | wget $kafka_url 14 | fi 15 | tar -C /opt -xzf $kafka_bin 16 | fi 17 | -------------------------------------------------------------------------------- /vagrant/scripts/kafka_version.txt: -------------------------------------------------------------------------------- 1 | 0.10.1.1 -------------------------------------------------------------------------------- /vagrant/scripts/zookeeper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /vagrant/scripts/env.sh 4 | 5 | mkdir -p /var/zookeeper 6 | mkdir -p /opt/kafka_2.10-${kafka_version}/logs 7 | sed -i -e 's/-Xmx512M/-Xmx1024M/' "/opt/kafka_2.10-${kafka_version}/bin/zookeeper-server-start.sh" 8 | sed -i -e 's/-Xms512M/-Xms1024M/' "/opt/kafka_2.10-${kafka_version}/bin/zookeeper-server-start.sh" 9 | 10 | cp /vagrant/config/zookeeper.properties "/opt/kafka_2.10-${kafka_version}/config/" 11 | chmod a-x "/opt/kafka_2.10-${kafka_version}/config/zookeeper.properties" 12 | 13 | cp /vagrant/config/zookeeper.conf /etc/init/ 14 | chmod a-x /etc/init/zookeeper.conf 15 | sed -i -e "s/kafka_version/${kafka_version}/g" "/etc/init/zookeeper.conf" 16 | 17 | service zookeeper start 18 | 19 | -------------------------------------------------------------------------------- /vagrant/server.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # see kafka.server.KafkaConfig for additional details and defaults 16 | 17 | ############################# Server Basics ############################# 18 | 19 | # The id of the broker. This must be set to a unique integer for each broker. 20 | broker.id=0 21 | 22 | ############################# Socket Server Settings ############################# 23 | 24 | # The port the socket server listens on 25 | port=9092 26 | 27 | # Hostname the broker will bind to. If not set, the server will bind to all interfaces 28 | #host.name=localhost 29 | 30 | # Hostname the broker will advertise to producers and consumers. If not set, it uses the 31 | # value for "host.name" if configured. Otherwise, it will use the value returned from 32 | # java.net.InetAddress.getCanonicalHostName(). 33 | #advertised.host.name= 34 | 35 | # The port to publish to ZooKeeper for clients to use. If this is not set, 36 | # it will publish the same port that the broker binds to. 37 | #advertised.port= 38 | 39 | # The number of threads handling network requests 40 | num.network.threads=3 41 | 42 | # The number of threads doing disk I/O 43 | num.io.threads=8 44 | 45 | # The send buffer (SO_SNDBUF) used by the socket server 46 | socket.send.buffer.bytes=102400 47 | 48 | # The receive buffer (SO_RCVBUF) used by the socket server 49 | socket.receive.buffer.bytes=102400 50 | 51 | # The maximum size of a request that the socket server will accept (protection against OOM) 52 | socket.request.max.bytes=104857600 53 | 54 | 55 | ############################# Log Basics ############################# 56 | 57 | # A comma seperated list of directories under which to store log files 58 | log.dirs=/tmp/kafka-logs 59 | 60 | # The default number of log partitions per topic. More partitions allow greater 61 | # parallelism for consumption, but this will also result in more files across 62 | # the brokers. 63 | num.partitions=1 64 | 65 | # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. 66 | # This value is recommended to be increased for installations with data dirs located in RAID array. 67 | num.recovery.threads.per.data.dir=1 68 | 69 | ############################# Log Flush Policy ############################# 70 | 71 | # Messages are immediately written to the filesystem but by default we only fsync() to sync 72 | # the OS cache lazily. The following configurations control the flush of data to disk. 73 | # There are a few important trade-offs here: 74 | # 1. Durability: Unflushed data may be lost if you are not using replication. 75 | # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. 76 | # 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. 77 | # The settings below allow one to configure the flush policy to flush data after a period of time or 78 | # every N messages (or both). This can be done globally and overridden on a per-topic basis. 79 | 80 | # The number of messages to accept before forcing a flush of data to disk 81 | #log.flush.interval.messages=10000 82 | 83 | # The maximum amount of time a message can sit in a log before we force a flush 84 | #log.flush.interval.ms=1000 85 | 86 | ############################# Log Retention Policy ############################# 87 | 88 | # The following configurations control the disposal of log segments. The policy can 89 | # be set to delete segments after a period of time, or after a given size has accumulated. 90 | # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens 91 | # from the end of the log. 92 | 93 | # The minimum age of a log file to be eligible for deletion 94 | log.retention.hours=168 95 | 96 | # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining 97 | # segments don't drop below log.retention.bytes. 98 | #log.retention.bytes=1073741824 99 | 100 | # The maximum size of a log segment file. When this size is reached a new log segment will be created. 101 | log.segment.bytes=1073741824 102 | 103 | # The interval at which log segments are checked to see if they can be deleted according 104 | # to the retention policies 105 | log.retention.check.interval.ms=300000 106 | 107 | # By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. 108 | # If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. 109 | log.cleaner.enable=false 110 | 111 | ############################# Zookeeper ############################# 112 | 113 | # Zookeeper connection string (see zookeeper docs for details). 114 | # This is a comma separated host:port pairs, each corresponding to a zk 115 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 116 | # You can also append an optional chroot string to the urls to specify the 117 | # root directory for all kafka znodes. 118 | zookeeper.connect=192.168.56.2:2181/kafka4nettest 119 | 120 | # Timeout in ms for connecting to zookeeper 121 | zookeeper.connection.timeout.ms=6000 122 | 123 | 124 | message.max.bytes=50000000 125 | replica.fetch.max.bytes=50000000 126 | --------------------------------------------------------------------------------