├── .gitignore
├── .nuget
└── packages.config
├── LICENSE
├── README.md
├── doc
└── img
│ ├── perfView-collect.png
│ ├── perfView-dump.png
│ ├── perfView-events.png
│ └── perfView-params.png
├── errors-rework.txt
├── examples
├── ConsumerExample.cs
├── ProducerExample.cs
├── Program.cs
├── examples.csproj
└── packages.config
├── kafka4net.sln
├── log4j.properties
├── nuget
├── nuget.exe.config
└── publish.bat
├── src
├── BrokerException.cs
├── Cluster.cs
├── Compression
│ ├── KafkaSnappyStream.cs
│ ├── Lz4KafkaStream.cs
│ └── StreamUtils.cs
├── CompressionType.cs
├── ConnState.cs
├── Connection.cs
├── Consumer.cs
├── ConsumerConfiguration.cs
├── ConsumerImpl
│ ├── Fetcher.cs
│ ├── PartitionFetchState.cs
│ ├── PositionProviders.cs
│ ├── TopicPartition.cs
│ └── TopicPartitionOffsets.cs
├── ErrorCode.cs
├── FletcherHashedMessagePartitioner.cs
├── ILogger.cs
├── IMessagePartitioner.cs
├── Internal
│ ├── PartitionRecoveryMonitor.cs
│ └── PartitionStateChangeEvent.cs
├── Logger.cs
├── Message.cs
├── Metadata
│ ├── BrokerMeta.cs
│ ├── PartitionMeta.cs
│ ├── PartitionOffsetInfo.cs
│ └── TopicMeta.cs
├── PartitionFailedException.cs
├── Producer.cs
├── ProducerConfiguration.cs
├── Properties
│ └── AssemblyInfo.cs
├── Protocols
│ ├── CorrelationLoopException.cs
│ ├── Protocol.cs
│ ├── Requests
│ │ ├── FetchRequest.cs
│ │ ├── MessageData.cs
│ │ ├── MessageSetItem.cs
│ │ ├── OffsetRequest.cs
│ │ ├── PartitionData.cs
│ │ ├── ProduceRequest.cs
│ │ ├── TopicData.cs
│ │ └── TopicRequest.cs
│ ├── ResponseCorrelation.cs
│ ├── Responses
│ │ ├── FetchResponse.cs
│ │ ├── MetadataResponse.cs
│ │ ├── OffsetResponse.cs
│ │ └── ProducerResponse.cs
│ └── Serializer.cs
├── ReceivedMessage.cs
├── Tracing
│ └── EtwTrace.cs
├── Utils
│ ├── AskEx.cs
│ ├── BigEndianConverter.cs
│ ├── CircularBuffer.cs
│ ├── CountObservable.cs
│ ├── Crc32.cs
│ ├── DateTimeExtensions.cs
│ ├── EnumerableEx.cs
│ ├── LittleEndianConverter.cs
│ ├── RxSyncContextFromScheduler.cs
│ ├── TaskEx.cs
│ └── WatchdogScheduler.cs
├── WorkingThreadHungException.cs
├── kafka4net.csproj
├── kafka4net.nuspec
└── packages.config
├── tests
├── CompressionTests.cs
├── Properties
│ └── AssemblyInfo.cs
├── RecoveryTest.cs
├── VagrantBrokerUtil.cs
├── packages.config
└── tests.csproj
├── tools
└── binary-console
│ ├── build.gradle
│ └── src
│ └── main
│ ├── resources
│ └── log4j.properties
│ └── scala
│ └── com
│ └── ntent
│ └── kafka
│ └── main.scala
└── vagrant
├── .gitignore
├── .hgignore
├── Vagrantfile
├── config
├── kafka.conf
├── zookeeper.conf
└── zookeeper.properties
├── log4j.properties
├── scripts
├── broker.sh
├── env.sh
├── init.sh
├── kafka_version.txt
└── zookeeper.sh
└── server.properties
/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | #ignore thumbnails created by windows
3 | Thumbs.db
4 | #Ignore files build by Visual Studio
5 | *.obj
6 | *.exe
7 | *.pdb
8 | *.user
9 | *.aps
10 | *.pch
11 | *.vspscc
12 | *_i.c
13 | *_p.c
14 | *.ncb
15 | *.suo
16 | *.tlb
17 | *.tlh
18 | *.bak
19 | *.cache
20 | *.ilk
21 | *.log
22 | [Bb]in
23 | [Dd]ebug*/
24 | *.lib
25 | *.sbr
26 | obj/
27 | [Rr]elease*/
28 | _ReSharper*/
29 | [Tt]est[Rr]esult*
30 | log.txt
31 | packages
32 | vagrant/files
33 | /Build.proj
34 | /src/Properties/AssemblyInfoGenerated.cs
35 | nuget/*.nupkg
36 | .gradle
37 | .idea
38 | *.iml
39 | build
--------------------------------------------------------------------------------
/.nuget/packages.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](https://badge.fury.io/nu/kafka4net)
2 | kafka4net: kafka-0.8 client
3 | =========
4 |
5 | #### Install:
6 | ```Install-Package kafka4net```
7 |
8 | ##Features:
9 | * Event-driven architecture, all asynchronous
10 | * Automatic following changes of Leader Partition in case of broker failure
11 | * Connection sharing: one connection per kafka broker is used
12 | * Flow control: slow consumer will suspend fetching and prevent memory exhausting
13 | * Integration tests are part of the codebase. Use Vagrant to provision 1 zookeeper and 3 kafka virtual servers
14 | * Use RxExtensions library to expose API and for internal implementation
15 | * Support compression (gzip, lz4, snappy). Unit-tested to be interoperable with Java implementation
16 |
17 | ##Not implemented:
18 | * Offset Fetch/Commit API
19 | * Only protocol for 0.8 (aka v0) is implemented at the moment
20 |
21 | ## Documentation
22 | * [Design](https://github.com/ntent-ad/kafka4net/wiki/Design)
23 | * [Troubleshooting](https://github.com/ntent-ad/kafka4net/wiki/Troubleshooting)
24 |
25 | ## Usage
26 | ### Consumer
27 |
28 | ```C#
29 | using System;
30 | using System.Linq;
31 | using System.Text;
32 | using System.Text.RegularExpressions;
33 | using System.Threading.Tasks;
34 | using kafka4net;
35 | using kafka4net.ConsumerImpl;
36 |
37 | namespace examples
38 | {
39 | public static class ConsumerExample
40 | {
41 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092.
42 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094"
43 | readonly static string _connectionString = "kafka1, kafka2, kafka3";
44 | readonly static string _topic = "some.topic";
45 |
46 | /// Simplest consumer with termination example
47 | public static async Task TakeForOneMinute()
48 | {
49 | // If you want to consume all messages in the topic, use TopicPositionFactory.Start
50 | // TopicPositionFactory.End will start waiting for new messages starting from the moment of subscription
51 | var consumer = new Consumer(new ConsumerConfiguration(_connectionString, _topic, new StartPositionTopicEnd()));
52 |
53 | consumer.OnMessageArrived.Subscribe(msg => {
54 | // Perform your own deserialization here
55 | var text = Encoding.UTF8.GetString(msg.Value);
56 | Console.WriteLine($"Got message: '{text}' Partition: {msg.Partition} Offset: {msg.Offset} Lag: {msg.HighWaterMarkOffset - msg.Offset}");
57 | });
58 |
59 | // Connecting starts when subscribing to OnMessageArrived. If you need to know when connection is actually one, wait for IsConnected task completion
60 | await consumer.IsConnected;
61 | Console.WriteLine("Connected");
62 |
63 | // Consume for one minute
64 | await Task.Delay(TimeSpan.FromMinutes(1));
65 |
66 | await consumer.CloseAsync();
67 | Console.WriteLine("Closed");
68 | }
69 | }
70 | }
71 | ```
72 |
73 |
74 |
75 | ### Consumer from multiple partitions
76 |
77 | ```C#
78 | using System;
79 | using System.Linq;
80 | using System.Text;
81 | using System.Text.RegularExpressions;
82 | using System.Threading.Tasks;
83 | using kafka4net;
84 | using kafka4net.ConsumerImpl;
85 |
86 | namespace examples
87 | {
88 | public static class ConsumerExample
89 | {
90 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092.
91 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094"
92 | readonly static string _connectionString = "kafka1, kafka2, kafka3";
93 | readonly static string _topic = "some.topic";
94 |
95 | public static async Task SubscribeToMultipleTopics()
96 | {
97 | var rx = new Regex("some\\.topic\\..+");
98 | var cluster = new Cluster(_connectionString);
99 | await cluster.ConnectAsync();
100 | var allTopics = await cluster.GetAllTopicsAsync();
101 | var wantedTopics = allTopics.Where(topic => rx.IsMatch(topic)).ToArray();
102 |
103 | var consumers = wantedTopics.Select(topic => new Consumer(new ConsumerConfiguration(_connectionString, topic, new StartPositionTopicEnd()))).ToArray();
104 | consumers.AsParallel().ForAll(consumer => consumer.OnMessageArrived.Subscribe(msg => {
105 | var text = Encoding.UTF8.GetString(msg.Value);
106 | Console.WriteLine($"Got message '{text}' from topic '{msg.Topic}' partition {msg.Partition} offset {msg.Offset}");
107 | }));
108 |
109 | await Task.Delay(TimeSpan.FromMinutes(1));
110 |
111 | await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync()));
112 | }
113 |
114 | }
115 | }
116 | ```
117 |
118 | ### Producer
119 | ```C#
120 | using System;
121 | using System.Linq;
122 | using System.Text;
123 | using System.Threading.Tasks;
124 | using kafka4net;
125 |
126 | namespace examples
127 | {
128 | public static class ProducerExample
129 | {
130 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092.
131 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094"
132 | readonly static string _connectionString = "kafka1, kafka2, kafka3";
133 | readonly static string _topic = "some.topic";
134 |
135 | public async static Task Produce100RandomMessages()
136 | {
137 | var rnd = new Random();
138 | var randomNumbers = Enumerable.Range(1, 100).Select(_ => rnd.Next().ToString());
139 | var producer = new Producer(_connectionString, new ProducerConfiguration(_topic));
140 |
141 | // Technically not mandatory, but good idea to listen to possible errors and act accordingly to your application requirements
142 | producer.OnPermError += (exception, messages) => Console.WriteLine($"Failed to write {messages.Length} because of {exception.Message}");
143 |
144 | // When message is confirmed by kafka broker to be persisted, OnSuccess is called. Can be used if upstream requires acknowlegement for extra reliability.
145 | producer.OnSuccess += messages => Console.WriteLine($"Sent {messages.Length} messages");
146 |
147 | await producer.ConnectAsync();
148 |
149 | foreach(var str in randomNumbers)
150 | {
151 | // Message.Key is optional. If not set, then driver will partition messages at random. In this case, for sake of example, partition by 1st character of the string
152 | var key = BitConverter.GetBytes(str[0]);
153 | // Implement your own serialization here.
154 | var msg = new Message { Value = Encoding.UTF8.GetBytes(str), Key = key};
155 | producer.Send(msg);
156 | }
157 |
158 | // Await for every producer to complete. Note, that calling CloseAsync is safe: all bessages currently in buffers will be awaited until flushed.
159 | await producer.CloseAsync(TimeSpan.FromMinutes(1));
160 | }
161 | }
162 | }
163 | ```
--------------------------------------------------------------------------------
/doc/img/perfView-collect.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-collect.png
--------------------------------------------------------------------------------
/doc/img/perfView-dump.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-dump.png
--------------------------------------------------------------------------------
/doc/img/perfView-events.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-events.png
--------------------------------------------------------------------------------
/doc/img/perfView-params.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ntent/kafka4net/a2c96aa987ddf3aa73446b02bfc0c1635a1a09b2/doc/img/perfView-params.png
--------------------------------------------------------------------------------
/errors-rework.txt:
--------------------------------------------------------------------------------
1 | Initial idea of implementing error recovery was to have single point, in tcp connection object where all error recovery would be made.
2 | But it turned out too low level, because depending on what this connection is doing, we might want different strategy for recovery.
3 | When RecoveryMonitor tests either broker is available, we do not want any recovery and want fail fast instead.
4 | Another issue is that currently only fetcher and producer are well protected, whereas earlier stages, such as connection, offset resolution,
5 | metadata fetching are more fragile, or have to implement their own recovery, thus polluting the code.
6 |
7 | Requirements:
8 | Fast reaction: consider partition failed immediately after tcp error, and not after many retries.
9 | Remember to fail all request tasks which are waiting for responses.
10 | While waiting for recovery, pay attention to changes in metadata
11 | Work nicely with shutdown and drain logic
12 |
13 | List of failure stages:
14 | In seed broker resolution
15 | In offset resolution
16 | In Metadata fetching
17 | In recovery monitor
18 | In produce message
19 | In fetch message
--------------------------------------------------------------------------------
/examples/ConsumerExample.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 | using System.Reactive.Linq;
4 | using System.Reactive.Threading.Tasks;
5 | using System.Text;
6 | using System.Text.RegularExpressions;
7 | using System.Threading;
8 | using System.Threading.Tasks;
9 | using kafka4net;
10 | using kafka4net.ConsumerImpl;
11 | using log4net;
12 |
13 | namespace examples
14 | {
15 | public static class ConsumerExample
16 | {
17 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092.
18 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094"
19 | readonly static string _connectionString = "kafka1, kafka2, kafka3";
20 | readonly static string _topic = "some.topic";
21 | private static ILog _log = LogManager.GetLogger(typeof (ConsumerExample));
22 |
23 | /// Simplest consumer with termination example
24 | public static async Task TakeForOneMinute()
25 | {
26 | // If you want to consume all messages in the topic, use TopicPositionFactory.Start
27 | // TopicPositionFactory.End will start waiting for new messages starting from the moment of subscription
28 | var consumer = new Consumer(new ConsumerConfiguration(_connectionString, _topic, new StartPositionTopicEnd()));
29 |
30 | consumer.OnMessageArrived.Subscribe(msg => {
31 | // Perform your own deserialization here
32 | var text = Encoding.UTF8.GetString(msg.Value);
33 | Console.WriteLine($"Got message: '{text}' Partition: {msg.Partition} Offset: {msg.Offset} Lag: {msg.HighWaterMarkOffset - msg.Offset}");
34 | });
35 |
36 | // Connecting starts when subscribing to OnMessageArrived. If you need to know when connection is actually one, wait for IsConnected task completion
37 | await consumer.IsConnected;
38 | Console.WriteLine("Connected");
39 |
40 | // Consume for one minute
41 | await Task.Delay(TimeSpan.FromMinutes(1));
42 |
43 | await consumer.CloseAsync();
44 | Console.WriteLine("Closed");
45 | }
46 |
47 | public static async Task SubscribeToMultipleTopics()
48 | {
49 | var rx = new Regex("some\\.topic\\..+");
50 | var cluster = new Cluster(_connectionString);
51 | await cluster.ConnectAsync();
52 | var allTopics = await cluster.GetAllTopicsAsync();
53 | var wantedTopics = allTopics.Where(topic => rx.IsMatch(topic)).ToArray();
54 |
55 | var consumers = wantedTopics.Select(topic => new Consumer(new ConsumerConfiguration(_connectionString, topic, new StartPositionTopicEnd()))).ToArray();
56 | consumers.AsParallel().ForAll(consumer => consumer.OnMessageArrived.Subscribe(msg => {
57 | var text = Encoding.UTF8.GetString(msg.Value);
58 | Console.WriteLine($"Got message '{text}' from topic '{msg.Topic}' partition {msg.Partition} offset {msg.Offset}");
59 | }));
60 |
61 | await Task.Delay(TimeSpan.FromMinutes(1));
62 |
63 | await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync()));
64 | }
65 |
66 | public static async Task ConsumeMessages(string brokers, string topic, string file, CancellationToken cancel)
67 | {
68 | _log.Info("Conecting...");
69 | var consumer = new Consumer(new ConsumerConfiguration(brokers, topic, new StartPositionTopicEnd()));
70 |
71 | consumer.OnMessageArrived.Subscribe(msg => {
72 | // Perform your own deserialization here
73 | var text = Encoding.UTF8.GetString(msg.Value);
74 | Console.WriteLine($"Received: '{text}' Partition: {msg.Partition} Offset: {msg.Offset} Lag: {msg.HighWaterMarkOffset - msg.Offset}");
75 | },
76 | e=> { _log.Error("Failed in Consumer", e); },
77 | () => { _log.Info("Complete...");}, cancel);
78 |
79 | // Connecting starts when subscribing to OnMessageArrived. If you need to know when connection is actually one, wait for IsConnected task completion
80 | await consumer.IsConnected;
81 | _log.Info("Consuming...");
82 |
83 | await cancel.WhenCanceled();
84 |
85 | _log.Info("Closing...");
86 | await consumer.CloseAsync();
87 | Console.WriteLine("Completed...");
88 |
89 | }
90 |
91 | public static Task WhenCanceled(this CancellationToken cancellationToken)
92 | {
93 | var tcs = new TaskCompletionSource();
94 | cancellationToken.Register(s => ((TaskCompletionSource)s).SetResult(true), tcs);
95 | return tcs.Task;
96 | }
97 |
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/examples/ProducerExample.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 | using System.Linq;
4 | using System.Reactive.Linq;
5 | using System.Text;
6 | using System.Threading.Tasks;
7 | using kafka4net;
8 | using log4net;
9 | using System.Reactive.Threading.Tasks;
10 | using System.Threading;
11 |
12 | namespace examples
13 | {
14 | public static class ProducerExample
15 | {
16 | // Notice, unlike java driver, port is not mandatory and will resolve to default value 9092.
17 | // Explicit ports are allowed though, for example "kafka1:9092, kafka2:9093, kafka3:9094"
18 | private static readonly string _connectionString = "kafka1, kafka2, kafka3";
19 | private static readonly string _topic = "some.topic";
20 | private static ILog _log = LogManager.GetLogger(typeof (ProducerExample));
21 |
22 | public static async Task Produce100RandomMessages()
23 | {
24 | var rnd = new Random();
25 | var randomNumbers = Enumerable.Range(1, 100).Select(_ => rnd.Next().ToString());
26 | var producer = new Producer(_connectionString, new ProducerConfiguration(_topic));
27 |
28 | // Technically not mandatory, but good idea to listen to possible errors and act accordingly to your application requirements
29 | producer.OnPermError +=
30 | (exception, messages) =>
31 | Console.WriteLine($"Failed to write {messages.Length} because of {exception.Message}");
32 |
33 | // When message is confirmed by kafka broker to be persisted, OnSuccess is called. Can be used if upstream requires acknowlegement for extra reliability.
34 | producer.OnSuccess += messages => Console.WriteLine($"Sent {messages.Length} messages");
35 |
36 | await producer.ConnectAsync();
37 |
38 | foreach (var str in randomNumbers)
39 | {
40 | // Message.Key is optional. If not set, then driver will partition messages at random. In this case, for sake of example, partition by 1st character of the string
41 | var key = BitConverter.GetBytes(str[0]);
42 | // Implement your own serialization here.
43 | var msg = new Message {Value = Encoding.UTF8.GetBytes(str), Key = key};
44 | producer.Send(msg);
45 | }
46 |
47 | // Await for every producer to complete. Note, that calling CloseAsync is safe: all bessages currently in buffers will be awaited until flushed.
48 | await producer.CloseAsync(TimeSpan.FromMinutes(1));
49 | }
50 |
51 | public static async Task ProduceMessages(
52 | string brokers,
53 | string topic,
54 | int produceDelayMs,
55 | int numMessages,
56 | string file,
57 | CancellationToken cancel)
58 | {
59 | var producer = new Producer(brokers, new ProducerConfiguration(topic));
60 |
61 | // Technically not mandatory, but good idea to listen to possible errors and act accordingly to your application requirements
62 | producer.OnPermError +=
63 | (exception, messages) =>
64 | _log.Error($"Failed to write {messages.Length} because of {exception.Message}");
65 |
66 | // When message is confirmed by kafka broker to be persisted, OnSuccess is called. Can be used if upstream requires acknowlegement for extra reliability.
67 | producer.OnSuccess +=
68 | messages => { foreach (var msg in messages) _log.Info($"Sent {Encoding.UTF8.GetString(msg.Value)}"); };
69 |
70 | _log.Info("Connecting...");
71 | await producer.ConnectAsync();
72 |
73 | _log.Info("Producing...");
74 | Func dataFunc;
75 | if (file == null)
76 | dataFunc = i => i.ToString();
77 | else
78 | {
79 | var fileData = File.ReadAllLines(file);
80 | dataFunc = i => fileData[i%fileData.Length];
81 | }
82 |
83 | var data = Observable.Interval(TimeSpan.FromMilliseconds(produceDelayMs)).Select(dataFunc);
84 | if (numMessages > 0)
85 | data = data.Take(numMessages);
86 |
87 | var sendTask = data.Do(d => producer.Send(new Message {Value = Encoding.UTF8.GetBytes(d)})).ToTask(cancel);
88 |
89 | await sendTask;
90 |
91 | _log.Info("Closing...");
92 | await producer.CloseAsync(TimeSpan.FromSeconds(10));
93 |
94 | }
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/examples/Program.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Configuration;
4 | using System.IO;
5 | using System.Linq;
6 | using System.Text;
7 | using System.Threading;
8 | using System.Threading.Tasks;
9 | using log4net;
10 | using log4net.Config;
11 |
12 | namespace examples
13 | {
14 | class Program
15 | {
16 | #region staticLog4netAppender
17 | const string StaticConsoleAppender = @"
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 | ";
46 | #endregion staticLog4netAppender
47 |
48 | private static ILog _log;
49 |
50 | static void Main(string[] args)
51 | {
52 | var log4NetConfig = ConfigurationManager.AppSettings.Get("LoggingConfigFile");
53 | if (string.IsNullOrWhiteSpace(log4NetConfig))
54 | log4NetConfig = "log4net.config";
55 |
56 | if (File.Exists(log4NetConfig))
57 | {
58 | XmlConfigurator.ConfigureAndWatch(new FileInfo(log4NetConfig));
59 | }
60 | else
61 | {
62 | XmlConfigurator.Configure(new MemoryStream(Encoding.UTF8.GetBytes(StaticConsoleAppender)));
63 | }
64 | _log = LogManager.GetLogger(typeof(Program));
65 |
66 | kafka4net.Logger.SetupLog4Net();
67 |
68 | var showHelp = false;
69 | string brokers = null;
70 | string topic = null;
71 | bool isProducer = false, isConsumer = false;
72 | int produceDelayMs = 1000;
73 | int totalMessages = -1;
74 | string file = null;
75 |
76 | var opts = new NDesk.Options.OptionSet
77 | {
78 | {"?|h|help", "Show Help", v => showHelp = true},
79 | {"p|producer", "Run Producer. Cannot be used with -c|-consumer.", v => isProducer = true },
80 | {"c|consumer", "Run Consumer. Cannot be used with -p|-producer.", v => isConsumer = true },
81 | {"d:|delay-ms:", "Producer delay between messages in ms (default 1000). Specify an integer number of milliseconds", (int v) => produceDelayMs = v },
82 | {"m:|messages:", "Total number of messages to produce or consume before closing. Specify -1 (default) to send/receive indefinitely.", (int v) => totalMessages = v },
83 | {"f:|file:", "File for producer to read from or consumer to write to. In producer mode, the file is read line by line, sending the content of each line as a message, and repeating as necessary. In Consumer mode, messages are written to this file line by line. If not specified, producer sends increasing numeric values, and consumer writes to console.", v => file = v },
84 | {"b=|brokers=", "Comman delimited list of Broker(s) to connect to. Can be just name/IP to use default port of 9092, or :", v => brokers = v },
85 | {"t=|topic=", "Topic to Produce or Consume from.", v => topic = v }
86 | };
87 | var unusedArgs = opts.Parse(args);
88 | if (unusedArgs.Any())
89 | ArgsFail($"Didn't recognize these command line options: {string.Join(", ", unusedArgs)}", opts);
90 |
91 | if (isProducer && isConsumer)
92 | ArgsFail("Cannot specify both producer and consumer",opts);
93 |
94 | if (!isProducer && !isConsumer)
95 | ArgsFail("Must specify either producer or consumer", opts);
96 |
97 | if (string.IsNullOrWhiteSpace(brokers))
98 | ArgsFail("Must specify one or more brokers to connect to.",opts);
99 |
100 | if (string.IsNullOrWhiteSpace(topic))
101 | ArgsFail("Must specify a topic name.",opts);
102 |
103 | if (!args.Any() || showHelp)
104 | {
105 | PrintHelp(opts);
106 | Environment.Exit(args.Any() ? 0 : 1);
107 | }
108 |
109 | if (isProducer)
110 | {
111 | var cancel = new CancellationTokenSource();
112 | var produceTask = ProducerExample.ProduceMessages(brokers, topic, produceDelayMs, totalMessages, file, cancel.Token);
113 |
114 | _log.Debug("Starting...");
115 | Task.WaitAny(produceTask, Task.Factory.StartNew(() =>
116 | {
117 | Console.Out.WriteLine("Press 'Q' to cancel.");
118 | var key = "";
119 | while (key != "q")
120 | {
121 | key = Console.ReadKey().KeyChar.ToString().ToLower();
122 | }
123 |
124 | Console.WriteLine("Cancelling...");
125 | cancel.Cancel();
126 | }, cancel.Token));
127 |
128 | }
129 |
130 | if (isConsumer)
131 | {
132 | var cancel = new CancellationTokenSource();
133 | var consumeTask = ConsumerExample.ConsumeMessages(brokers, topic, file, cancel.Token);
134 |
135 | _log.Debug("Starting...");
136 | Task.WaitAny(consumeTask, Task.Factory.StartNew(() =>
137 | {
138 | Console.Out.WriteLine("Press 'Q' to cancel.");
139 | var key = "";
140 | while (key != "q")
141 | {
142 | key = Console.ReadKey().KeyChar.ToString().ToLower();
143 | }
144 |
145 | Console.WriteLine("Cancelling...");
146 | cancel.Cancel();
147 | }, cancel.Token));
148 |
149 | }
150 |
151 | }
152 |
153 | private static void ArgsFail(string errMsg, NDesk.Options.OptionSet opts)
154 | {
155 | _log.Error(errMsg);
156 | Console.Error.WriteLine(errMsg);
157 | PrintHelp(opts);
158 | Environment.Exit(1);
159 | }
160 |
161 | private static void PrintHelp(NDesk.Options.OptionSet opts)
162 | {
163 | _log.InfoFormat("Showing options in console...");
164 |
165 | Console.Out.WriteLine($"Usage: examples (--producer|--consumer) --brokers [:port][,[:port]...] --topic= [Options]+");
166 | Console.Out.WriteLine("Options:");
167 | Console.Out.WriteLine();
168 | opts.WriteOptionDescriptions(Console.Out);
169 | Console.Out.WriteLine("Press any key to continue...");
170 | Console.In.Read();
171 | }
172 | }
173 | }
174 |
--------------------------------------------------------------------------------
/examples/examples.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}
8 | Exe
9 | Properties
10 | examples
11 | examples
12 | v4.5.2
13 | 512
14 |
15 |
16 | true
17 | full
18 | false
19 | bin\Debug\
20 | DEBUG;TRACE
21 | prompt
22 | 4
23 |
24 |
25 | pdbonly
26 | true
27 | bin\Release\
28 | TRACE
29 | prompt
30 | 4
31 |
32 |
33 |
34 |
35 |
36 |
37 | ..\packages\log4net.2.0.5\lib\net45-full\log4net.dll
38 | True
39 |
40 |
41 | ..\packages\NDesk.Options.0.2.1\lib\NDesk.Options.dll
42 | True
43 |
44 |
45 |
46 |
47 |
48 | ..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll
49 | True
50 |
51 |
52 | ..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll
53 | True
54 |
55 |
56 | ..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll
57 | True
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 | {e3a39e66-db86-4e9c-b967-3aa71fa47c5d}
71 | kafka4net
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
88 |
--------------------------------------------------------------------------------
/examples/packages.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/kafka4net.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio 14
4 | VisualStudioVersion = 14.0.25420.1
5 | MinimumVisualStudioVersion = 10.0.40219.1
6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kafka4net", "src\kafka4net.csproj", "{E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}"
7 | EndProject
8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "tests", "tests\tests.csproj", "{8BAF9CFE-8324-4604-BCA1-96925E8B82AC}"
9 | EndProject
10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{35FED764-3D31-4487-B0F4-837F2ADE072B}"
11 | ProjectSection(SolutionItems) = preProject
12 | .nuget\packages.config = .nuget\packages.config
13 | EndProjectSection
14 | EndProject
15 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "examples", "examples\examples.csproj", "{D76203FD-59DA-4FA2-8793-B8810DA44D9C}"
16 | EndProject
17 | Global
18 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
19 | Debug|Any CPU = Debug|Any CPU
20 | Release|Any CPU = Release|Any CPU
21 | EndGlobalSection
22 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
23 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
24 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Debug|Any CPU.Build.0 = Debug|Any CPU
25 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Release|Any CPU.ActiveCfg = Release|Any CPU
26 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}.Release|Any CPU.Build.0 = Release|Any CPU
27 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
28 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Debug|Any CPU.Build.0 = Debug|Any CPU
29 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Release|Any CPU.ActiveCfg = Release|Any CPU
30 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}.Release|Any CPU.Build.0 = Release|Any CPU
31 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
32 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
33 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
34 | {D76203FD-59DA-4FA2-8793-B8810DA44D9C}.Release|Any CPU.Build.0 = Release|Any CPU
35 | EndGlobalSection
36 | GlobalSection(SolutionProperties) = preSolution
37 | HideSolutionNode = FALSE
38 | EndGlobalSection
39 | EndGlobal
40 |
--------------------------------------------------------------------------------
/log4j.properties:
--------------------------------------------------------------------------------
1 | # Root logger option
2 | log4j.rootLogger=INFO, stdout
3 |
4 | # Direct log messages to stdout
5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
6 | log4j.appender.stdout.Target=System.out
7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
--------------------------------------------------------------------------------
/nuget/nuget.exe.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/nuget/publish.bat:
--------------------------------------------------------------------------------
1 | @rem \bin\nuget-2.8.5\NuGet.exe pack ..\src\kafka4net.csproj -Build -Symbols -Prop Configuration=Release
2 | nuget.exe pack ..\src\kafka4net.csproj -Build -Symbols -Prop Configuration=Release; -MSBuildVersion 14 -Version 2.0.1
3 |
4 | @rem -Verbosity detailed
--------------------------------------------------------------------------------
/src/BrokerException.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net
4 | {
5 | public class BrokerException : Exception
6 | {
7 | public BrokerException(string message) : base(message) {}
8 | public BrokerException(string message, Exception inner) : base(message, inner) { }
9 | }
10 | }
11 |
--------------------------------------------------------------------------------
/src/Compression/KafkaSnappyStream.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 | using kafka4net.Utils;
4 | using Snappy;
5 |
6 | namespace kafka4net.Compression
7 | {
8 | ///
9 | /// Kafka's snappy framing is non-standard. See https://github.com/xerial/snappy-java
10 | /// [magic header:16 bytes]([block size:int32][compressed data:byte array])*
11 | /// Version and min compatible version are both 1
12 | ///
13 | sealed class KafkaSnappyStream : Stream
14 | {
15 | static readonly byte[] _snappyMagic = { 0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0 };
16 | const int SnappyHeaderLen = 16; // 8 bytes of magic and 2 ints for version compatibility
17 |
18 | CompressionStreamMode _mode;
19 | readonly Stream _base;
20 | byte[] _uncompressedBuffer;
21 | // Working buffer to store data from compressed stream and avoid allocations. Can be resized
22 | byte[] _compressedBuffer;// = new byte[10*1024];
23 | int _bufferLen;
24 | int _bufferPtr;
25 | readonly byte[] _headerWorkingBuffer = new byte[SnappyHeaderLen];
26 | static readonly byte[] _versionsHeader = { 0, 0, 0, 1, 0, 0, 0, 1};
27 |
28 | ///
29 | ///
30 | ///
31 | ///
32 | ///
33 | /// Recommended size is 32Kb, as default in java xerces implementation
34 | public KafkaSnappyStream(Stream stream, CompressionStreamMode mode, byte[] uncompressedBuffer, byte[] compressedBuffer)
35 | {
36 | _mode = mode;
37 | _base = stream;
38 | _uncompressedBuffer = uncompressedBuffer;
39 | _compressedBuffer = compressedBuffer;
40 |
41 | if (mode == CompressionStreamMode.Decompress)
42 | {
43 | if (!ReadHeader())
44 | throw new InvalidDataException("Failed to read snappy header");
45 | ReadBlock();
46 | }
47 | else
48 | {
49 | WriteHeader();
50 | }
51 | }
52 |
53 | ///
54 | /// Allocate buffers of size, recommended for kafka usage. CompressedBuffer will be allocated of enough size, which will not require reallocation.
55 | ///
56 | ///
57 | ///
58 | public static void AllocateBuffers(out byte[] uncompressedBuffer, out byte[] compressedBuffer)
59 | {
60 | uncompressedBuffer = new byte[32 * 1024]; // 32K is default buffer size in xerces
61 | compressedBuffer = new byte[SnappyCodec.GetMaxCompressedLength(uncompressedBuffer.Length)];
62 | }
63 |
64 | public override void Flush()
65 | {
66 | if(_bufferLen == 0)
67 | return;
68 |
69 | Flush(_uncompressedBuffer, 0, _bufferLen);
70 | _bufferLen = 0;
71 | }
72 |
73 | void Flush(byte[] buff, int offset, int count)
74 | {
75 | var compressedSize = SnappyCodec.Compress(buff, offset, count, _compressedBuffer, 0);
76 |
77 | BigEndianConverter.Write(_base, compressedSize);
78 | _base.Write(_compressedBuffer, 0, compressedSize);
79 | }
80 |
81 | public override long Seek(long offset, SeekOrigin origin)
82 | {
83 | throw new NotImplementedException();
84 | }
85 |
86 | public override void SetLength(long value)
87 | {
88 | throw new NotImplementedException();
89 | }
90 |
91 | public override int Read(byte[] buffer, int offset, int count)
92 | {
93 | if(!CanRead)
94 | throw new InvalidOperationException("Not a read stream");
95 |
96 | if(_bufferPtr < _bufferLen)
97 | return ReadInternalBuffer(buffer, offset, count);
98 |
99 | if (!ReadBlock())
100 | return 0;
101 | else
102 | return ReadInternalBuffer(buffer, offset, count);
103 | }
104 |
105 | public override void Write(byte[] buffer, int offset, int count)
106 | {
107 | if(!CanWrite)
108 | throw new InvalidOperationException("Not a write stream");
109 |
110 | while(count > 0)
111 | {
112 | var size = Math.Min(count, _uncompressedBuffer.Length - _bufferLen);
113 |
114 | // Optimization: if full frame is going to be flushed, skip array copying and compress stright from input buffer
115 | if (_bufferLen == 0 && size == _uncompressedBuffer.Length)
116 | {
117 | Flush(buffer, offset, size);
118 | count -= size;
119 | offset += size;
120 | continue;
121 | }
122 |
123 | Buffer.BlockCopy(buffer, offset, _uncompressedBuffer, _bufferLen, size);
124 | _bufferLen += size;
125 | count -= size;
126 | offset += size;
127 |
128 | if(_bufferLen == _uncompressedBuffer.Length)
129 | Flush();
130 | }
131 | }
132 |
133 | public override bool CanRead => _mode == CompressionStreamMode.Decompress;
134 | public override bool CanSeek => false;
135 | public override bool CanWrite => _mode == CompressionStreamMode.Compress;
136 | public override long Length { get { throw new NotImplementedException(); } }
137 | public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } }
138 |
139 | //
140 | // Implementation
141 | //
142 |
143 | bool ReadHeader()
144 | {
145 | //
146 | // Validate snappy header
147 | //
148 | if (!StreamUtils.ReadAll(_base, _headerWorkingBuffer, SnappyHeaderLen))
149 | return false;
150 |
151 | for(int i=0; i<_snappyMagic.Length; i++)
152 | if(_headerWorkingBuffer[i] != _snappyMagic[i])
153 | throw new InvalidDataException("Invalid snappy magic bytes");
154 |
155 | // TODO: validate snappy min compatible version
156 |
157 | return true;
158 | }
159 |
160 | void WriteHeader()
161 | {
162 | _base.Write(_snappyMagic, 0, _snappyMagic.Length);
163 | _base.Write(_versionsHeader, 0, 8);
164 | }
165 |
166 | bool ReadBlock()
167 | {
168 | if (!StreamUtils.ReadAll(_base, _compressedBuffer, 4))
169 | return false;
170 |
171 | var blockSize = BigEndianConverter.ReadInt32(_compressedBuffer);
172 |
173 | if (blockSize > _compressedBuffer.Length)
174 | _compressedBuffer = new byte[blockSize];
175 |
176 | if(!StreamUtils.ReadAll(_base, _compressedBuffer, blockSize))
177 | throw new InvalidDataException("Unexpected end of snappy data block");
178 |
179 | // Reallocate compressed buffer if needed
180 | var uncompressedLen = Snappy.SnappyCodec.GetUncompressedLength(_compressedBuffer, 0, blockSize);
181 | if (uncompressedLen > _uncompressedBuffer.Length)
182 | _uncompressedBuffer = new byte[uncompressedLen];
183 |
184 | var read = SnappyCodec.Uncompress(_compressedBuffer, 0, blockSize, _uncompressedBuffer, 0);
185 |
186 | _bufferLen = read;
187 | _bufferPtr = 0;
188 |
189 | return true;
190 | }
191 |
192 | int ReadInternalBuffer(byte[] buffer, int offset, int count)
193 | {
194 | var canRead = Math.Min(count, _bufferLen - _bufferPtr);
195 | Buffer.BlockCopy(_uncompressedBuffer, _bufferPtr, buffer, offset, canRead);
196 | _bufferPtr += canRead;
197 | return canRead;
198 | }
199 |
200 | protected override void Dispose(bool disposing)
201 | {
202 | if(_mode == CompressionStreamMode.Compress && _bufferLen != 0)
203 | Flush();
204 |
205 | base.Dispose(disposing);
206 | }
207 | }
208 |
209 | enum CompressionStreamMode
210 | {
211 | Compress,
212 | Decompress
213 | }
214 | }
215 |
--------------------------------------------------------------------------------
/src/Compression/StreamUtils.cs:
--------------------------------------------------------------------------------
1 | using System.IO;
2 |
3 | namespace kafka4net.Compression
4 | {
5 | static class StreamUtils
6 | {
7 | internal static bool ReadAll(Stream stream, byte[] dst, int count)
8 | {
9 | while (true)
10 | {
11 | var res = stream.Read(dst, 0, count);
12 | count -= res;
13 | if (count == 0)
14 | return true;
15 | if (res == 0)
16 | return false;
17 | }
18 | }
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/CompressionType.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net
2 | {
3 | public enum CompressionType
4 | {
5 | None = 0,
6 | Gzip = 1,
7 | Snappy = 2,
8 |
9 | // Beaware of https://issues.apache.org/jira/browse/KAFKA-3160
10 | Lz4 = 3,
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/src/ConnState.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net
2 | {
3 | enum ConnState
4 | {
5 | None,
6 | Connecting,
7 | Connected,
8 | Failed
9 | }
10 | }
11 |
--------------------------------------------------------------------------------
/src/Connection.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 | using System.Net.Sockets;
4 | using System.Threading;
5 | using System.Threading.Tasks;
6 | using kafka4net.Protocols;
7 | using kafka4net.Tracing;
8 |
9 | namespace kafka4net
10 | {
11 | internal class Connection
12 | {
13 | private static readonly ILogger _log = Logger.GetLogger();
14 |
15 | private readonly string _host;
16 | private readonly int _port;
17 | private readonly Action _onError;
18 | private TcpClient _client;
19 | private readonly SemaphoreSlim _connectionLock = new SemaphoreSlim(1, 1);
20 | private bool _closed;
21 | private Task _loopTask;
22 | private CancellationTokenSource _loopTaskCancel;
23 |
24 | internal ResponseCorrelation Correlation;
25 |
26 | internal Connection(string host, int port, Action onError = null)
27 | {
28 | _host = host;
29 | _port = port;
30 | _onError = onError;
31 | }
32 |
33 |
34 | internal static Tuple[] ParseAddress(string seedConnections)
35 | {
36 | return seedConnections.Split(',').
37 | Select(_ => _.Trim()).
38 | Where(_ => _ != null).
39 | Select(s =>
40 | {
41 | int port = 9092;
42 | string host = null;
43 | if (s.Contains(':'))
44 | {
45 | var parts = s.Split(new[] { ":" }, StringSplitOptions.RemoveEmptyEntries);
46 | if (parts.Length == 2)
47 | {
48 | host = parts[0];
49 | port = int.Parse(parts[1]);
50 | }
51 | }
52 | else
53 | {
54 | host = s;
55 | }
56 | return Tuple.Create(host, port);
57 | }).ToArray();
58 | }
59 |
60 | internal async Task GetClientAsync(bool noTransportErrors=false)
61 | {
62 | if (_closed)
63 | throw new ApplicationException("Attempt to reuse connection which is not supposed to be used anymore");
64 |
65 | //EtwTrace.Log.ConnectionWaitingForLock(_host, _port);
66 | await _connectionLock.WaitAsync();
67 | //EtwTrace.Log.ConnectionGotLock(_host, _port);
68 |
69 | try
70 | {
71 | if (_client != null && !_client.Connected)
72 | {
73 | _log.Debug("Replacing closed connection {0}:{1} with a new one", _host, _port);
74 | EtwTrace.Log.ConnectionReplaceClosedClient(_host, _port);
75 | _client = null;
76 | }
77 |
78 | if (_client == null)
79 | {
80 | _client = new TcpClient();
81 | EtwTrace.Log.ConnectionConnecting(_host, _port);
82 | await _client.ConnectAsync(_host, _port);
83 | EtwTrace.Log.ConnectionConnected(_host, _port);
84 |
85 | var currentClient = _client;
86 | Correlation = new ResponseCorrelation(async e =>
87 | {
88 | await MarkSocketAsFailed(currentClient);
89 | if(_onError != null && !noTransportErrors)
90 | _onError(e);
91 | }, _host+":"+_port+" conn object hash: " + GetHashCode());
92 |
93 | // If there was a prior connection and loop task, cancel them before creating a new one.
94 | if (_loopTask != null && _loopTaskCancel != null)
95 | {
96 | _loopTaskCancel.Cancel();
97 | _loopTask = null;
98 | }
99 |
100 | _loopTaskCancel = new CancellationTokenSource();
101 |
102 | // Close connection in case of any exception. It is important, because in case of deserialization exception,
103 | // we are out of sync and can't continue.
104 | _loopTask = Correlation.CorrelateResponseLoop(_client, _loopTaskCancel.Token)
105 | .ContinueWith(async t =>
106 | {
107 | _log.Debug("CorrelationLoop completed with status {0}. {1}", t.Status, t.Exception==null?"": string.Format("Closing connection because of error. {0}",t.Exception.Message));
108 | await _connectionLock.WaitAsync();//.ConfigureAwait(false);
109 | try
110 | {
111 | if (_client != null)
112 | {
113 | _client.Close();
114 | EtwTrace.Log.ConnectionDisconnected(_host, _port);
115 | }
116 | }
117 | // ReSharper disable once EmptyGeneralCatchClause
118 | catch {}
119 | finally
120 | {
121 | _client = null;
122 | _connectionLock.Release();
123 | }
124 | });
125 |
126 | return _client;
127 | }
128 | }
129 | catch(Exception e)
130 | {
131 | // some exception getting a connection, clear what we have for next time.
132 | _client = null;
133 | if (_onError != null && !noTransportErrors)
134 | _onError(e);
135 | throw;
136 | }
137 | finally
138 | {
139 | _connectionLock.Release();
140 | //EtwTrace.Log.ConnectionLockRelease(_host, _port);
141 | }
142 |
143 | return _client;
144 | }
145 |
146 | public override string ToString()
147 | {
148 | return string.Format("Connection: {0}:{1}", _host, _port);
149 | }
150 |
151 | ///
152 | /// If correlation loop experienced an error, data in Tcp stream become unreliable because we've lost synchronization.
153 | /// So it is needed to close socket and reopen a new one later to recover from error.
154 | ///
155 | ///
156 | ///
157 | async Task MarkSocketAsFailed(TcpClient tcp)
158 | {
159 | await _connectionLock.WaitAsync();
160 | try
161 | {
162 | if (_client != tcp)
163 | return;
164 |
165 | _log.Debug("Marking connection as failed. {0}", this);
166 |
167 | if (_client != null)
168 | {
169 | try
170 | {
171 | if (_loopTaskCancel != null)
172 | {
173 | EtwTrace.Log.Connection_MarkSocketAsFailed_CorrelationLoopCancelling(_host, _port);
174 | _loopTaskCancel.Cancel();
175 | }
176 | EtwTrace.Log.Connection_MarkSocketAsFailed_TcpClosing(_host, _port);
177 | _client.Close();
178 | }
179 | // ReSharper disable once EmptyGeneralCatchClause
180 | catch { /*empty*/ }
181 | finally { EtwTrace.Log.ConnectionDisconnected(_host, _port); }
182 | _client = null;
183 | }
184 | }
185 | finally
186 | {
187 | _connectionLock.Release();
188 | }
189 | }
190 |
191 | ///
192 | /// Is called when Cluster is shut down, or when seed broker is replaced with a permanent one.
193 | ///
194 | public async Task ShutdownAsync()
195 | {
196 | _closed = true;
197 | _log.Debug("{0} Shutting down.", this);
198 | try
199 | {
200 | if (_loopTaskCancel != null)
201 | _loopTaskCancel.Cancel();
202 |
203 | if (_loopTask != null)
204 | {
205 | try
206 | {
207 | _client.Close();
208 | await _loopTask;
209 | _log.Debug("Loop task completed");
210 | }
211 | catch (TaskCanceledException){}
212 | }
213 | }
214 | // ReSharper disable once EmptyGeneralCatchClause
215 | catch { }
216 | _log.Debug("{0} closed.", this);
217 | }
218 | }
219 | }
220 |
--------------------------------------------------------------------------------
/src/ConsumerConfiguration.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Reactive.Concurrency;
3 | using kafka4net.ConsumerImpl;
4 |
5 | namespace kafka4net
6 | {
7 | ///
8 | /// Where to begin consuming.
9 | /// - TopicHead starts at the first message in each partition (beginning of the queue).
10 | /// - TopicTail starts at the most recent message of each partition (end of the queue)
11 | /// - SpecifiedLocations starts at the locations specified by the PartitionOffsetProvider function (see constructor paramters).
12 | ///
13 | public enum ConsumerLocation : long
14 | {
15 | TopicStart = -2L,
16 | TopicEnd = -1L,
17 | SpecifiedLocations
18 | }
19 |
20 | ///
21 | /// Interface to configuration to describe to the consumer where to start consuming (and which partitions to consume from)
22 | ///
23 | public interface IStartPositionProvider
24 | {
25 | ///
26 | /// Called to determine which partitions to consume from. Return true to consume from it or false not to.
27 | /// If this method returns true, it is expected that GetStartOffset will return a valid offset.
28 | ///
29 | /// The partition ID to check if we should consume from
30 | /// true to subscribe and consume from this partition, false to skip it
31 | bool ShouldConsumePartition(int partitionId);
32 |
33 | ///
34 | /// Gets the starting offset to use to consume at for this partition.
35 | ///
36 | /// The partition ID
37 | /// The next offset to consume. This will be the next offset consumed.
38 | long GetStartOffset(int partitionId);
39 |
40 | ///
41 | /// The logical start location represented by this IStartPositionProvider.
42 | /// If this returns one of TopicHead or TopicTail, then all partitions returned from ShouldConsumePartition
43 | /// Will be started from their current head or tail offset, and the GetStartOffset method will not be called.
44 | ///
45 | ConsumerLocation StartLocation { get; }
46 | }
47 | public interface IStopPositionProvider
48 | {
49 | ///
50 | /// Determines whether the consumer is finished consuming from a given partition based on the current message.
51 | /// Once this function returns false, the current message will be delivered but no further messages will be consumed from this partition.
52 | ///
53 | ///
54 | ///
55 | bool IsPartitionConsumingComplete(ReceivedMessage currentMessage);
56 | }
57 |
58 | public class ConsumerConfiguration
59 | {
60 | ///
61 | /// Subscription is performed asynchronously.
62 | ///
63 | /// Comma separated list of seed brokers. Port numbers are optional.
64 | /// 192.168.56.10,192.168.56.20:8081,broker3.local.net:8181
65 | ///
66 | ///
67 | ///
68 | ///
69 | ///
70 | ///
71 | ///
72 | ///
73 | /// If set to true, subscriber must call consumers Ack function to keep data flowing.
74 | /// Is used to prevent out of memory errors when subscriber is slow and data velocity is high
75 | /// (re-reading the log from beginning for example).
76 | /// Make sure that subscriber consumes more than lowWatermark, or data might stop flowing because driver would
77 | /// wait for subscriber to drain until lowWatermark and subscriber would wait for more data until continue processing
78 | /// (could happen when buffering is used).
79 | ///
80 | ///
81 | ///
82 | /// Driver will schedule outgoing messages events using this scheduler. By default it is
83 | /// EventLoopScheduler. Be careful if you want to redefine it. Concurrent scheduler can rearrange order of messages
84 | /// within the same partition!
85 | public ConsumerConfiguration(
86 | string seedBrokers,
87 | string topic,
88 | IStartPositionProvider startPosition,
89 | int maxWaitTimeMs=500,
90 | int minBytesPerFetch = 1,
91 | int maxBytesPerFetch=256*1024,
92 | int lowWatermark = 500,
93 | int highWatermark = 2000,
94 | bool useFlowControl = false,
95 | IStopPositionProvider stopPosition = null,
96 | IScheduler scheduler = null)
97 | {
98 | LowWatermark = lowWatermark;
99 | HighWatermark = highWatermark;
100 | UseFlowControl = useFlowControl;
101 |
102 | if(lowWatermark < 0)
103 | throw new ArgumentException("Can not be negative", "lowWatermark");
104 |
105 | if (highWatermark < 0)
106 | throw new ArgumentException("Can not be negative", "highWatermark");
107 |
108 | if(highWatermark < lowWatermark)
109 | throw new InvalidOperationException("highWatermark must be greater than lowWatermark");
110 |
111 | SeedBrokers = seedBrokers;
112 | StartPosition = startPosition;
113 | Topic = topic;
114 | MaxWaitTimeMs = maxWaitTimeMs;
115 | MinBytesPerFetch = minBytesPerFetch;
116 | MaxBytesPerFetch = maxBytesPerFetch;
117 | StopPosition = stopPosition ?? new StopPositionNever();
118 | OutgoingScheduler = scheduler ?? Scheduler.Default/*(ts => new Thread(ts) { IsBackground = true, Name = "Consumer outgoing scheduler" })*/;
119 | }
120 |
121 | public string SeedBrokers { get; private set; }
122 | public IStartPositionProvider StartPosition { get; private set; }
123 | public IStopPositionProvider StopPosition { get; private set; }
124 | public string Topic { get; private set; }
125 | public int MaxWaitTimeMs { get; private set; }
126 | public int MinBytesPerFetch { get; private set; }
127 | public int MaxBytesPerFetch { get; private set; }
128 | public readonly int LowWatermark;
129 | public readonly int HighWatermark;
130 | public readonly bool UseFlowControl;
131 | public readonly IScheduler OutgoingScheduler;
132 | }
133 | }
134 |
--------------------------------------------------------------------------------
/src/ConsumerImpl/PartitionFetchState.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net.ConsumerImpl
2 | {
3 | class PartitionFetchState
4 | {
5 | public readonly int PartId;
6 | public readonly ConsumerLocation StartLocation;
7 | public long Offset;
8 |
9 | public PartitionFetchState(int partId, ConsumerLocation startLocation, long offset)
10 | {
11 | PartId = partId;
12 | StartLocation = startLocation;
13 | Offset = offset;
14 | }
15 |
16 | public override string ToString()
17 | {
18 | return string.Format("Part: {0}, Offset: {1} StartLocation: {2}", PartId, Offset, StartLocation);
19 | }
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/ConsumerImpl/PositionProviders.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 |
4 | namespace kafka4net.ConsumerImpl
5 | {
6 | public class StopPositionNever : IStopPositionProvider
7 | {
8 | public bool IsPartitionConsumingComplete(ReceivedMessage currentMessage)
9 | {
10 | return false;
11 | }
12 | }
13 |
14 | public class StartAndStopAtExplicitOffsets : IStartPositionProvider, IStopPositionProvider
15 | {
16 | private readonly TopicPartitionOffsets _startingOffsets;
17 | private readonly TopicPartitionOffsets _stoppingOffsets;
18 |
19 | public StartAndStopAtExplicitOffsets(Dictionary startingOffsets, Dictionary stoppingOffsets)
20 | {
21 | _startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets);
22 | _stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets);
23 | }
24 | public StartAndStopAtExplicitOffsets(IEnumerable> startingOffsets, IEnumerable> stoppingOffsets)
25 | {
26 | _startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets);
27 | _stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets);
28 | }
29 | public StartAndStopAtExplicitOffsets(IEnumerable> startingOffsets, IEnumerable> stoppingOffsets)
30 | {
31 | _startingOffsets = new TopicPartitionOffsets("__xxx__",startingOffsets);
32 | _stoppingOffsets = new TopicPartitionOffsets("__xxx__",stoppingOffsets);
33 | }
34 | public StartAndStopAtExplicitOffsets(TopicPartitionOffsets startingOffsets, TopicPartitionOffsets stoppingOffsets)
35 | {
36 | _startingOffsets = startingOffsets;
37 | _stoppingOffsets = stoppingOffsets;
38 | }
39 |
40 | public bool ShouldConsumePartition(int partitionId)
41 | {
42 | // we should only consume from this partition if we were told to by the start offsets and also if we are not stopping where we already are.
43 | return _startingOffsets.ShouldConsumePartition(partitionId) && _startingOffsets.NextOffset(partitionId) != _stoppingOffsets.NextOffset(partitionId);
44 | }
45 |
46 | public long GetStartOffset(int partitionId)
47 | {
48 | return _startingOffsets.GetStartOffset(partitionId);
49 | }
50 |
51 | public ConsumerLocation StartLocation
52 | {
53 | get { return ConsumerLocation.SpecifiedLocations; }
54 | }
55 |
56 | public bool IsPartitionConsumingComplete(ReceivedMessage currentMessage)
57 | {
58 | return _stoppingOffsets.IsPartitionConsumingComplete(currentMessage);
59 | }
60 | }
61 |
62 | public class StartPositionTopicEnd : StartPositionTopicTime
63 | {
64 | public override ConsumerLocation StartLocation { get { return ConsumerLocation.TopicEnd; } }
65 | }
66 |
67 | public class StartPositionTopicStart : StartPositionTopicTime
68 | {
69 | public override ConsumerLocation StartLocation { get { return ConsumerLocation.TopicStart; } }
70 | }
71 |
72 | public abstract class StartPositionTopicTime : IStartPositionProvider
73 | {
74 | private readonly ISet _partitionsToConsume;
75 |
76 | public StartPositionTopicTime() : this(null) { }
77 | public StartPositionTopicTime(IEnumerable partitionsToConsume) : this(new HashSet(partitionsToConsume)) {}
78 | public StartPositionTopicTime(ISet partitionsToConsume)
79 | {
80 | _partitionsToConsume = partitionsToConsume;
81 | }
82 |
83 | public bool ShouldConsumePartition(int partitionId) { return _partitionsToConsume == null || _partitionsToConsume.Contains(partitionId); }
84 | public long GetStartOffset(int partitionId) { throw new NotImplementedException(); }
85 | public abstract ConsumerLocation StartLocation { get; }
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/src/ConsumerImpl/TopicPartition.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Reactive.Disposables;
4 | using System.Reactive.Linq;
5 | using kafka4net.Internal;
6 |
7 | namespace kafka4net.ConsumerImpl
8 | {
9 | class TopicPartition : IObserver
10 | {
11 | private static readonly ILogger _log = Logger.GetLogger();
12 | private Consumer _subscribedConsumer;
13 | private readonly Cluster _cluster;
14 | private readonly string _topic;
15 | private readonly int _partitionId;
16 | private readonly PartitionFetchState _partitionFetchState;
17 |
18 | // subscription handles
19 | private IDisposable _fetcherChangesSubscription;
20 | private IDisposable _currentfetcherSubscription;
21 |
22 | public TopicPartition(Cluster cluster, string topic, int partitionId, long initialOffset)
23 | {
24 | _cluster = cluster;
25 | _topic = topic;
26 | _partitionId = partitionId;
27 | _partitionFetchState = new PartitionFetchState(
28 | PartitionId,
29 | ConsumerLocation.SpecifiedLocations,
30 | initialOffset);
31 | }
32 |
33 | public string Topic { get { return _topic; } }
34 | public int PartitionId { get { return _partitionId; } }
35 |
36 | public long CurrentOffset
37 | {
38 | get
39 | {
40 | // if we have not been initialized, something went wrong.
41 | if (_partitionFetchState == null)
42 | throw new Exception("Partition Fetch State is not initialized. Has a consumer subscribed?");
43 |
44 | return _partitionFetchState.Offset;
45 | }
46 | }
47 |
48 | public IObservable FlowControl { get { return _subscribedConsumer.FlowControl; } }
49 | public bool FlowControlEnabled { get { return _subscribedConsumer.FlowControlEnabled; } }
50 |
51 | ///
52 | /// Subscribe a consumer to this topic partition. The act of subscribing
53 | /// will cause this partition to seek out and connect to the "correct" Fetcher.
54 | ///
55 | /// The consumer subscribing. This is not IObservable because we want to subscribe to the FlowControlState of the consumer.
56 | ///
57 | public IDisposable Subscribe(Consumer consumer)
58 | {
59 | if (_subscribedConsumer != null && consumer != _subscribedConsumer)
60 | throw new Exception(string.Format("TopicPartition {0} is already subscribed to by a consumer!", this));
61 |
62 | _subscribedConsumer = consumer;
63 |
64 | // subscribe to fetcher changes for this partition.
65 | // We will immediately get a call with the "current" fetcher if it is available, and connect to it then.
66 | var fetchers = new List();
67 | _fetcherChangesSubscription = _cluster
68 | .GetFetcherChanges(_topic, _partitionId, consumer.Configuration).
69 | Do(fetchers.Add).
70 | Subscribe(OnNewFetcher,OnFetcherChangesError,OnFetcherChangesComplete);
71 |
72 | _log.Debug("Starting {0} fetchers", fetchers.Count);
73 | fetchers.ForEach(fetcher => fetcher.PartitionsUpdated());
74 |
75 | // give back a handle to close this topic partition.
76 | return Disposable.Create(DisposeImpl);
77 |
78 | }
79 |
80 | ///
81 | /// Handle the connection of a fetcher (potentially a new fetcher) for this partition
82 | ///
83 | /// The fetcher to use to fetch for this TopicPartition
84 | private void OnNewFetcher(Fetcher newFetcher)
85 | {
86 | // first close the subscription to the old fetcher if there is one.
87 | if (_currentfetcherSubscription != null)
88 | _currentfetcherSubscription.Dispose();
89 |
90 | // now subscribe to the new fetcher. This will begin pumping messages through to the consumer.
91 | if (newFetcher != null)
92 | {
93 | _log.Debug("{0} Received new fetcher. Fetcher: {1}. Subscribing to this fetcher.", this, newFetcher);
94 | _currentfetcherSubscription = newFetcher.Subscribe(this);
95 | newFetcher.PartitionsUpdated();
96 | }
97 | }
98 |
99 | private void OnFetcherChangesComplete()
100 | {
101 | // we aren't getting any more fetcher changes... shouldn't happen!
102 | _log.Warn("{0} Received FetcherChanges OnComplete event.... shouldn't happen unless we're shutting down.", this);
103 | DisposeImpl();
104 | }
105 |
106 | private void OnFetcherChangesError(Exception ex)
107 | {
108 | // we aren't getting any more fetcher changes... shouldn't happen!
109 | _log.Fatal(ex, "{0} Received FetcherChanges OnError event.... shouldn't happen!", this);
110 | DisposeImpl();
111 | }
112 |
113 | ///
114 | /// Called when there is a new message received. Pass it to the Consumer
115 | ///
116 | ///
117 | public void OnNext(ReceivedMessage value)
118 | {
119 | if (_subscribedConsumer == null)
120 | {
121 | _log.Error("{0} Recieved Message with no subscribed consumer. Discarding message.", this);
122 | }
123 | else
124 | {
125 | if (_partitionFetchState.Offset <= value.Offset)
126 | {
127 | // if we are sending back an offset greater than we asked for then we likely skipped an offset.
128 | // This happens when using Log Compaction (see https://kafka.apache.org/documentation.html#compaction )
129 | if (_partitionFetchState.Offset < value.Offset)
130 | _log.Info("{0} was expecting offset {1} but received larger offset {2}",this,_partitionFetchState.Offset,value.Offset);
131 |
132 | _subscribedConsumer.OnMessageArrivedInput.OnNext(value);
133 | // track that we handled this offset so the next time around, we fetch the next message
134 | _partitionFetchState.Offset = value.Offset + 1;
135 | }
136 | else
137 | {
138 | // the returned message offset was less than the offset we asked for, just skip this message.
139 | _log.Debug("{0} Skipping message offset {1} as it is less than requested offset {2}",this,value.Offset,_partitionFetchState.Offset);
140 | }
141 |
142 | }
143 | }
144 |
145 | public void OnError(Exception error)
146 | {
147 | // don't pass this error up to the consumer, log it and wait for a new fetcher
148 | _log.Warn("{0} Recieved Error from Fetcher. Waiting for new or updated Fetcher. Message: {1}", this, error.Message);
149 | _currentfetcherSubscription.Dispose();
150 | _currentfetcherSubscription = null;
151 | _cluster.NotifyPartitionStateChange(new PartitionStateChangeEvent(Topic, PartitionId, ErrorCode.FetcherException));
152 | }
153 |
154 | public void OnCompleted()
155 | {
156 | // this shouldn't happen, but don't pass this up to the consumer, log it and wait for a new fetcher
157 | _log.Warn("{0} Recieved OnComplete from Fetcher. Fetcher may have errored out. Waiting for new or updated Fetcher.", this);
158 | _currentfetcherSubscription.Dispose();
159 | _currentfetcherSubscription = null;
160 | }
161 |
162 | private void DisposeImpl()
163 | {
164 | // just end the subscription to the current fetcher and to the consumer.
165 | if (_subscribedConsumer != null)
166 | {
167 | _subscribedConsumer.OnTopicPartitionComplete(this);
168 | _subscribedConsumer = null;
169 | }
170 |
171 | if (_fetcherChangesSubscription != null)
172 | {
173 | _fetcherChangesSubscription.Dispose();
174 | _fetcherChangesSubscription = null;
175 | }
176 |
177 | if (_currentfetcherSubscription != null)
178 | {
179 | _currentfetcherSubscription.Dispose();
180 | _currentfetcherSubscription = null;
181 | }
182 | }
183 |
184 | public override string ToString()
185 | {
186 | return string.Format("'{0}' part: {1} offset: {2}",Topic, PartitionId, CurrentOffset);
187 | }
188 |
189 | public override bool Equals(object obj)
190 | {
191 | if (obj == null)
192 | return false;
193 |
194 | var objTopicPartition = obj as TopicPartition;
195 | if (objTopicPartition == null)
196 | return false;
197 |
198 | return _cluster == objTopicPartition._cluster && _topic == objTopicPartition._topic && _partitionId == objTopicPartition._partitionId;
199 | }
200 |
201 | public override int GetHashCode()
202 | {
203 | unchecked // disable overflow, for the unlikely possibility that you
204 | { // are compiling with overflow-checking enabled
205 | int hash = 27;
206 | hash = (13 * hash) + _topic.GetHashCode();
207 | hash = (13 * hash) + _partitionId.GetHashCode();
208 | return hash;
209 | }
210 | }
211 | }
212 | }
213 |
--------------------------------------------------------------------------------
/src/ConsumerImpl/TopicPartitionOffsets.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.IO;
4 | using System.Linq;
5 |
6 | namespace kafka4net.ConsumerImpl
7 | {
8 | ///
9 | /// Container Class to aid in storing offsets for all partitions in a topic. The class stores internally the NEXT offset to process for a partition. When the topic is empty, the offsets stored would be 0 (the next offset to fetch)
10 | /// When initialized to the TopicTail, it will contain the offset of the NEXT message to be written to the partition.
11 | ///
12 | public class TopicPartitionOffsets : IStartPositionProvider, IStopPositionProvider
13 | {
14 | public readonly string Topic;
15 | private readonly Dictionary _offsets;
16 |
17 | ///
18 | /// Initializes a TopicPartitionOffsets structure from a dictionary. The offsets in the dictionary must be the NEXT offset to process for each partition, not the *Current* offset.
19 | ///
20 | ///
21 | ///
22 | public TopicPartitionOffsets(string topic, Dictionary initialOffsets)
23 | {
24 | Topic = topic;
25 | _offsets = new Dictionary(initialOffsets);
26 | }
27 | public TopicPartitionOffsets(string topic, IEnumerable> startingOffsets)
28 | {
29 | Topic = topic;
30 | _offsets = startingOffsets.ToDictionary(kv => kv.Key, kv => kv.Value);
31 | }
32 | public TopicPartitionOffsets(string topic, IEnumerable> startingOffsets)
33 | {
34 | Topic = topic;
35 | _offsets = startingOffsets.ToDictionary(kv => kv.Item1, kv => kv.Item2);
36 | }
37 | public TopicPartitionOffsets(string topic)
38 | {
39 | Topic = topic;
40 | _offsets = new Dictionary();
41 | }
42 |
43 | ///
44 | /// Construct a new set of offsets from the given stream. It is assumed that the stream contains serialized data created with WriteOffsets and is positioned at the start of this data.
45 | ///
46 | ///
47 | public TopicPartitionOffsets(byte[] serializedBytes) : this(new MemoryStream(serializedBytes))
48 | {
49 | }
50 |
51 | ///
52 | /// Construct a new set of offsets from the given stream. It is assumed that the stream contains serialized data created with WriteOffsets and is positioned at the start of this data.
53 | ///
54 | ///
55 | public TopicPartitionOffsets(Stream inputStream)
56 | {
57 | var reader = new BinaryReader(inputStream);
58 | Topic = reader.ReadString();
59 |
60 | // read from the input stream and initialize the offsets.
61 | _offsets = ReadOffsets(reader);
62 | }
63 |
64 | ///
65 | /// Get the next offset to request for the given partition ID (current offset + 1)
66 | ///
67 | ///
68 | /// the next offset for the given partition
69 | public long NextOffset(int partitionId)
70 | {
71 | long offset;
72 | if (_offsets.TryGetValue(partitionId, out offset))
73 | return offset;
74 |
75 | throw new Exception(string.Format("Partition ID {0} does not exist in offsets for topic {1}", partitionId, Topic));
76 | }
77 |
78 | public IEnumerable Partitions { get { return new List(_offsets.Keys); }}
79 |
80 | public Dictionary GetPartitionsOffset { get { return new Dictionary(_offsets); } }
81 |
82 | ///
83 | /// Update the offset for the given partition ID to include the given offset. This will cause offset+1 to be returned by the NextOffset function.
84 | /// Note that this function expects to be passed the *Current* offset that has been processed.
85 | ///
86 | ///
87 | ///
88 | public void UpdateOffset(int partitionId, long offset)
89 | {
90 | _offsets[partitionId] = offset+1;
91 | }
92 |
93 | ///
94 | /// Returns the total number of messages difference between this set of offsets, and the given set.
95 | ///
96 | /// The earlier offsets. If the passed offsets are larger than this instance offsets, the result will be negative.
97 | ///
98 | public long MessagesSince(TopicPartitionOffsets priorOffsets)
99 | {
100 | if (_offsets.Count != priorOffsets._offsets.Count || Topic != priorOffsets.Topic)
101 | throw new ArgumentException("priorOffsets does not match Topic or Number of Partitions of this set of offsets.");
102 |
103 | return _offsets.Keys.Select(p => _offsets[p] - priorOffsets._offsets[p]).Sum();
104 | }
105 |
106 | ///
107 | /// Serialize all offsets in this object to the given stream
108 | ///
109 | ///
110 | public void WriteOffsets(Stream stream)
111 | {
112 | var writer = new BinaryWriter(stream);
113 | writer.Write(Topic);
114 | // number of offsets
115 | writer.Write(_offsets.Count);
116 | foreach (var partOffset in _offsets)
117 | {
118 | writer.Write(partOffset.Key);
119 | writer.Write(partOffset.Value);
120 | }
121 | }
122 |
123 | public byte[] WriteOffsets()
124 | {
125 | var memStream = new MemoryStream(800);
126 | WriteOffsets(memStream);
127 | return memStream.ToArray();
128 | }
129 |
130 | ///
131 | /// Deserialize all offsets in this object from the given stream.
132 | ///
133 | ///
134 | ///
135 | private Dictionary ReadOffsets(BinaryReader reader)
136 | {
137 | // number of offsets
138 | var offsetCount = reader.ReadInt32();
139 | var offsets = new Dictionary(offsetCount);
140 | for (var i = 0; i < offsetCount; i++)
141 | {
142 | offsets.Add(reader.ReadInt32(),reader.ReadInt64());
143 | }
144 | return offsets;
145 | }
146 |
147 | public override string ToString()
148 | {
149 | return string.Format("{0} Offsets:[{1}]",Topic,string.Join(",", _offsets.Select(o=>o.Key + "|" + o.Value)));
150 | }
151 |
152 | #region IStopPositionProvider
153 | public bool IsPartitionConsumingComplete(ReceivedMessage currentMessage)
154 | {
155 | // "NextOffset" is the "Next" offset, so subtract one to get the current offset.
156 | var stopOffset = NextOffset(currentMessage.Partition) - 1;
157 | return currentMessage.Offset == stopOffset;
158 | }
159 | #endregion IStopPositionProvider
160 |
161 | #region IStartPositionProvider
162 | public bool ShouldConsumePartition(int partitionId)
163 | {
164 | return _offsets.ContainsKey(partitionId);
165 | }
166 |
167 | public long GetStartOffset(int partitionId)
168 | {
169 | return NextOffset(partitionId);
170 | }
171 |
172 | public ConsumerLocation StartLocation
173 | {
174 | get { return ConsumerLocation.SpecifiedLocations; }
175 | }
176 | #endregion IStartPositionProvider
177 | }
178 | }
179 |
--------------------------------------------------------------------------------
/src/ErrorCode.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net
2 | {
3 | public enum ErrorCode : short
4 | {
5 | ///No error--it worked!
6 | NoError = 0,
7 | ///An unexpected server error
8 | Unknown = -1,
9 | ///The requested offset is outside the range of offsets maintained by the server for the given
10 | /// topic/partition.
11 | OffsetOutOfRange = 1,
12 | ///This indicates that a message contents does not match its CRC
13 | InvalidMessage = 2,
14 | ///This request is for a topic or partition that does not exist on this broker.
15 | UnknownTopicOrPartition = 3,
16 | ///The message has a negative size
17 | InvalidMessageSize = 4,
18 | ///This error is thrown if we are in the middle of a leadership election and there is currently
19 | /// no leader for this partition and hence it is unavailable for writes.
20 | LeaderNotAvailable = 5,
21 | ///This error is thrown if the client attempts to send messages to a replica that is not
22 | /// the leader for some partition. It indicates that the clients metadata is out of date.
23 | NotLeaderForPartition = 6,
24 | ///This error is thrown if the request exceeds the user-specified time limit in the request
25 | RequestTimedOut = 7,
26 | ///This is not a client facing error and is used only internally by intra-cluster
27 | /// broker communication
28 | BrokerNotAvailable = 8,
29 | ///Unused
30 | ReplicaNotAvailable = 9,
31 | ///The server has a configurable maximum message size to avoid unbounded memory allocation.
32 | /// This error is thrown if the client attempt to produce a message larger than this maximum.
33 | MessageSizeTooLarge = 10,
34 | ///Internal error code for broker-to-broker communication
35 | StaleControllerEpoch = 11,
36 | ///If you specify a string larger than configured maximum for offset metadata
37 | OffsetMetadataTooLarge = 12,
38 | /// The broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition)
39 | OffsetsLoadInProgress = 14,
40 | /// The broker returns this error code for consumer metadata requests or offset commit requests
41 | /// if the offsets topic has not yet been created
42 | ConsumerCoordinatorNotAvailable = 15,
43 | /// The broker returns this error code if it receives an offset fetch or commit request for a consumer group that it is not a coordinator for
44 | NotCoordinatorForConsumer = 16,
45 |
46 | // New in 0.8.2
47 | InvalidTopic = 17,
48 | MessageSetSizeTooLarge = 18,
49 | NotEnoughReplicas = 19,
50 | NotEnoughReplicasAfterAppend = 20,
51 |
52 | // Below are driver's error codes. They are not returned by kafka server but are "virtual" states, such as transport error for example.
53 | FetcherException = 1001,
54 | TransportError
55 | }
56 |
57 | static class ErrorCodeClassification
58 | {
59 | ///
60 | /// Compare error codes, ignoring ReplicaNotAvailable
61 | ///
62 | public static bool IsDifferent(this ErrorCode c1, ErrorCode c2)
63 | {
64 | if (c1 == ErrorCode.ReplicaNotAvailable)
65 | c1 = ErrorCode.NoError;
66 | if (c2 == ErrorCode.ReplicaNotAvailable)
67 | c2 = ErrorCode.NoError;
68 |
69 | return c1 != c2;
70 | }
71 |
72 | public static bool IsSuccess(this ErrorCode errorCode)
73 | {
74 | return errorCode == ErrorCode.NoError || errorCode == ErrorCode.ReplicaNotAvailable;
75 | }
76 |
77 | public static bool IsFailure(this ErrorCode errorCode)
78 | {
79 | return !IsSuccess(errorCode);
80 | }
81 |
82 | public static bool IsPermanentFailure(this ErrorCode errorCode)
83 | {
84 | return errorCode == ErrorCode.MessageSetSizeTooLarge ||
85 | errorCode == ErrorCode.MessageSizeTooLarge ||
86 | errorCode == ErrorCode.InvalidMessageSize ||
87 | errorCode == ErrorCode.InvalidMessage ||
88 | errorCode == ErrorCode.OffsetOutOfRange;
89 | }
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/src/FletcherHashedMessagePartitioner.cs:
--------------------------------------------------------------------------------
1 | using kafka4net.Metadata;
2 | using System;
3 | using System.Threading;
4 |
5 | namespace kafka4net
6 | {
7 | ///
8 | /// Default message partitioner. Uses a Fletcher hash to assign given message key to a partition. If no key, then randomly selects a partition.
9 | ///
10 | internal class FletcherHashedMessagePartitioner : IMessagePartitioner
11 | {
12 | private readonly ThreadLocal _rnd = new ThreadLocal(() => new Random());
13 |
14 | public PartitionMeta GetMessagePartition(Message message, PartitionMeta[] allPartitions)
15 | {
16 | var index = message.Key == null ?
17 | _rnd.Value.Next(allPartitions.Length) :
18 | Fletcher32HashOptimized(message.Key) % allPartitions.Length;
19 |
20 | return allPartitions[index];
21 |
22 | }
23 |
24 | /// Optimized Fletcher32 checksum implementation.
25 | ///
26 | private uint Fletcher32HashOptimized(byte[] msg)
27 | {
28 | if (msg == null)
29 | return 0;
30 | var words = msg.Length;
31 | int i = 0;
32 | uint sum1 = 0xffff, sum2 = 0xffff;
33 |
34 | while (words != 0)
35 | {
36 | var tlen = words > 359 ? 359 : words;
37 | words -= tlen;
38 | do
39 | {
40 | sum2 += sum1 += msg[i++];
41 | } while (--tlen != 0);
42 | sum1 = (sum1 & 0xffff) + (sum1 >> 16);
43 | sum2 = (sum2 & 0xffff) + (sum2 >> 16);
44 | }
45 | /* Second reduction step to reduce sums to 16 bits */
46 | sum1 = (sum1 & 0xffff) + (sum1 >> 16);
47 | sum2 = (sum2 & 0xffff) + (sum2 >> 16);
48 | return (sum2 << 16 | sum1);
49 | }
50 |
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/src/ILogger.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net
4 | {
5 | internal interface ILogger
6 | {
7 | void Debug(string msg);
8 | void Debug(string msg, params object[] args);
9 | void Debug(Exception e, string msg, params object[] args);
10 | void Info(string msg);
11 | void Info(string msg, params object[] args);
12 | void Info(Exception e, string msg, params object[] args);
13 | void Warn(string msg);
14 | void Warn(string msg, params object[] args);
15 | void Warn(Exception e, string msg, params object[] args);
16 | void Error(string msg);
17 | void Error(string msg, params object[] args);
18 | void Error(Exception e, string msg, params object[] args);
19 | void Fatal(string msg);
20 | void Fatal(string msg, params object[] args);
21 | void Fatal(Exception e, string msg, params object[] args);
22 | bool IsDebugEnabled { get; }
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/src/IMessagePartitioner.cs:
--------------------------------------------------------------------------------
1 | using kafka4net.Metadata;
2 |
3 | namespace kafka4net
4 | {
5 | ///
6 | /// Assign message to certain partition.
7 | /// WARNING: this interface is called in caller (client) thread. Thus, if caller is multithreaded,
8 | /// IMessagePartitioner implementation must be thread-safe.
9 | ///
10 | /// It is possible to move call to IMessagePartitioner at later stage in the driver, where it would be thread-safe, but it would mean that
11 | /// message will get its partition at unpredictable time from client point of view, which could be confusing.
12 | ///
13 | public interface IMessagePartitioner
14 | {
15 | PartitionMeta GetMessagePartition(Message message, PartitionMeta[] allPartitions);
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/Internal/PartitionStateChangeEvent.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net.Internal
4 | {
5 | internal class PartitionStateChangeEvent
6 | {
7 | public PartitionStateChangeEvent(string topic, int partitionId, ErrorCode errorCode)
8 | {
9 | Topic = topic;
10 | PartitionId = partitionId;
11 | ErrorCode = errorCode;
12 | }
13 |
14 | public readonly string Topic;
15 | public readonly int PartitionId;
16 | public readonly ErrorCode ErrorCode;
17 |
18 | public override bool Equals(object obj)
19 | {
20 | if (obj == null)
21 | return false;
22 |
23 | var pse = obj as PartitionStateChangeEvent;
24 | if (pse == null)
25 | return false;
26 |
27 | return Topic == pse.Topic && PartitionId == pse.PartitionId && ErrorCode == pse.ErrorCode;
28 | }
29 |
30 | public override int GetHashCode()
31 | {
32 | unchecked // disable overflow, for the unlikely possibility that you
33 | { // are compiling with overflow-checking enabled
34 | int hash = 27;
35 | hash = (13 * hash) + Topic.GetHashCode();
36 | hash = (13 * hash) + PartitionId.GetHashCode();
37 | hash = (13 * hash) + ErrorCode.GetHashCode();
38 | return hash;
39 | }
40 | }
41 |
42 | public override string ToString()
43 | {
44 | return string.Format("PartitionStateChangeEvent: '{0}'/{1}/{2}", Topic, PartitionId, ErrorCode);
45 | }
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/src/Logger.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Diagnostics;
3 | using System.Reflection;
4 | using System.Runtime.CompilerServices;
5 |
6 | namespace kafka4net
7 | {
8 | ///
9 | /// Dynamic logging binder. Allows to log to log4net and NLog without linking kafka4net with those libraries.
10 | /// If you want to enable kafka4net logs, configure your logger as usual and call Logger.SetupLog4Net() or Logger.SetupNLog().
11 | ///
12 | public class Logger
13 | {
14 | static Func LoggerFactory = t => new NullLogger();
15 |
16 | public static void SetupNLog()
17 | {
18 | var logManagerType = Type.GetType("NLog.LogManager, NLog");
19 | var method = logManagerType.GetMethod("GetLogger", BindingFlags.InvokeMethod | BindingFlags.Public | BindingFlags.Static, null, CallingConventions.Standard, new[] {typeof(string)}, null);
20 |
21 | var retType = method.ReturnType;
22 | var delegateTypeGeneric = typeof(Func<,>);
23 | var delegateType = delegateTypeGeneric.MakeGenericType(new[] {typeof(string), retType});
24 | var del = Delegate.CreateDelegate(delegateType, method);
25 | var func = (Func)del;
26 | LoggerFactory = t => new NLogAdaprter(func(t.FullName));
27 | }
28 |
29 | public static void SetupLog4Net()
30 | {
31 | var logManagerType = Type.GetType("log4net.LogManager, log4net");
32 | var method = logManagerType.GetMethod("GetLogger", BindingFlags.InvokeMethod | BindingFlags.Public | BindingFlags.Static, null, CallingConventions.Standard, new[] { typeof(string) }, null);
33 |
34 | var retType = method.ReturnType;
35 | var delegateTypeGeneric = typeof(Func<,>);
36 | var delegateType = delegateTypeGeneric.MakeGenericType(new[] { typeof(string), retType });
37 | var del = Delegate.CreateDelegate(delegateType, method);
38 | var func = (Func)del;
39 | LoggerFactory = t => new Log4NetAdaprter(func(t.FullName));
40 | }
41 |
42 | [MethodImpl(MethodImplOptions.NoInlining)]
43 | internal static ILogger GetLogger()
44 | {
45 | var frame = new StackFrame(1, false);
46 | var klass = frame.GetMethod().DeclaringType;
47 | return LoggerFactory(klass);
48 | }
49 |
50 | class NullLogger : ILogger
51 | {
52 | public void Debug(string msg) { }
53 | public void Debug(Exception e, string msg, params object[] args) {}
54 | public void Debug(string msg, params object[] args) { }
55 | public void Info(string msg) {}
56 | public void Info(string msg, params object[] args) {}
57 | public void Info(Exception e, string msg, params object[] args) {}
58 | public void Warn(string msg) {}
59 | public void Warn(string msg, params object[] args) { }
60 | public void Warn(Exception e, string msg, params object[] args) { }
61 | public void Error(string msg) {}
62 | public void Error(string msg, params object[] args) {}
63 | public void Error(Exception e, string msg, params object[] args) {}
64 | public void Fatal(string msg) {}
65 | public void Fatal(string msg, params object[] args) {}
66 | public void Fatal(Exception e, string msg, params object[] args) {}
67 | public bool IsDebugEnabled { get { return false; } }
68 | }
69 |
70 | class NLogAdaprter : ILogger
71 | {
72 | readonly dynamic _actualLogger;
73 |
74 | public NLogAdaprter(object actualLogger)
75 | {
76 | _actualLogger = actualLogger;
77 | }
78 |
79 | public void Debug(string msg)
80 | {
81 | _actualLogger.Debug(msg);
82 | }
83 |
84 | public void Debug(string msg, params object[] args)
85 | {
86 | _actualLogger.Debug(msg, args);
87 | }
88 |
89 | public void Debug(Exception e, string msg, params object[] args)
90 | {
91 | _actualLogger.Debug(string.Format(msg, args), e);
92 | }
93 |
94 | public bool IsDebugEnabled { get { return _actualLogger.IsDebugEnabled; } }
95 |
96 | public void Info(string msg)
97 | {
98 | _actualLogger.Info(msg);
99 | }
100 |
101 | public void Info(string msg, params object[] args)
102 | {
103 | _actualLogger.Info(msg, args);
104 | }
105 |
106 | public void Info(Exception e, string msg, params object[] args)
107 | {
108 | _actualLogger.Info(string.Format(msg, args), e);
109 | }
110 |
111 | public void Warn(string msg)
112 | {
113 | _actualLogger.Warn(msg);
114 | }
115 |
116 | public void Warn(string msg, params object[] args)
117 | {
118 | _actualLogger.Warn(msg, args);
119 | }
120 |
121 | public void Warn(Exception e, string msg, params object[] args)
122 | {
123 | _actualLogger.Warn(string.Format(msg, args), e);
124 | }
125 |
126 | public void Error(string msg)
127 | {
128 | _actualLogger.Error(msg);
129 | }
130 |
131 | public void Error(string msg, params object[] args)
132 | {
133 | _actualLogger.Error(msg, args);
134 | }
135 |
136 | public void Error(Exception e, string msg, params object[] args)
137 | {
138 | _actualLogger.Error(string.Format(msg, args), e);
139 | }
140 |
141 | public void Fatal(string msg)
142 | {
143 | _actualLogger.Fatal(msg);
144 | }
145 |
146 | public void Fatal(string msg, params object[] args)
147 | {
148 | _actualLogger.Fatal(msg, args);
149 | }
150 |
151 | public void Fatal(Exception e, string msg, params object[] args)
152 | {
153 | _actualLogger.Fatal(string.Format(msg, args), e);
154 | }
155 | }
156 |
157 | class Log4NetAdaprter : ILogger
158 | {
159 | readonly dynamic _actualLogger;
160 |
161 | public Log4NetAdaprter(object actualLogger)
162 | {
163 | _actualLogger = actualLogger;
164 | }
165 |
166 | public void Debug(string msg)
167 | {
168 | _actualLogger.Debug(msg);
169 | }
170 |
171 | public void Debug(string msg, params object[] args)
172 | {
173 | _actualLogger.DebugFormat(msg, args);
174 | }
175 |
176 | public void Debug(Exception e, string msg, params object[] args)
177 | {
178 | _actualLogger.Debug(string.Format(msg, args), e);
179 | }
180 |
181 | public bool IsDebugEnabled { get { return _actualLogger.IsDebugEnabled; } }
182 |
183 | public void Info(string msg)
184 | {
185 | _actualLogger.Info(msg);
186 | }
187 |
188 | public void Info(string msg, params object[] args)
189 | {
190 | _actualLogger.InfoFormat(msg, args);
191 | }
192 |
193 | public void Info(Exception e, string msg, params object[] args)
194 | {
195 | _actualLogger.Info(string.Format(msg, args), e);
196 | }
197 |
198 | public void Warn(string msg)
199 | {
200 | _actualLogger.Warn(msg);
201 | }
202 |
203 | public void Warn(string msg, params object[] args)
204 | {
205 | _actualLogger.WarnFormat(msg, args);
206 | }
207 |
208 | public void Warn(Exception e, string msg, params object[] args)
209 | {
210 | _actualLogger.Warn(string.Format(msg, args), e);
211 | }
212 |
213 | public void Error(string msg)
214 | {
215 | _actualLogger.Error(msg);
216 | }
217 |
218 | public void Error(string msg, params object[] args)
219 | {
220 | _actualLogger.ErrorFormat(msg, args);
221 | }
222 |
223 | public void Error(Exception e, string msg, params object[] args)
224 | {
225 | _actualLogger.Error(string.Format(msg, args), e);
226 | }
227 |
228 | public void Fatal(string msg)
229 | {
230 | _actualLogger.Fatal(msg);
231 | }
232 |
233 | public void Fatal(string msg, params object[] args)
234 | {
235 | _actualLogger.FatalFormat(msg, args);
236 | }
237 |
238 | public void Fatal(Exception e, string msg, params object[] args)
239 | {
240 | _actualLogger.Fatal(string.Format(msg, args), e);
241 | }
242 | }
243 | }
244 | }
245 |
--------------------------------------------------------------------------------
/src/Message.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net
2 | {
3 | public class Message
4 | {
5 | public byte[] Key;
6 | public byte[] Value;
7 |
8 | internal long Offset;
9 |
10 | ///
11 | /// Not part of the protocol. Is used to group messages belonging to the same partition before sending.
12 | /// Is set by Publisher before enqueing to partition queue.
13 | ///
14 | internal int PartitionId { get; set; }
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/Metadata/BrokerMeta.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 |
4 | namespace kafka4net.Metadata
5 | {
6 | class BrokerMeta
7 | {
8 | public int NodeId;
9 | public string Host;
10 | public int Port;
11 |
12 | // Not serialized, just a link to connection associated with this broker
13 | internal Connection Conn;
14 |
15 | public override string ToString()
16 | {
17 | return string.Format("{0}:{1} Id:{2}", Host, Port, NodeId);
18 | }
19 |
20 | #region comparer
21 | public static readonly IEqualityComparer NodeIdComparer = new ComparerImpl();
22 | class ComparerImpl : IEqualityComparer
23 | {
24 | public bool Equals(BrokerMeta x, BrokerMeta y)
25 | {
26 | if(x.NodeId != -99 || y.NodeId != -99)
27 | return x.NodeId == y.NodeId;
28 |
29 | // If those are non-resolved seed brokers, do property comparison, because they all have NodeId==-99
30 | return string.Equals(x.Host, y.Host, StringComparison.OrdinalIgnoreCase) && x.Port == y.Port;
31 | }
32 |
33 | public int GetHashCode(BrokerMeta obj)
34 | {
35 | return obj.NodeId;
36 | }
37 | }
38 | #endregion
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Metadata/PartitionMeta.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 |
3 | namespace kafka4net.Metadata
4 | {
5 | public class PartitionMeta
6 | {
7 | ///
8 | /// ReplicaNotAvailable is not really an error. Use IsSuccess() function instead of checking this value
9 | ///
10 | public ErrorCode ErrorCode;
11 | public int Id;
12 |
13 | ///
14 | /// The node id for the kafka broker currently acting as leader for this partition.
15 | /// If no leader exists because we are in the middle of a leader election this id will be -1.
16 | ///
17 | public int Leader;
18 |
19 | /// The set of alive nodes that currently acts as slaves for the leader for this partition
20 | public int[] Replicas;
21 |
22 | /// The set subset of the replicas that are "caught up" to the leader
23 | public int[] Isr;
24 |
25 | public override string ToString()
26 | {
27 | return string.Format("Id: {0} Leader: {1} Error: {2}, Replicas: {3}, Isr: {4}", Id, Leader, ErrorCode, IntToStringList(Replicas), IntToStringList(Isr));
28 | }
29 |
30 | static string IntToStringList(int[] ints)
31 | {
32 | if (ints == null)
33 | return "";
34 | return string.Join(",", ints.Select(i => i.ToString()).ToArray());
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/Metadata/PartitionOffsetInfo.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net.Metadata
2 | {
3 | public class PartitionOffsetInfo
4 | {
5 | public int Partition;
6 | /// If queue is empty than -1
7 | public long Head;
8 | public long Tail;
9 |
10 | public override string ToString()
11 | {
12 | return string.Format("Partition: {0} Head: {1} Tail {2}", Partition, Head, Tail);
13 | }
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Metadata/TopicMeta.cs:
--------------------------------------------------------------------------------
1 | using System.Collections.Generic;
2 | using System.Linq;
3 |
4 | namespace kafka4net.Metadata
5 | {
6 | public class TopicMeta
7 | {
8 | public ErrorCode ErrorCode;
9 | public string TopicName;
10 | public PartitionMeta[] Partitions;
11 |
12 | public override string ToString()
13 | {
14 | return string.Format("Topic '{0}' {1} Partitions [{2}]", TopicName, ErrorCode, string.Join(",", Partitions.AsEnumerable()));
15 | }
16 |
17 | #region name comparer
18 | static readonly NameCompareImpl NameComparerInstance = new NameCompareImpl();
19 | public static IEqualityComparer NameComparer { get { return NameComparerInstance; } }
20 | class NameCompareImpl : IEqualityComparer
21 | {
22 | public bool Equals(TopicMeta x, TopicMeta y)
23 | {
24 | return x.TopicName == y.TopicName;
25 | }
26 |
27 | public int GetHashCode(TopicMeta obj)
28 | {
29 | return obj.TopicName.GetHashCode();
30 | }
31 | }
32 | #endregion
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/PartitionFailedException.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net
4 | {
5 | class PartitionFailedException : Exception
6 | {
7 | public readonly string Topic;
8 | public readonly int Partition;
9 | public readonly ErrorCode ErrorCode;
10 |
11 | public PartitionFailedException(string topic, int partition, ErrorCode errorCode)
12 | {
13 | Topic = topic;
14 | Partition = partition;
15 | ErrorCode = errorCode;
16 | }
17 |
18 | public override string Message
19 | {
20 | get { return string.Format("Topic '{0}' partition {1} failed with error code {2}", Topic, Partition, ErrorCode); }
21 | }
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/ProducerConfiguration.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net
4 | {
5 | public class ProducerConfiguration
6 | {
7 | ///
8 | /// Full constructor for ProducerConfiguration. Specify all settings.
9 | /// Default values are:
10 | /// RequiredAcks: 1
11 | /// Timeout: 1s
12 | /// BatchFlushTime: 500ms
13 | /// BatchFlushSize: 1000
14 | ///
15 | ///
16 | /// Number of Kafka servers required to acknowledge the write for it to be considered successful. Default 1.
17 | /// Max time to wait before flushing messages. Default 500ms.
18 | /// Max number of messages to accumulate before flushing messages. Default 1000.
19 | /// Whether or not to automatically expand the send buffers (a buffer per partition of messages waiting to be sent). If set to false, an OnPermError will be triggered with messages that fail to get added to a full buffer.
20 | /// The initial size (in number of messages) of each send buffer. There is one send buffer per partition. If AutoGrowSendBuffers is true, the size will be expanded if necessary.
21 | /// The maximum size of MessageSet to send to the server. If exceed server's Segment Size,
22 | /// server will fail with MessageSetSizeTooLarge error. Default is 1Gb
23 | /// Type of compression
24 | public ProducerConfiguration(
25 | string topic,
26 | TimeSpan? batchFlushTime = null,
27 | int batchFlushSize = 1000,
28 | short requiredAcks = 1,
29 | bool autoGrowSendBuffers = true,
30 | int sendBuffersInitialSize = 200,
31 | int maxMessageSetSizeInBytes = 1024 * 1024 * 1024,
32 | TimeSpan? producerRequestTimeout = null,
33 | IMessagePartitioner partitioner = null,
34 | CompressionType compressionType = CompressionType.None)
35 | {
36 | Topic = topic;
37 | BatchFlushTime = batchFlushTime ?? TimeSpan.FromMilliseconds(500);
38 | BatchFlushSize = batchFlushSize;
39 | RequiredAcks = requiredAcks;
40 | AutoGrowSendBuffers = autoGrowSendBuffers;
41 | SendBuffersInitialSize = sendBuffersInitialSize;
42 | MaxMessageSetSizeInBytes = maxMessageSetSizeInBytes;
43 | ProduceRequestTimeout = producerRequestTimeout ?? TimeSpan.FromSeconds(1);
44 | Partitioner = partitioner ?? new FletcherHashedMessagePartitioner();
45 | CompressionType = compressionType;
46 | }
47 |
48 | public string Topic { get; private set; }
49 | public short RequiredAcks { get; private set; }
50 | public TimeSpan BatchFlushTime { get; private set; }
51 | public int BatchFlushSize { get; private set; }
52 | public TimeSpan ProduceRequestTimeout { get; private set; }
53 | public int ProduceRequestTimeoutMs { get { return (int)Math.Floor(ProduceRequestTimeout.TotalMilliseconds); } }
54 | public IMessagePartitioner Partitioner { get; private set; }
55 | public bool AutoGrowSendBuffers { get; private set; }
56 | public int SendBuffersInitialSize { get; private set; }
57 | public int MaxMessageSetSizeInBytes { get; private set; }
58 | public CompressionType CompressionType { get; set; }
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.CompilerServices;
3 | using System.Runtime.InteropServices;
4 |
5 | [assembly: AssemblyTitle("kafka4net")]
6 | [assembly: AssemblyDescription("C# Kafka driver")]
7 | [assembly: AssemblyCompany("NTent")]
8 | [assembly: AssemblyProduct("kafka4net")]
9 | [assembly: AssemblyCopyright("Copyright © NTent 2014")]
10 |
11 | [assembly: ComVisible(false)]
12 | [assembly: Guid("b47c3611-9b2d-4124-9a87-4a094a5dc928")]
13 |
14 | [assembly: InternalsVisibleTo("kafka4net-tests")]
15 |
16 |
--------------------------------------------------------------------------------
/src/Protocols/CorrelationLoopException.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net.Protocols
4 | {
5 | class CorrelationLoopException : Exception
6 | {
7 | public bool IsRequestedClose;
8 | public CorrelationLoopException(string message) : base(message)
9 | {
10 |
11 | }
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/src/Protocols/Protocol.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Net.Sockets;
3 | using System.Threading;
4 | using System.Threading.Tasks;
5 | using kafka4net.Metadata;
6 | using kafka4net.Protocols.Requests;
7 | using kafka4net.Protocols.Responses;
8 | using kafka4net.Tracing;
9 |
10 | namespace kafka4net.Protocols
11 | {
12 | // TODO: can this class be made static?
13 | internal class Protocol
14 | {
15 | private static readonly ILogger _log = Logger.GetLogger();
16 | private readonly Cluster _cluster;
17 | static readonly EtwTrace _etw = EtwTrace.Log;
18 |
19 | internal Protocol(Cluster cluster)
20 | {
21 | _cluster = cluster;
22 | }
23 |
24 | internal async Task Produce(ProduceRequest request)
25 | {
26 | var conn = request.Broker.Conn;
27 | var client = await conn.GetClientAsync();
28 | _log.Debug("Sending ProduceRequest to {0}, Request: {1}", conn, request);
29 | if(_etw.IsEnabled())
30 | _etw.ProtocolProduceRequest(request.ToString(), request.Broker.NodeId);
31 |
32 | var response = await conn.Correlation.SendAndCorrelateAsync(
33 | id => Serializer.Serialize(request, id),
34 | Serializer.GetProducerResponse,
35 | client,
36 | CancellationToken.None
37 | );
38 | _log.Debug("Got ProduceResponse: {0}", response);
39 | if (_etw.IsEnabled())
40 | _etw.ProtocolProduceResponse(response.ToString(), request.Broker.NodeId);
41 |
42 | return response;
43 | }
44 |
45 | internal async Task MetadataRequest(TopicRequest request, BrokerMeta broker = null, bool noTransportErrors = false)
46 | {
47 | TcpClient tcp;
48 | Connection conn;
49 |
50 | if (broker != null)
51 | {
52 | conn = broker.Conn;
53 | tcp = await conn.GetClientAsync(noTransportErrors);
54 | }
55 | else
56 | {
57 | var clientAndConnection = await _cluster.GetAnyClientAsync();
58 | conn = clientAndConnection.Item1;
59 | tcp = clientAndConnection.Item2;
60 | }
61 |
62 | //var tcp = await (broker != null ? broker.Conn.GetClientAsync() : _cluster.GetAnyClientAsync());
63 | _log.Debug("Sending MetadataRequest to {0}", tcp.Client.RemoteEndPoint);
64 | if (_etw.IsEnabled())
65 | {
66 | _etw.ProtocolMetadataRequest(request.ToString());
67 | }
68 |
69 | var response = await conn.Correlation.SendAndCorrelateAsync(
70 | id => Serializer.Serialize(request, id),
71 | Serializer.DeserializeMetadataResponse,
72 | tcp, CancellationToken.None);
73 |
74 | if (_etw.IsEnabled())
75 | {
76 | _etw.ProtocolMetadataResponse(response.ToString(),
77 | broker != null ? broker.Host : "",
78 | broker != null ? broker.Port : -1,
79 | broker != null ? broker.NodeId : -1);
80 | }
81 |
82 |
83 | return response;
84 | }
85 |
86 | internal async Task GetOffsets(OffsetRequest req, Connection conn)
87 | {
88 | var tcp = await conn.GetClientAsync();
89 |
90 | if(_etw.IsEnabled())
91 | _etw.ProtocolOffsetRequest(req.ToString());
92 |
93 | var response = await conn.Correlation.SendAndCorrelateAsync(
94 | id => Serializer.Serialize(req, id),
95 | Serializer.DeserializeOffsetResponse,
96 | tcp, CancellationToken.None);
97 |
98 | _log.Debug("Got OffsetResponse {0}", response);
99 | if (_etw.IsEnabled())
100 | _etw.ProtocolOffsetResponse(response.ToString());
101 |
102 | return response;
103 | }
104 |
105 | internal async Task Fetch(FetchRequest req, Connection conn)
106 | {
107 | _log.Debug("Sending FetchRequest to broker {1}. Request: {0}", req, conn);
108 | if (_etw.IsEnabled())
109 | _etw.ProtocolFetchRequest(req.ToString());
110 |
111 | // Detect disconnected server. Wait no less than 5sec.
112 | // If wait time exceed wait time + 3sec, consider it a timeout too
113 | //var timeout = Math.Max(5000, req.MaxWaitTime + 3000);
114 | //var cancel = new CancellationTokenSource(timeout);
115 |
116 | var tcp = await conn.GetClientAsync();
117 | var response = await conn.Correlation.SendAndCorrelateAsync(
118 | id => Serializer.Serialize(req, id),
119 | Serializer.DeserializeFetchResponse,
120 | tcp, /*cancel.Token*/ CancellationToken.None);
121 |
122 | if (_etw.IsEnabled())
123 | _etw.ProtocolFetchResponse(response.ToString());
124 |
125 | return response;
126 | }
127 | }
128 | }
129 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/FetchRequest.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 |
3 | namespace kafka4net.Protocols.Requests
4 | {
5 | class FetchRequest
6 | {
7 | ///
8 | /// The max wait time is the maximum amount of time in milliseconds to block waiting if
9 | /// insufficient data is available at the time the request is issued.
10 | ///
11 | public int MaxWaitTime;
12 |
13 | ///
14 | /// This is the minimum number of bytes of messages that must be available to give a response.
15 | /// If the client sets this to 0 the server will always respond immediately, however if there is
16 | /// no new data since their last request they will just get back empty message sets.
17 | /// If this is set to 1, the server will respond as soon as at least one partition has
18 | /// at least 1 byte of data or the specified timeout occurs. By setting higher values in combination
19 | /// with the timeout the consumer can tune for throughput and trade a little additional latency
20 | /// for reading only large chunks of data (e.g. setting MaxWaitTime to 100 ms and setting
21 | /// MinBytes to 64k would allow the server to wait up to 100ms to try to accumulate 64k
22 | /// of data before responding).
23 | ///
24 | public int MinBytes;
25 |
26 | public TopicData[] Topics;
27 |
28 | public class TopicData
29 | {
30 | public string Topic;
31 | public PartitionData[] Partitions;
32 |
33 | public override string ToString()
34 | {
35 | return string.Format("Topic: {0} Parts: [{1}]", Topic, Partitions == null ? "null" : string.Join(", ", Partitions.AsEnumerable()));
36 | }
37 | }
38 |
39 | public class PartitionData
40 | {
41 | public int Partition;
42 | public long FetchOffset;
43 | public int MaxBytes;
44 |
45 | public override string ToString()
46 | {
47 | return string.Format("Partition: {0} Offset: {1} MaxBytes: {2}", Partition, FetchOffset, MaxBytes);
48 | }
49 | }
50 |
51 | public override string ToString()
52 | {
53 | return string.Format("MaxTime: {0} MinBytes: {1} [{2}]", MaxWaitTime, MinBytes, Topics == null ? "null" : string.Join(",", Topics.AsEnumerable()));
54 | }
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/MessageData.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net.Protocols.Requests
4 | {
5 | class MessageData
6 | {
7 | /// This byte holds metadata attributes about the message. The lowest 2 bits contain
8 | /// the compression codec used for the message. The other bits should be set to 0
9 | //public byte Attributes;
10 |
11 | /// The key is an optional message key that was used for partition assignment.
12 | /// The key can be null
13 | public byte[] Key;
14 | /// The value is the actual message contents as an opaque byte array.
15 | /// Kafka supports recursive messages in which case this may itself contain a message set.
16 | /// The message can be null
17 | public ArraySegment Value;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/MessageSetItem.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net.Protocols.Requests
2 | {
3 | // TODO: delete this class, its unused
4 | class MessageSetItem
5 | {
6 | /// This is the offset used in kafka as the log sequence number. When the producer is sending
7 | /// messages it doesn't actually know the offset and can fill in any value here it likes
8 | public long Offset;
9 | public int MessageSize;
10 | public MessageData MessageData;
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/OffsetRequest.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 |
3 | namespace kafka4net.Protocols.Requests
4 | {
5 | class OffsetRequest
6 | {
7 | public string TopicName;
8 | public PartitionData[] Partitions;
9 | /// -1 for last offset, -2 for first offset, otherwise time
10 | //public long Time;
11 |
12 | public class PartitionData
13 | {
14 | public int Id;
15 | public long Time;
16 | public int MaxNumOffsets;
17 |
18 | public override string ToString()
19 | {
20 | return string.Format("[{0}:{1}:{2}]", Id, Time, MaxNumOffsets);
21 | }
22 | }
23 |
24 | public override string ToString()
25 | {
26 | return string.Format("{0} Id:Time:MaxNumOffsets [{1}]", TopicName, Partitions == null ? "null" : string.Join("\n ", Partitions.AsEnumerable()));
27 | }
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/PartitionData.cs:
--------------------------------------------------------------------------------
1 | using System.Collections.Generic;
2 | using System.Linq;
3 |
4 | namespace kafka4net.Protocols.Requests
5 | {
6 | class PartitionData
7 | {
8 | public int Partition;
9 | // is calculated at serizlization time
10 | //public int MessageSetSize;
11 | public IEnumerable Messages;
12 |
13 | /// Is not serialized. Is carried through to send error/success notifications
14 | /// if herror happen
15 | public Producer Pub;
16 |
17 | ///
18 | /// Not serialized.
19 | /// Copy of origianl, application provided messages. Is needed when error happen
20 | /// and driver notifys app that those messages have failed.
21 | ///
22 | public Message[] OriginalMessages;
23 |
24 | ///
25 | /// Non serialazable. Carries compression type until serialization happen, where messages are re-packaged
26 | /// into compressed message set.
27 | ///
28 | public CompressionType CompressionType;
29 |
30 | public override string ToString()
31 | {
32 | return string.Format("Part: {0} Messages: {1}", Partition, Messages == null ? "null" : Messages.Count().ToString());
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/ProduceRequest.cs:
--------------------------------------------------------------------------------
1 | using System.Collections.Generic;
2 | using kafka4net.Metadata;
3 |
4 | namespace kafka4net.Protocols.Requests
5 | {
6 | class ProduceRequest
7 | {
8 | ///
9 | /// This field indicates how many acknowledgements the servers should receive before responding to the request.
10 | /// If it is 0 the server will not send any response (this is the only case where the server will not reply
11 | /// to a request). If it is 1, the server will wait the data is written to the local log before sending
12 | /// a response. If it is -1 the server will block until the message is committed by all in sync replicas
13 | /// before sending a response. For any number > 1 the server will block waiting for this number of
14 | /// acknowledgements to occur (but the server will never wait for more acknowledgements than there
15 | /// are in-sync replicas).
16 | ///
17 | public short RequiredAcks;
18 |
19 | ///
20 | /// This provides a maximum time in milliseconds the server can await the receipt of the number
21 | /// of acknowledgements in RequiredAcks. The timeout is not an exact limit on the request time
22 | /// for a few reasons: (1) it does not include network latency, (2) the timer begins at the
23 | /// beginning of the processing of this request so if many requests are queued due to server
24 | /// overload that wait time will not be included, (3) we will not terminate a local write so if
25 | /// the local write time exceeds this timeout it will not be respected. To get a hard timeout of
26 | /// this type the client should use the socket timeout.
27 | ///
28 | public int Timeout;
29 |
30 | public IEnumerable TopicData;
31 |
32 | /// This propert is not serialized and is used only to carry connection info from grouping
33 | /// by broker to sending phase
34 | public BrokerMeta Broker;
35 |
36 | public override string ToString()
37 | {
38 | var topicDataStr = TopicData == null ? "" : string.Join("\n ", TopicData);
39 | return string.Format("Broker: {0} Acks: {1} Timeout: {2} Topics: [{3}]", Broker, RequiredAcks, Timeout, topicDataStr);
40 | }
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/TopicData.cs:
--------------------------------------------------------------------------------
1 | using System.Collections.Generic;
2 |
3 | namespace kafka4net.Protocols.Requests
4 | {
5 | class TopicData
6 | {
7 | public string TopicName;
8 | public IEnumerable PartitionsData;
9 |
10 | public override string ToString()
11 | {
12 | var partDataStr = PartitionsData == null ? "" : string.Join(", ", PartitionsData);
13 | return string.Format("Topic: {0} Parts: [{1}]", TopicName, partDataStr);
14 | }
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/Protocols/Requests/TopicRequest.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net.Protocols.Requests
2 | {
3 | class TopicRequest
4 | {
5 | public string[] Topics;
6 |
7 | public override string ToString()
8 | {
9 | return string.Format("[{0}]", string.Join(",", Topics));
10 | }
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/src/Protocols/ResponseCorrelation.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Concurrent;
3 | using System.IO;
4 | using System.Linq;
5 | using System.Net.Sockets;
6 | using System.Text;
7 | using System.Threading;
8 | using System.Threading.Tasks;
9 | using kafka4net.Tracing;
10 | using kafka4net.Utils;
11 |
12 | namespace kafka4net.Protocols
13 | {
14 | class ResponseCorrelation
15 | {
16 | // It is possible to have id per connection, but it's just simpler code and debug/tracing when it's global
17 | private static int _correlationId;
18 | private readonly ConcurrentDictionary> _corelationTable = new ConcurrentDictionary>();
19 | private readonly Action _onError;
20 | private static readonly ILogger _log = Logger.GetLogger();
21 | readonly string _id;
22 | private CancellationToken _cancellation;
23 | static readonly EtwTrace _etw = EtwTrace.Log;
24 |
25 | public ResponseCorrelation(Action onError, string id = "")
26 | {
27 | _onError = onError;
28 | _id = id;
29 | _etw.CorrelationCreate();
30 | }
31 |
32 | static async Task ReadBuffer(TcpClient client, byte[] buff, int len, CancellationToken cancel)
33 | {
34 | var pos = 0;
35 | var left = len;
36 | do
37 | {
38 | _etw.Correlation_ReadingBodyChunk(left);
39 | var read = await client.GetStream().ReadAsync(buff, pos, left, cancel);
40 |
41 | if (read == 0)
42 | {
43 | _log.Info("Server closed connection");
44 | _etw.CorrelationServerClosedConnection();
45 | throw new IOException("Server closed connection. 0 bytes read.");
46 | }
47 |
48 | pos += read;
49 | left -= read;
50 | _etw.CorrelationReadBodyChunk(read, left);
51 | } while (left > 0);
52 |
53 | return pos;
54 | }
55 |
56 | internal async Task CorrelateResponseLoop(TcpClient client, CancellationToken cancel)
57 | {
58 | try
59 | {
60 | _cancellation = cancel;
61 | _log.Debug("Starting reading loop from socket. {0}", _id);
62 | _etw.CorrelationStart();
63 |
64 | var buff = new byte[16 * 1024];
65 |
66 | while (client.Connected && !cancel.IsCancellationRequested)
67 | {
68 | try
69 | {
70 | // read message size
71 | //var buff = new byte[4];
72 | _etw.CorrelationReadingMessageSize();
73 | await ReadBuffer(client, buff, 4, cancel);
74 | if (cancel.IsCancellationRequested)
75 | {
76 | _log.Debug("Stopped reading from {0} because cancell requested", _id);
77 | return;
78 | }
79 |
80 | var size = BigEndianConverter.ToInt32(buff);
81 | _etw.CorrelationReadMessageSize(size);
82 |
83 | // TODO: size sanity check. What is the reasonable max size?
84 | //var body = new byte[size];
85 | if(size > buff.Length)
86 | buff = new byte[size];
87 | await ReadBuffer(client, buff, size, cancel);
88 | _etw.CorrelationReadBody(size);
89 |
90 | try
91 | {
92 | int correlationId = -1;
93 | // TODO: check read==size && read > 4
94 | correlationId = BigEndianConverter.ToInt32(buff);
95 | _etw.CorrelationReceivedCorrelationId(correlationId);
96 |
97 | // find correlated action
98 | Action handler;
99 | // TODO: if correlation id is not found, there is a chance of corrupt
100 | // connection. Maybe recycle the connection?
101 | if (!_corelationTable.TryRemove(correlationId, out handler))
102 | {
103 | _log.Error("Unknown correlationId: " + correlationId);
104 | continue;
105 | }
106 | _etw.CorrelationExecutingHandler();
107 | handler(buff, size, null);
108 | _etw.CorrelationExecutedHandler();
109 | }
110 | catch (Exception ex)
111 | {
112 | var error = string.Format("Error with handling message. Message bytes:\n{0}\n", FormatBytes(buff, size));
113 | _etw.CorrelationError(ex.Message + " " + error);
114 | _log.Error(ex, error);
115 | throw;
116 | }
117 | }
118 | catch (SocketException e)
119 | {
120 | // shorter version of socket exception, without stack trace dump
121 | _log.Info("CorrelationLoop socket exception. {0}. {1}", e.Message, _id);
122 | throw;
123 | }
124 | catch (ObjectDisposedException)
125 | {
126 | _log.Debug("CorrelationLoop socket exception. Object disposed. {0}", _id);
127 | throw;
128 | }
129 | catch (IOException)
130 | {
131 | _log.Info("CorrelationLoop IO exception. {0}", _id);
132 | throw;
133 | }
134 | catch (Exception e)
135 | {
136 | _log.Error(e, "CorrelateResponseLoop error. {0}", _id);
137 | throw;
138 | }
139 | }
140 |
141 | _log.Debug("Finished reading loop from socket. {0}", _id);
142 | EtwTrace.Log.CorrelationComplete();
143 | }
144 | catch (Exception e)
145 | {
146 | _corelationTable.Values.ForEach(c => c(null, 0, e));
147 | if (_onError != null && !cancel.IsCancellationRequested) // don't call back OnError if we were told to cancel
148 | _onError(e);
149 |
150 | if (!cancel.IsCancellationRequested)
151 | throw;
152 | }
153 | finally
154 | {
155 | _log.Debug("Finishing CorrelationLoop. Calling back error to clear waiters.");
156 | _corelationTable.Values.ForEach(c => c(null, 0, new CorrelationLoopException("Correlation loop closed. Request will never get a response.") { IsRequestedClose = cancel.IsCancellationRequested }));
157 | _log.Debug("Finished CorrelationLoop.");
158 | }
159 | }
160 |
161 | internal async Task SendAndCorrelateAsync(Func serialize, Func deserialize, TcpClient tcp, CancellationToken cancel)
162 | {
163 | var correlationId = Interlocked.Increment(ref _correlationId);
164 |
165 | var callback = new TaskCompletionSource();
166 | // TODO: configurable timeout
167 | // TODO: coordinate with Message's timeout
168 | // TODO: set exception in case of tcp socket exception and end of correlation loop
169 | //var timeout = new CancellationTokenSource(5 * 1000);
170 | try
171 | {
172 | var res = _corelationTable.TryAdd(correlationId, (body, size, ex) =>
173 | {
174 | if (ex == null)
175 | callback.TrySetResult(deserialize(body, size));
176 | else
177 | callback.TrySetException(ex);
178 | //timeout.Dispose();
179 | });
180 | if (!res)
181 | throw new ApplicationException("Failed to add correlationId: " + correlationId);
182 | var buff = serialize(correlationId);
183 |
184 | // TODO: think how buff can be reused. Would it be safe to declare buffer as a member? Is there a guarantee of one write at the time?
185 | _etw.CorrelationWritingMessage(correlationId, buff.Length);
186 | await tcp.GetStream().WriteAsync(buff, 0, buff.Length, cancel);
187 | }
188 | catch (Exception e)
189 | {
190 | callback.TrySetException(e);
191 | if (_onError != null)
192 | _onError(e);
193 | throw;
194 | }
195 |
196 | // TODO: is stream safe to use after timeout?
197 | cancel.Register(() => callback.TrySetCanceled(), useSynchronizationContext: false);
198 | try
199 | {
200 | return await callback.Task;
201 | }
202 | catch (TaskCanceledException e)
203 | {
204 | if (_onError != null)
205 | _onError(e);
206 | throw;
207 | }
208 | }
209 |
210 | private static string FormatBytes(byte[] buff, int len)
211 | {
212 | return buff.Take(Math.Min(256, len)).Aggregate(new StringBuilder(), (builder, b) => builder.Append(b.ToString("x2")),
213 | str => str.ToString());
214 | }
215 | }
216 | }
217 |
--------------------------------------------------------------------------------
/src/Protocols/Responses/FetchResponse.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 |
3 | namespace kafka4net.Protocols.Responses
4 | {
5 | class FetchResponse
6 | {
7 | public TopicFetchData[] Topics;
8 |
9 | public class TopicFetchData
10 | {
11 | public string Topic;
12 | public PartitionFetchData[] Partitions;
13 |
14 | public override string ToString()
15 | {
16 | return ToString(false);
17 | }
18 |
19 | public string ToString(bool onlyPartitionsWithMessages)
20 | {
21 | return string.Format("'{0}' PartId:ErrorCode:HighWatermarkOffset:MessageCount [{1}]", Topic, Partitions == null ? "null" : string.Join("\n ", Partitions.Where(p => !onlyPartitionsWithMessages || p.Messages.Length > 0).AsEnumerable()));
22 | }
23 | }
24 |
25 | public class PartitionFetchData
26 | {
27 | public int Partition;
28 | public ErrorCode ErrorCode;
29 | public long HighWatermarkOffset;
30 | public Message[] Messages;
31 |
32 | public override string ToString()
33 | {
34 | return string.Format("{0}:{1}:{2}:{3}", Partition, ErrorCode, HighWatermarkOffset, Messages == null ? "null" : Messages.Length.ToString());
35 | }
36 | }
37 |
38 | public override string ToString()
39 | {
40 | return ToString(false);
41 | }
42 |
43 | public string ToString(bool onlyPartitionsWithMessages)
44 | {
45 | if (Topics == null)
46 | return "null";
47 |
48 | return string.Format("[{0}]", string.Join("\n ", Topics.Select(t=>t.ToString(onlyPartitionsWithMessages))));
49 | }
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/src/Protocols/Responses/MetadataResponse.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 | using kafka4net.Metadata;
3 |
4 | namespace kafka4net.Protocols.Responses
5 | {
6 | class MetadataResponse
7 | {
8 | public BrokerMeta[] Brokers;
9 | public TopicMeta[] Topics;
10 |
11 | public override string ToString()
12 | {
13 | return string.Format("Brokers: [{0}], TopicMeta: [{1}]",
14 | Brokers == null ? "null" : string.Join(",", Brokers.AsEnumerable()),
15 | Topics == null ? "null" : string.Join("\n", Topics.AsEnumerable())
16 | );
17 | }
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/Protocols/Responses/OffsetResponse.cs:
--------------------------------------------------------------------------------
1 | using System.Globalization;
2 | using System.Linq;
3 |
4 | namespace kafka4net.Protocols.Responses
5 | {
6 | class OffsetResponse
7 | {
8 | public string TopicName;
9 | public PartitionOffsetData[] Partitions;
10 |
11 | public class PartitionOffsetData
12 | {
13 | public int Partition;
14 | public ErrorCode ErrorCode;
15 | public long[] Offsets;
16 |
17 | public override string ToString()
18 | {
19 | return string.Format("{0}:{1}:[{2}]", Partition, ErrorCode,
20 | Offsets == null ? "null" : string.Join(",", Offsets.Select(o => o.ToString(CultureInfo.InvariantCulture))));
21 | }
22 | }
23 |
24 | public override string ToString()
25 | {
26 | return string.Format("'{0}' [{1}]", TopicName, Partitions == null ? "null" : string.Join(",", Partitions.Select(p => p.ToString())));
27 | }
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/Protocols/Responses/ProducerResponse.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 |
3 | namespace kafka4net.Protocols.Responses
4 | {
5 | class ProducerResponse
6 | {
7 | internal TopicResponse[] Topics;
8 |
9 | internal class TopicResponse
10 | {
11 | public string TopicName;
12 | public PartitionResponse[] Partitions;
13 |
14 | public override string ToString()
15 | {
16 | return string.Format("{0} [{1}]", TopicName, Partitions == null ? "null" : string.Join("\n ", Partitions.AsEnumerable()));
17 | }
18 | }
19 |
20 | internal class PartitionResponse
21 | {
22 | public int Partition;
23 | public ErrorCode ErrorCode;
24 | public long Offset;
25 |
26 | public override string ToString()
27 | {
28 | return string.Format("Part: {0}, Err: {1} Offset: {2}", Partition, ErrorCode, Offset);
29 | }
30 | }
31 |
32 | public override string ToString()
33 | {
34 | return string.Format("Topics: [{0}]", Topics == null ? "null" : string.Join("\n ", Topics.AsEnumerable()));
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/ReceivedMessage.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net
2 | {
3 | public class ReceivedMessage
4 | {
5 | public string Topic;
6 | public int Partition;
7 | public byte[] Key;
8 | public byte[] Value;
9 | public long Offset;
10 | public long HighWaterMarkOffset;
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/src/Utils/AskEx.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Reactive.Concurrency;
3 | using System.Threading.Tasks;
4 |
5 | namespace kafka4net.Utils
6 | {
7 | static class AskEx
8 | {
9 | public static Task Ask(this IScheduler scheduler, Func action)
10 | {
11 | var src = new TaskCompletionSource();
12 | scheduler.Schedule(() =>
13 | {
14 | try
15 | {
16 | var res = action();
17 | src.SetResult(res);
18 | }
19 | catch (Exception e)
20 | {
21 | src.SetException(e);
22 | }
23 | });
24 |
25 | return src.Task;
26 | }
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/src/Utils/BigEndianConverter.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 |
4 | namespace kafka4net.Utils
5 | {
6 | static class BigEndianConverter
7 | {
8 | public static int ReadInt32(Stream s)
9 | {
10 | if(s.CanSeek && s.Position + 4 > s.Length)
11 | throw new Exception(string.Format("ReadInt32 needs 4 bytes but got ony {0}", s.Length - s.Position));
12 | return s.ReadByte() << 3 * 8 | s.ReadByte() << 2 * 8 | s.ReadByte() << 8 | s.ReadByte();
13 | }
14 |
15 | public static int ReadInt32(byte[] buff, int offset=0)
16 | {
17 | return buff[offset] << 3 * 8 | buff[offset + 1] << 2 * 8 | buff[offset+2] << 8 | buff[offset+3];
18 | }
19 |
20 |
21 | public static short ReadInt16(Stream s)
22 | {
23 | if (s.CanSeek && s.Position + 2 > s.Length)
24 | throw new Exception(string.Format("ReadInt16 needs 2 bytes but got ony {0}", s.Length - s.Position));
25 | return (short)((s.ReadByte() << 8) | s.ReadByte());
26 | }
27 |
28 | public static long ReadInt64(Stream stream)
29 | {
30 | if (stream.CanSeek && stream.Position + 8 > stream.Length)
31 | throw new Exception(string.Format("ReadInt64 needs 8 bytes but got ony {0}", stream.Length - stream.Position));
32 |
33 | var res = 0L;
34 | for (int i = 0; i < 8; i++)
35 | res = res << 8 | stream.ReadByte();
36 | return res;
37 | }
38 |
39 | public static void Write(Stream stream, long i)
40 | {
41 | ulong ui = (ulong)i;
42 | for (int j = 7; j >= 0; j--)
43 | stream.WriteByte((byte)(ui >> j * 8 & 0xff));
44 | }
45 |
46 | public static void Write(Stream stream, int i)
47 | {
48 | WriteByte(stream, i >> 8 * 3);
49 | WriteByte(stream, i >> 8 * 2);
50 | WriteByte(stream, i >> 8);
51 | WriteByte(stream, i);
52 | }
53 |
54 | public static void Write(Stream stream, uint i)
55 | {
56 | Write(stream, (int)i);
57 | }
58 |
59 | public static void Write(Stream stream, short i)
60 | {
61 | WriteByte(stream, i >> 8);
62 | WriteByte(stream, i);
63 | }
64 |
65 | public static void WriteByte(Stream stream, int i)
66 | {
67 | stream.WriteByte((byte)(i & 0xff));
68 | }
69 |
70 | public static void Write(byte[] buff, int i)
71 | {
72 | buff[0] = (byte)(i >> 8 * 3);
73 | buff[1] = (byte)((i & 0xff0000) >> 8 * 2);
74 | buff[2] = (byte)((i & 0xff00) >> 8);
75 | buff[3] = (byte)(i & 0xff);
76 | }
77 |
78 | public static int ToInt32(byte[] buff)
79 | {
80 | return (buff[0] << 8 * 3) | (buff[1] << 8 * 2) | (buff[2] << 8) | buff[3];
81 | }
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/src/Utils/CountObservable.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Reactive.Linq;
3 | using System.Reactive.Subjects;
4 | using System.Threading;
5 |
6 | namespace kafka4net.Utils
7 | {
8 | public sealed class CountObservable : IObservable
9 | {
10 | readonly ISubject _subj;
11 | readonly IConnectableObservable _publishedCounter;
12 | int _count;
13 |
14 | public CountObservable()
15 | {
16 | _subj = new Subject();
17 | _publishedCounter = _subj.Replay(1);
18 | _publishedCounter.Connect();
19 | _subj.OnNext(0);
20 | }
21 |
22 | public void Incr()
23 | {
24 | _subj.OnNext(Interlocked.Increment(ref _count));
25 | }
26 |
27 | public void Decr()
28 | {
29 | _subj.OnNext(Interlocked.Decrement(ref _count));
30 | }
31 |
32 | public IDisposable Subscribe(IObserver observer)
33 | {
34 | return _publishedCounter.Subscribe(observer);
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/Utils/Crc32.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net.Utils
4 | {
5 | /// Simplfied version of http://damieng.com/blog/2006/08/08/calculating_crc32_in_c_and_net
6 | static class Crc32
7 | {
8 | const UInt32 _polynomial = 0xEDB88320u;
9 | static readonly UInt32[] _table = InitializeTable();
10 |
11 | public static UInt32 Update(byte[] buffer, UInt32 state = ~0U, int len = -1, int offset = 0)
12 | {
13 | if (len == -1)
14 | len = buffer.Length;
15 |
16 | for (int i = offset; i < offset + len; i++)
17 | state = (state >> 8) ^ _table[buffer[i] ^ state & 0xff];
18 |
19 | return state;
20 | }
21 |
22 | public static UInt32 Update(ArraySegment buffer, UInt32 state = ~0U)
23 | {
24 | var array = buffer.Array;
25 | var end = buffer.Offset + buffer.Count;
26 |
27 | for (int i = buffer.Offset; i < end; i++)
28 | state = (state >> 8) ^ _table[array[i] ^ state & 0xff];
29 |
30 | return state;
31 | }
32 |
33 | public static UInt32 Update(int i, UInt32 state)
34 | {
35 | state = (state >> 8) ^ _table[(i >> 8 * 3 & 0xff) ^ state & 0xff];
36 | state = (state >> 8) ^ _table[(i >> 8 * 2 & 0xff) ^ state & 0xff];
37 | state = (state >> 8) ^ _table[(i >> 8 & 0xff) ^ state & 0xff];
38 | state = (state >> 8) ^ _table[(i & 0xff) ^ state & 0xff];
39 | return state;
40 | }
41 |
42 | public static UInt32 Update(byte b, UInt32 state = ~0U)
43 | {
44 | state = (state >> 8) ^ _table[b ^ state & 0xff];
45 | return state;
46 | }
47 |
48 | public static UInt32 GetHash(UInt32 state)
49 | {
50 | return ~state;
51 | }
52 |
53 | private static UInt32[] InitializeTable()
54 | {
55 | var createTable = new UInt32[256];
56 | for (var i = 0; i < 256; i++)
57 | {
58 | var entry = (UInt32)i;
59 | for (var j = 0; j < 8; j++)
60 | if ((entry & 1) == 1)
61 | entry = (entry >> 1) ^ _polynomial;
62 | else
63 | entry = entry >> 1;
64 | createTable[i] = entry;
65 | }
66 |
67 | return createTable;
68 | }
69 |
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/src/Utils/DateTimeExtensions.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace kafka4net.Utils
4 | {
5 | public static class DateTimeExtensions
6 | {
7 |
8 | ///
9 | /// Unix epoch time from a datetime
10 | ///
11 | ///
12 | ///
13 | public static int DateTimeToEpochSeconds(this DateTimeOffset date)
14 | {
15 | var t = (date - new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero));
16 | return (int)t.TotalSeconds;
17 | }
18 |
19 | ///
20 | /// Datetime from an int representing a unix epoch time
21 | ///
22 | ///
23 | ///
24 | public static DateTimeOffset EpochSecondsToDateTime(this int secondsSinceEpoch)
25 | {
26 | var date = new DateTimeOffset(1970, 1, 1,0,0,0,TimeSpan.Zero);
27 | return date.AddSeconds(secondsSinceEpoch);
28 | }
29 |
30 | ///
31 | /// Unix epoch time from a datetime
32 | ///
33 | ///
34 | ///
35 | public static long DateTimeToEpochMilliseconds(this DateTimeOffset date)
36 | {
37 | var t = (date - new DateTimeOffset(1970, 1, 1,0,0,0,TimeSpan.Zero));
38 | return (long)t.TotalMilliseconds;
39 | }
40 |
41 | ///
42 | /// Datetime from an int representing a unix epoch time
43 | ///
44 | ///
45 | ///
46 | public static DateTimeOffset EpochMillisecondsToDateTime(this long msSinceEpoch)
47 | {
48 | var date = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero);
49 | return date.AddMilliseconds(msSinceEpoch);
50 | }
51 |
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/src/Utils/EnumerableEx.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Diagnostics;
4 |
5 | namespace kafka4net.Utils
6 | {
7 | static class EnumerableEx
8 | {
9 | [DebuggerStepThrough]
10 | public static void ForEach(this IEnumerable items, Action action)
11 | {
12 | foreach (T item in items)
13 | action(item);
14 | }
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/Utils/LittleEndianConverter.cs:
--------------------------------------------------------------------------------
1 |
2 | namespace kafka4net.Utils
3 | {
4 | internal static class LittleEndianConverter
5 | {
6 | public static uint ReadUInt32(byte[] buff, int offset)
7 | {
8 | return (uint)(buff[offset+3] << 3 * 8 | buff[offset+2] << 2 * 8 | buff[offset+1] << 8 | buff[offset]);
9 | }
10 |
11 | public static void Write(uint i, byte[] buff, int offset)
12 | {
13 | buff[offset] = (byte)(i & 0xff);
14 | buff[offset + 1] = (byte)(i >> 8 & 0xff);
15 | buff[offset + 2] = (byte)(i >> 8*2 & 0xff);
16 | buff[offset + 3] = (byte)(i >> 8*3 & 0xff);
17 | }
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/Utils/RxSyncContextFromScheduler.cs:
--------------------------------------------------------------------------------
1 | using System.Reactive.Concurrency;
2 | using System.Threading;
3 |
4 | namespace kafka4net.Utils
5 | {
6 | ///
7 | /// Implements SynchronizationContext which routes "async" keyword callbacks to Rx Scheduler
8 | ///
9 | class RxSyncContextFromScheduler : SynchronizationContext
10 | {
11 | private readonly IScheduler _scheduler;
12 |
13 | public RxSyncContextFromScheduler(IScheduler scheduler)
14 | {
15 | _scheduler = scheduler;
16 | }
17 |
18 | public override void Post(SendOrPostCallback d, object state)
19 | {
20 | _scheduler.Schedule(() => d(state));
21 | }
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/Utils/TaskEx.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 |
7 | namespace kafka4net.Utils
8 | {
9 | static class TaskEx
10 | {
11 | ///
12 | /// Not very performant, but good enough to use in shutdown routine.
13 | /// http://blogs.msdn.com/b/pfxteam/archive/2011/11/10/10235834.aspx
14 | ///
15 | public async static Task TimeoutAfter(this Task task, TimeSpan timeout)
16 | {
17 | var delay = Task.Delay(timeout);
18 | var res = await Task.WhenAny(task, delay).ConfigureAwait(false);
19 | return res == task;
20 | }
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/src/Utils/WatchdogScheduler.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Diagnostics;
3 | using System.Reactive.Concurrency;
4 | using System.Reactive.Linq;
5 |
6 | namespace kafka4net.Utils
7 | {
8 | ///
9 | /// Wrapper around scheduler, which check execution duration every second and expose the duration as DelaySampler observable.
10 | /// The caller can use DelaySampler to set treshold and take an action if it is suspected that working thread is hang.
11 | /// Also it capture the last executed stack, which help finding offending code.
12 | ///
13 | class WatchdogScheduler : LocalScheduler, ISchedulerPeriodic, IDisposable
14 | {
15 | readonly IScheduler _scheduler;
16 | DateTime _actionStart = DateTime.MaxValue;
17 | public readonly IObservable DelaySampler;
18 | public StackTrace LastStack;
19 | bool _disposed;
20 | private static readonly ILogger _log = Logger.GetLogger();
21 |
22 |
23 | public WatchdogScheduler(IScheduler scheduler)
24 | {
25 | _scheduler = scheduler;
26 |
27 | DelaySampler = Observable.Interval(TimeSpan.FromSeconds(1)).
28 | Select(_ => _actionStart).
29 | Where(_ => _ != DateTime.MaxValue).
30 | Select(_ => DateTime.Now - _);
31 | }
32 |
33 | public override IDisposable Schedule(TState state, TimeSpan dueTime, Func action)
34 | {
35 | var stack = new StackTrace(false);
36 | Func actionWrapper = (scheduler, state1) =>
37 | {
38 | _actionStart = DateTime.Now;
39 | LastStack = stack;
40 | try
41 | {
42 | return action(scheduler, state1);
43 | }
44 | finally
45 | {
46 | LastStack = null;
47 | _actionStart = DateTime.MaxValue;
48 | }
49 | };
50 |
51 | return _scheduler.Schedule(state, dueTime, actionWrapper);
52 | }
53 |
54 | public IDisposable SchedulePeriodic(TState state, TimeSpan period, Func action)
55 | {
56 | var stack = new StackTrace(false);
57 | Func actionWrapper = (s1) =>
58 | {
59 | _actionStart = DateTime.Now;
60 | LastStack = stack;
61 | try
62 | {
63 | return action(s1);
64 | }
65 | finally
66 | {
67 | LastStack = null;
68 | _actionStart = DateTime.MaxValue;
69 | }
70 | };
71 |
72 | return _scheduler.SchedulePeriodic(state, period, actionWrapper);
73 | }
74 |
75 | public void Dispose()
76 | {
77 | if(_disposed)
78 | return;
79 | _disposed = true;
80 |
81 | (_scheduler as IDisposable)?.Dispose();
82 | }
83 | }
84 | }
--------------------------------------------------------------------------------
/src/WorkingThreadHungException.cs:
--------------------------------------------------------------------------------
1 | namespace kafka4net
2 | {
3 | public class WorkingThreadHungException : BrokerException
4 | {
5 | public WorkingThreadHungException(string message) : base(message) {}
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/src/kafka4net.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {E3A39E66-DB86-4E9C-B967-3AA71FA47C5D}
8 | Library
9 | Properties
10 | kafka4net
11 | kafka4net
12 | v4.5
13 | 512
14 |
15 |
16 |
17 |
18 |
19 | true
20 | full
21 | false
22 | bin\Debug\
23 | DEBUG;TRACE
24 | prompt
25 | 4
26 | false
27 |
28 |
29 | pdbonly
30 | true
31 | bin\Release\
32 | TRACE
33 | prompt
34 | 4
35 |
36 |
37 |
38 | ..\packages\Crc32C.NET.1.0.5.0\lib\net20\Crc32C.NET.dll
39 | True
40 |
41 |
42 | ..\packages\lz4net.1.0.10.93\lib\net4-client\LZ4.dll
43 | True
44 |
45 |
46 | ..\packages\Snappy.NET.1.1.1.8\lib\net45\Snappy.NET.dll
47 | True
48 |
49 |
50 |
51 |
52 |
53 | ..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll
54 |
55 |
56 | ..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll
57 |
58 |
59 | ..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll
60 |
61 |
62 | ..\packages\Rx-PlatformServices.2.2.5\lib\net45\System.Reactive.PlatformServices.dll
63 |
64 |
65 | False
66 | ..\packages\Microsoft.Tpl.Dataflow.4.5.24\lib\portable-net45+win8+wpa81\System.Threading.Tasks.Dataflow.dll
67 |
68 |
69 | ..\packages\xxHashSharp.1.0.0\lib\net45\xxHashSharp.dll
70 | True
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 | Designer
139 |
140 |
141 |
142 |
143 | true
144 |
145 |
146 |
147 |
148 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.
149 |
150 |
151 |
152 |
153 |
160 |
161 |
162 |
--------------------------------------------------------------------------------
/src/kafka4net.nuspec:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | kafka4net
5 | $version$
6 | NTent
7 | http://www.apache.org/licenses/
8 | https://github.com/ntent-ad/kafka4net
9 | false
10 | kafka4net
11 | Event-driven async Kafka driver for .NET
12 |
13 | Kafka driver for .NET
14 |
15 | Event-driven architecture, all asynchronous.
16 | Automatic following changes of Leader Partition in case of broker failure.
17 | Integration tests are part of the codebase. Use Vagrant to provision 1 zookeeper and 3 kafka virtual servers.
18 | Use RxExtensions library to expose API and for internal implementation.
19 | Support compression (gzip, lz4, snappy).
20 |
21 | Bug fix release
22 | Copyright NTent 2015
23 | kafka kafka4net
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/src/packages.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/tests/CompressionTests.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 | using System.Linq;
4 | using kafka4net.Compression;
5 | using NUnit.Framework;
6 |
7 | namespace tests
8 | {
9 | [TestFixture]
10 | public class CompressionTests
11 | {
12 | [Test]
13 | [Category("Compression")]
14 | public void TestLz4Stream()
15 | {
16 | var buffers = Enumerable.Range(1, 256 * 1024).
17 | AsParallel().
18 | Select(size => {
19 | var buff = new byte[size];
20 | new Random().NextBytes(buff);
21 | return buff;
22 | });
23 |
24 | buffers.
25 | ForAll(buffer =>
26 | {
27 | var compressed = new MemoryStream();
28 | var lz = new Lz4KafkaStream(compressed, CompressionStreamMode.Compress);
29 | lz.Write(buffer, 0, buffer.Length);
30 | lz.Close();
31 | var compressedBuff = compressed.ToArray();
32 |
33 | var lz2 = new Lz4KafkaStream(new MemoryStream(compressedBuff), CompressionStreamMode.Decompress);
34 | var uncompressed = new MemoryStream();
35 | lz2.CopyTo(uncompressed);
36 | var uncompressedBuffer = uncompressed.ToArray();
37 |
38 | var res = buffer.SequenceEqual(uncompressedBuffer);
39 | Assert.IsTrue(res, $"Buffer size {buffer.Length}");
40 | });
41 | }
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/tests/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.CompilerServices;
3 | using System.Runtime.InteropServices;
4 |
5 | // General Information about an assembly is controlled through the following
6 | // set of attributes. Change these attribute values to modify the information
7 | // associated with an assembly.
8 | [assembly: AssemblyTitle("tests")]
9 | [assembly: AssemblyDescription("")]
10 | [assembly: AssemblyConfiguration("")]
11 | [assembly: AssemblyCompany("NTent")]
12 | [assembly: AssemblyProduct("tests")]
13 | [assembly: AssemblyCopyright("Copyright © NTent 2014")]
14 | [assembly: AssemblyTrademark("")]
15 | [assembly: AssemblyCulture("")]
16 |
17 | // Setting ComVisible to false makes the types in this assembly not visible
18 | // to COM components. If you need to access a type in this assembly from
19 | // COM, set the ComVisible attribute to true on that type.
20 | [assembly: ComVisible(false)]
21 |
22 | // The following GUID is for the ID of the typelib if this project is exposed to COM
23 | [assembly: Guid("d9726911-6f07-4eb3-a862-48ba69c23a3a")]
24 |
25 | // Version information for an assembly consists of the following four values:
26 | //
27 | // Major Version
28 | // Minor Version
29 | // Build Number
30 | // Revision
31 | //
32 | // You can specify all the values or you can default the Build and Revision Numbers
33 | // by using the '*' as shown below:
34 | // [assembly: AssemblyVersion("1.0.*")]
35 | [assembly: AssemblyVersion("0.1.0.0")]
36 | [assembly: AssemblyFileVersion("0.1.0.0")]
37 |
--------------------------------------------------------------------------------
/tests/VagrantBrokerUtil.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Diagnostics;
4 | using System.IO;
5 | using System.Linq;
6 | using System.Net.Sockets;
7 | using System.Text;
8 | using System.Threading;
9 | using kafka4net;
10 | using kafka4net.Utils;
11 |
12 | namespace tests
13 | {
14 | internal static class VagrantBrokerUtil
15 | {
16 | static readonly NLog.Logger _log = NLog.LogManager.GetCurrentClassLogger();
17 |
18 | public static Dictionary BrokerIpToName = new Dictionary
19 | {
20 | {"192.168.56.10","broker1"},
21 | {"192.168.56.20","broker2"},
22 | {"192.168.56.30","broker3"},
23 | };
24 |
25 | static readonly string _kafkaVersion = File.ReadLines(
26 | Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..\\..\\..\\vagrant\\scripts\\kafka_version.txt")
27 | ).First();
28 |
29 | public static string GetBrokerNameFromIp(string ip)
30 | {
31 | if (!BrokerIpToName.ContainsKey(ip))
32 | throw new Exception("Unknown ip: " + ip);
33 | return BrokerIpToName[ip];
34 | }
35 |
36 |
37 | public static void CreateTopic(string topicName, int numPartitions, int replicationFactor)
38 | {
39 | _log.Info("Creating topic: '{0}'", topicName);
40 | string createTopicScript = "ssh -c '/opt/kafka_2.10-"+_kafkaVersion+"/bin/kafka-topics.sh --create --topic {0} --partitions {1} --replication-factor {2} --zookeeper 192.168.56.2' broker1";
41 | Vagrant(string.Format(createTopicScript, topicName, numPartitions, replicationFactor));
42 | _log.Info("Created topic: '{0}'", topicName);
43 | }
44 |
45 | public static void RebalanceLeadership()
46 | {
47 | _log.Info("Rebalancing Leadership");
48 | string rebalanceScript = "ssh -c '/opt/kafka_2.10-"+_kafkaVersion+"/bin/kafka-preferred-replica-election.sh --zookeeper 192.168.56.2' broker1";
49 | Vagrant(rebalanceScript);
50 | _log.Info("Rebalanced Leadership");
51 | }
52 |
53 | public static void DescribeTopic(string topic)
54 | {
55 | _log.Info("Getting topic description");
56 | string script = "ssh -c '/opt/kafka_2.10-" + _kafkaVersion + "/bin/kafka-topics.sh --zookeeper 192.168.56.2 --describe --topic " + topic + "' broker1";
57 | Vagrant(script);
58 | _log.Info("Got topic description");
59 | }
60 |
61 | public static void ReassignPartitions(Cluster cluster, string topic, int partition)
62 | {
63 | var brokerMeta = cluster.FindBrokerMetaForPartitionId(topic, partition);
64 | var brokerToMoveTo = brokerMeta.NodeId == 1 ? 2 : 1;
65 |
66 | var partitionsJson = string.Format("{{\"partitions\":[{{\"topic\":\"{0}\",\"partition\":{1},\"replicas\":[{2}]}}], \"version\":1}}",
67 | topic, partition, brokerToMoveTo);
68 |
69 | _log.Info(string.Format("Reassigning Partitions (topic {0}, partition {1}, from node {2} to node {3})", topic, partition, brokerMeta.NodeId, brokerToMoveTo));
70 |
71 | var generateJson = "ssh -c \"printf '" + partitionsJson.Replace("\"", @"\\\""") + "' >partitions-to-move.json\" broker1";
72 | Vagrant(generateJson);
73 |
74 | var reassignScript = "ssh -c '/opt/kafka_2.10-" + _kafkaVersion + "/bin/kafka-reassign-partitions.sh --zookeeper 192.168.56.2 --reassignment-json-file partitions-to-move.json --execute' broker1";
75 | Vagrant(reassignScript);
76 |
77 | _log.Info("Reassigned Partitions");
78 | }
79 |
80 | public static void RestartBrokers()
81 | {
82 | BrokerIpToName.Where(np=>!IsBrokerResponding(np.Key)).ForEach(np=>StartBroker(np.Value));
83 | }
84 |
85 | public static void StopBroker(string broker)
86 | {
87 | _log.Info("Stopping broker {0}", broker);
88 | Vagrant("ssh -c 'sudo service kafka stop' " + broker);
89 | _log.Info("Stopped broker {0}", broker);
90 | }
91 |
92 | public static string StopBrokerLeaderForPartition(Cluster cluster, string topic, int partition)
93 | {
94 | var brokerMeta = cluster.FindBrokerMetaForPartitionId(topic, partition);
95 | var brokerName = GetBrokerNameFromIp(brokerMeta.Host);
96 | StopBroker(brokerName);
97 | return brokerName;
98 | }
99 |
100 | public static void StartBroker(string broker)
101 | {
102 | var brokerIp = BrokerIpToName.Single(b => b.Value == broker).Key;
103 | if (IsBrokerResponding(brokerIp))
104 | {
105 | _log.Info("Broker already running: '{0}'", broker);
106 | return;
107 | }
108 |
109 | _log.Info("Starting broker: '{0}'", broker);
110 | Vagrant("ssh -c 'sudo service kafka start' " + broker);
111 |
112 | // await for tcp to become accessible, because process start does not mean that server has done initializing and start listening
113 | while(!IsBrokerResponding(brokerIp))
114 | Thread.Sleep(200);
115 |
116 | _log.Info("Started broker: '{0}'", broker);
117 | }
118 |
119 | private static bool IsBrokerResponding(string ip)
120 | {
121 | try
122 | {
123 | var client = new TcpClient();
124 | return client.ConnectAsync(ip, 9092).Wait(TimeSpan.FromSeconds(2));
125 | }
126 | catch
127 | {
128 | return false;
129 | }
130 | }
131 |
132 | private static string Vagrant(string script)
133 | {
134 | // TODO: implement vagrant-control
135 | var dir = AppDomain.CurrentDomain.BaseDirectory;
136 | dir = Path.Combine(dir, @"..\..\..\vagrant");
137 | dir = Path.GetFullPath(dir);
138 | var output = new StringBuilder();
139 | var pi = new ProcessStartInfo(@"vagrant.exe", script)
140 | {
141 | CreateNoWindow = true,
142 | RedirectStandardError = true,
143 | RedirectStandardOutput = true,
144 | UseShellExecute = false,
145 | WorkingDirectory = dir
146 | };
147 |
148 | var p = Process.Start(pi);
149 | p.OutputDataReceived += (sender, args) =>
150 | {
151 | output.Append(args.Data);
152 | _log.Info(args.Data);
153 | } ;
154 | p.ErrorDataReceived += (sender, args) =>
155 | {
156 | output.Append(args.Data);
157 | _log.Info(args.Data);
158 | };
159 | p.BeginOutputReadLine();
160 | p.BeginErrorReadLine();
161 | p.WaitForExit();
162 |
163 | if(p.ExitCode != 0)
164 | throw new Exception($"Vagrant failed with exit code {p.ExitCode}. Output: {output}");
165 |
166 | return output.ToString();
167 | }
168 |
169 | public static string GenerateMessagesWithJava(string codec, string topic)
170 | {
171 | return Vagrant($"ssh broker1 -c 'java -jar /vagrant/files/binary-console-all.jar {topic} {codec} produce'");
172 | }
173 | }
174 | }
175 |
--------------------------------------------------------------------------------
/tests/packages.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
--------------------------------------------------------------------------------
/tests/tests.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Debug
5 | AnyCPU
6 | {8BAF9CFE-8324-4604-BCA1-96925E8B82AC}
7 | Library
8 | Properties
9 | tests
10 | kafka4net-tests
11 | v4.5
12 | 512
13 | {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}
14 | 10.0
15 | $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)
16 | $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages
17 | False
18 | UnitTest
19 |
20 |
21 | true
22 | full
23 | false
24 | bin\Debug\
25 | DEBUG;TRACE
26 | prompt
27 | 4
28 |
29 |
30 | pdbonly
31 | true
32 | bin\Release\
33 | TRACE
34 | prompt
35 | 4
36 |
37 |
38 |
39 | ..\packages\log4net.2.0.5\lib\net45-full\log4net.dll
40 | True
41 |
42 |
43 | ..\packages\NLog.4.4.0\lib\net45\NLog.dll
44 | True
45 |
46 |
47 | ..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll
48 | True
49 |
50 |
51 |
52 | ..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll
53 |
54 |
55 | ..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll
56 |
57 |
58 | ..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll
59 |
60 |
61 | ..\packages\Rx-PlatformServices.2.2.5\lib\net45\System.Reactive.PlatformServices.dll
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 | {e3a39e66-db86-4e9c-b967-3aa71fa47c5d}
88 | kafka4net
89 |
90 |
91 |
92 |
93 |
94 |
95 | False
96 |
97 |
98 | False
99 |
100 |
101 | False
102 |
103 |
104 | False
105 |
106 |
107 |
108 |
109 |
110 |
111 |
118 |
--------------------------------------------------------------------------------
/tools/binary-console/build.gradle:
--------------------------------------------------------------------------------
1 | plugins {
2 | id 'scala'
3 | id 'java'
4 | id 'application'
5 | id 'com.github.johnrengelman.shadow' version '1.2.3'
6 | }
7 | /*apply plugin: 'scala'
8 | apply plugin: 'application'*/
9 |
10 | mainClassName = 'com.ntent.kafka.main'
11 |
12 | repositories{mavenCentral()}
13 |
14 | dependencies {
15 | compile "org.scala-lang:scala-library:2.10.6"
16 | compile 'org.apache.kafka:kafka_2.10:0.8.2.2'
17 | }
18 |
19 |
--------------------------------------------------------------------------------
/tools/binary-console/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | # Root logger option
2 | log4j.rootLogger=INFO, stdout
3 |
4 | # Direct log messages to stdout
5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
6 | log4j.appender.stdout.Target=System.out
7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
--------------------------------------------------------------------------------
/tools/binary-console/src/main/scala/com/ntent/kafka/main.scala:
--------------------------------------------------------------------------------
1 | package com.ntent.kafka
2 |
3 | import java.io._
4 | import java.security.MessageDigest
5 | import java.util.Properties
6 | import java.util.concurrent.atomic.AtomicInteger
7 | import java.util.concurrent.{Callable, Executors}
8 |
9 | import kafka.consumer.{Consumer, ConsumerConfig}
10 | import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
11 |
12 | import scala.io.Source
13 | import scala.util.Random
14 |
15 | /**
16 | * Created by vchekan on 4/29/2016.
17 | */
18 | object main extends App {
19 | val topic = args(0)
20 | val compression = args(1)
21 | val action = args(2)
22 | val md5 = MessageDigest.getInstance("MD5")
23 |
24 | action match {
25 | case "produce" => produce()
26 | case "consume" => consume()
27 | case _ => throw new RuntimeException("Unknown action. Should be 'produce' or 'consume'")
28 | }
29 |
30 | def produce() = {
31 | val hashFileName = "/vagrant/files/hashes.txt"
32 | val sizesFileName = "/vagrant/files/sizes.txt"
33 |
34 | val hashFile = new PrintWriter(new File(hashFileName))
35 | val sizes = readSizes(sizesFileName)
36 |
37 | val producer = new KafkaProducer[Array[Byte], Array[Byte]](producerProps)
38 | println(s"Read ${sizes.length} lines")
39 | val futures = sizes.map(size => {
40 | producer.send(generateMessage(size, hashFile))
41 | })
42 |
43 | producer.close()
44 |
45 | futures.foreach(f => {
46 | val res = f.get()
47 | println(s"Sent part:${res.partition()} offset: ${res.offset()}")
48 | })
49 |
50 |
51 | hashFile.close()
52 | }
53 |
54 | def consume(): Unit = {
55 | val hashFileName = "C:\\projects\\kafka4net\\vagrant\\files\\hashes.txt"
56 | val sizesFileName = "C:\\projects\\kafka4net\\vagrant\\files\\sizes.txt"
57 |
58 | val hashFile = new PrintWriter(new File(hashFileName))
59 | val consumer = Consumer.create(new ConsumerConfig(consumerProps))
60 | val consumerMap = consumer.createMessageStreams(Map(topic -> 3))
61 | val streams = consumerMap.get(topic).get
62 | val expectedCount = readSizes(sizesFileName).length
63 | val count = new AtomicInteger()
64 |
65 | val threads = for(stream <- streams) yield {
66 | new Thread(new Runnable {
67 | override def run(): Unit = {
68 | try {
69 | println(s"Starting stream ${stream}")
70 | val it = stream.iterator()
71 | while (it.hasNext()) {
72 | try {
73 | val msg = it.next()
74 | val value = msg.message()
75 | count.incrementAndGet()
76 | //println(s"Got ${count.get()}/$expectedCount size: ${value.length} part: ${msg.partition} offset: ${msg.offset}")
77 | val hash = md5.digest(value).map("%02X".format(_)).mkString
78 | hashFile.println(hash)
79 | if (count.get() == expectedCount) {
80 | consumer.shutdown()
81 | }
82 | } catch {
83 | case e: Throwable => println(e.getMessage)
84 | }
85 | }
86 | } catch {
87 | case e: Throwable => println(s"Error: ${e.getMessage}")
88 | }
89 | }
90 | })
91 | }
92 |
93 | threads.foreach(_.start())
94 |
95 | threads.foreach(_.join(5*60*1000))
96 |
97 | hashFile.close()
98 | }
99 |
100 | def generateMessage(size: Int, hashFile: PrintWriter): ProducerRecord[Array[Byte], Array[Byte]] = {
101 | val buff = new Array[Byte](size)
102 | Random.nextBytes(buff)
103 | val hash = md5.digest(buff).map("%02X".format(_)).mkString
104 | hashFile.println(hash)
105 |
106 | // put all messages into the same partition
107 | val key = Array[Byte](0)
108 | println(hash)
109 | new ProducerRecord(topic, key, buff)
110 | }
111 |
112 | def producerProps = {
113 | val props = new Properties()
114 | props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.56.10:9092")
115 | props.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, compression)
116 | props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "10000")
117 | props.setProperty(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG, "true")
118 | props.setProperty(ProducerConfig.BUFFER_MEMORY_CONFIG, (11*1024*1024).toString) // 11Mb
119 | props.setProperty(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, (8*1000*1000).toString)
120 | props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, (1024*1024).toString)
121 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer")
122 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer")
123 | props
124 | }
125 |
126 | def consumerProps: Properties = {
127 | val props = new Properties()
128 |
129 | //props.put("bootstrap.servers", "192.168.56.10:9092")
130 | props.setProperty("zookeeper.connect", "192.168.56.2:2181")
131 | props.setProperty("group.id", "kafka4net-testing")
132 | //props.setProperty("auto.offset.reset", "largest")
133 | props.setProperty("fetch.message.max.bytes", (500*1024*1024).toString)
134 | props.put("auto.commit.enable", "false")
135 | props
136 | }
137 |
138 | def readSizes(sizesFileName: String) = {
139 | Source.fromFile(sizesFileName).getLines().map(Integer.parseInt(_)).toArray
140 | }
141 | }
142 |
--------------------------------------------------------------------------------
/vagrant/.gitignore:
--------------------------------------------------------------------------------
1 | /.vagrant/
2 | /rpm/
3 |
--------------------------------------------------------------------------------
/vagrant/.hgignore:
--------------------------------------------------------------------------------
1 | syntax: glob
2 | .git
3 | .gitignore
4 | .vagrant
5 | files/
6 |
--------------------------------------------------------------------------------
/vagrant/Vagrantfile:
--------------------------------------------------------------------------------
1 | # -*- mode: ruby -*-
2 | # vi: set ft=ruby :
3 |
4 | VAGRANTFILE_API_VERSION = "2"
5 |
6 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
7 | k_count=3
8 |
9 | config.vm.box = "ubuntu/trusty64"
10 | config.vm.box_check_update = false
11 |
12 | # zookeeper
13 | config.vm.define "zookeeper1" do |s|
14 | s.vm.hostname = "zookeeper1"
15 |
16 | s.vm.network :private_network, ip: "192.168.56.2"
17 | # zookeeper1 is cache for packages
18 | s.vm.provision "shell", inline: "apt-get update"
19 | s.vm.provision "shell", inline: "apt-get install -y squid-deb-proxy avahi-utils"
20 | s.vm.provision "shell", inline: "apt-get install -y squid-deb-proxy-client"
21 | s.vm.provision "shell", path: "scripts/init.sh"
22 | s.vm.provision "shell", path: "scripts/zookeeper.sh"
23 | end
24 |
25 | # configure brokers
26 | (1..k_count).each do |i|
27 | config.vm.define "broker#{i}" do |s|
28 | s.vm.hostname = "broker#{i}"
29 | #s.vm.network "private_network", ip: "10.30.3.#{4-i}0", netmask: "255.255.255.0", virtualbox__intnet: "servidors", drop_nat_interface_default_route: true
30 | s.vm.network :private_network, ip: "192.168.56.#{i}0"
31 | s.vm.provision "shell", inline: "apt-get update"
32 | s.vm.provision "shell", inline: "apt-get install -y squid-deb-proxy-client"
33 | s.vm.provision "shell", path: "scripts/init.sh"
34 | s.vm.provision "shell", path: "scripts/broker.sh", args:"#{i}"
35 | end
36 | end
37 |
38 | config.vm.provider "virtualbox" do |v|
39 | #v.gui = true
40 | v.customize ["modifyvm", :id, "--cpuexecutioncap", "50", "--memory", "3072"]
41 | end
42 | end
43 |
44 |
--------------------------------------------------------------------------------
/vagrant/config/kafka.conf:
--------------------------------------------------------------------------------
1 | description "Apache Kafka server"
2 | author "Vadim Chekan "
3 |
4 | start on runlevel [2345]
5 | stop on runlevel [016]
6 | kill signal SIGINT
7 | exec /bin/bash /opt/kafka_2.10-kafka_version/bin/kafka-server-start.sh /opt/kafka_2.10-kafka_version/config/server.properties > /tmp/kafka.log 2>&1
--------------------------------------------------------------------------------
/vagrant/config/zookeeper.conf:
--------------------------------------------------------------------------------
1 | description "Apache Zookeeper server"
2 | author "Vadim Chekan "
3 |
4 | start on runlevel [2345]
5 | stop on runlevel [016]
6 | kill signal SIGINT
7 | exec /bin/bash /opt/kafka_2.10-kafka_version/bin/zookeeper-server-start.sh /opt/kafka_2.10-kafka_version/config/zookeeper.properties > /tmp/zookeeper.log 2>&1
--------------------------------------------------------------------------------
/vagrant/config/zookeeper.properties:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | # the directory where the snapshot is stored.
16 | dataDir=/tmp/zookeeper
17 | # the port at which the clients will connect
18 | clientPort=2181
19 | # clientPortAddress=192.168.56.2
20 | # disable the per-ip limit on the number of connections since this is a non-production config
21 | maxClientCnxns=0
22 | tickTime=2000
23 | initLimit=5
24 | syncLimit=2
25 |
--------------------------------------------------------------------------------
/vagrant/log4j.properties:
--------------------------------------------------------------------------------
1 | # Root logger option
2 | log4j.rootLogger=INFO, stdout
3 |
4 | # Direct log messages to stdout
5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
6 | log4j.appender.stdout.Target=System.out
7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
--------------------------------------------------------------------------------
/vagrant/scripts/broker.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | id=$1
4 | . /vagrant/scripts/env.sh
5 |
6 | sed -i -e "s/^broker\.id=.*/broker.id=$id/" "/opt/kafka_2.10-${kafka_version}/config/server.properties"
7 | sed -i -e "s|^#*listeners=.*|listeners=PLAINTEXT://192.168.56.${id}0:9092|" "/opt/kafka_2.10-${kafka_version}/config/server.properties"
8 | #sed -i -e "s/^log.segment.bytes=.*/log.segment.bytes=1000048576/" "/opt/kafka_2.10-${kafka_version}/config/server.properties"
9 | sed -i -e "s/^#*num\.io\.threads=.*/num.io.threads=2/" "/opt/kafka_2.10-${kafka_version}/config/server.properties"
10 | sed -i -e 's/^num\.partitions=.*/num\.partitions=6\
11 | default.replication.factor=3/' "/opt/kafka_2.10-${kafka_version}/config/server.properties"
12 | sed -i -e "s/^#*zookeeper\.connect=.*/zookeeper.connect=192.168.56.2:2181/" \
13 | "/opt/kafka_2.10-${kafka_version}/config/server.properties"
14 | sed -i -e 's/-Xmx1G/-Xmx1G/' "/opt/kafka_2.10-${kafka_version}/bin/kafka-server-start.sh"
15 | sed -i -e 's/-Xms1G/-Xms1G/' "/opt/kafka_2.10-${kafka_version}/bin/kafka-server-start.sh"
16 |
17 | echo 'message.max.bytes=50000000' >> "/opt/kafka_2.10-${kafka_version}/config/server.properties"
18 | echo 'replica.fetch.max.bytes=50000000' >> "/opt/kafka_2.10-${kafka_version}/config/server.properties"
19 |
20 | cp /vagrant/config/kafka.conf /etc/init/
21 | chmod a-x /etc/init/kafka.conf
22 | sed -i -e "s/kafka_version/${kafka_version}/g" "/etc/init/kafka.conf"
23 |
24 | service kafka start
25 |
--------------------------------------------------------------------------------
/vagrant/scripts/env.sh:
--------------------------------------------------------------------------------
1 | kafka_version=`cat /vagrant/scripts/kafka_version.txt`
2 | kafka="kafka_2.10-${kafka_version}"
3 | kafka_bin="$kafka.tgz"
4 | #kafka_url="http://www.eng.lsu.edu/mirrors/apache/kafka/${kafka_version}/${kafka_bin}"
5 | #kafka_url="https://people.apache.org/~junrao/kafka-${kafka_version}-candidate1/${kafka_bin}"
6 | kafka_url="http://apache.mirrors.lucidnetworks.net/kafka/${kafka_version}/${kafka_bin}"
7 |
--------------------------------------------------------------------------------
/vagrant/scripts/init.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Executed as root
4 |
5 | . /vagrant/scripts/env.sh
6 |
7 | apt-get install -y openjdk-7-jre-headless
8 |
9 | if [ ! -d "/opt/$kafka" ]; then
10 | mkdir -p /vagrant/files
11 | cd /vagrant/files
12 | if [ ! -f "/vagrant/files/$kafka_bin" ]; then
13 | wget $kafka_url
14 | fi
15 | tar -C /opt -xzf $kafka_bin
16 | fi
17 |
--------------------------------------------------------------------------------
/vagrant/scripts/kafka_version.txt:
--------------------------------------------------------------------------------
1 | 0.10.1.1
--------------------------------------------------------------------------------
/vagrant/scripts/zookeeper.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | . /vagrant/scripts/env.sh
4 |
5 | mkdir -p /var/zookeeper
6 | mkdir -p /opt/kafka_2.10-${kafka_version}/logs
7 | sed -i -e 's/-Xmx512M/-Xmx1024M/' "/opt/kafka_2.10-${kafka_version}/bin/zookeeper-server-start.sh"
8 | sed -i -e 's/-Xms512M/-Xms1024M/' "/opt/kafka_2.10-${kafka_version}/bin/zookeeper-server-start.sh"
9 |
10 | cp /vagrant/config/zookeeper.properties "/opt/kafka_2.10-${kafka_version}/config/"
11 | chmod a-x "/opt/kafka_2.10-${kafka_version}/config/zookeeper.properties"
12 |
13 | cp /vagrant/config/zookeeper.conf /etc/init/
14 | chmod a-x /etc/init/zookeeper.conf
15 | sed -i -e "s/kafka_version/${kafka_version}/g" "/etc/init/zookeeper.conf"
16 |
17 | service zookeeper start
18 |
19 |
--------------------------------------------------------------------------------
/vagrant/server.properties:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | # see kafka.server.KafkaConfig for additional details and defaults
16 |
17 | ############################# Server Basics #############################
18 |
19 | # The id of the broker. This must be set to a unique integer for each broker.
20 | broker.id=0
21 |
22 | ############################# Socket Server Settings #############################
23 |
24 | # The port the socket server listens on
25 | port=9092
26 |
27 | # Hostname the broker will bind to. If not set, the server will bind to all interfaces
28 | #host.name=localhost
29 |
30 | # Hostname the broker will advertise to producers and consumers. If not set, it uses the
31 | # value for "host.name" if configured. Otherwise, it will use the value returned from
32 | # java.net.InetAddress.getCanonicalHostName().
33 | #advertised.host.name=
34 |
35 | # The port to publish to ZooKeeper for clients to use. If this is not set,
36 | # it will publish the same port that the broker binds to.
37 | #advertised.port=
38 |
39 | # The number of threads handling network requests
40 | num.network.threads=3
41 |
42 | # The number of threads doing disk I/O
43 | num.io.threads=8
44 |
45 | # The send buffer (SO_SNDBUF) used by the socket server
46 | socket.send.buffer.bytes=102400
47 |
48 | # The receive buffer (SO_RCVBUF) used by the socket server
49 | socket.receive.buffer.bytes=102400
50 |
51 | # The maximum size of a request that the socket server will accept (protection against OOM)
52 | socket.request.max.bytes=104857600
53 |
54 |
55 | ############################# Log Basics #############################
56 |
57 | # A comma seperated list of directories under which to store log files
58 | log.dirs=/tmp/kafka-logs
59 |
60 | # The default number of log partitions per topic. More partitions allow greater
61 | # parallelism for consumption, but this will also result in more files across
62 | # the brokers.
63 | num.partitions=1
64 |
65 | # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
66 | # This value is recommended to be increased for installations with data dirs located in RAID array.
67 | num.recovery.threads.per.data.dir=1
68 |
69 | ############################# Log Flush Policy #############################
70 |
71 | # Messages are immediately written to the filesystem but by default we only fsync() to sync
72 | # the OS cache lazily. The following configurations control the flush of data to disk.
73 | # There are a few important trade-offs here:
74 | # 1. Durability: Unflushed data may be lost if you are not using replication.
75 | # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
76 | # 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
77 | # The settings below allow one to configure the flush policy to flush data after a period of time or
78 | # every N messages (or both). This can be done globally and overridden on a per-topic basis.
79 |
80 | # The number of messages to accept before forcing a flush of data to disk
81 | #log.flush.interval.messages=10000
82 |
83 | # The maximum amount of time a message can sit in a log before we force a flush
84 | #log.flush.interval.ms=1000
85 |
86 | ############################# Log Retention Policy #############################
87 |
88 | # The following configurations control the disposal of log segments. The policy can
89 | # be set to delete segments after a period of time, or after a given size has accumulated.
90 | # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
91 | # from the end of the log.
92 |
93 | # The minimum age of a log file to be eligible for deletion
94 | log.retention.hours=168
95 |
96 | # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
97 | # segments don't drop below log.retention.bytes.
98 | #log.retention.bytes=1073741824
99 |
100 | # The maximum size of a log segment file. When this size is reached a new log segment will be created.
101 | log.segment.bytes=1073741824
102 |
103 | # The interval at which log segments are checked to see if they can be deleted according
104 | # to the retention policies
105 | log.retention.check.interval.ms=300000
106 |
107 | # By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires.
108 | # If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction.
109 | log.cleaner.enable=false
110 |
111 | ############################# Zookeeper #############################
112 |
113 | # Zookeeper connection string (see zookeeper docs for details).
114 | # This is a comma separated host:port pairs, each corresponding to a zk
115 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
116 | # You can also append an optional chroot string to the urls to specify the
117 | # root directory for all kafka znodes.
118 | zookeeper.connect=192.168.56.2:2181/kafka4nettest
119 |
120 | # Timeout in ms for connecting to zookeeper
121 | zookeeper.connection.timeout.ms=6000
122 |
123 |
124 | message.max.bytes=50000000
125 | replica.fetch.max.bytes=50000000
126 |
--------------------------------------------------------------------------------