├── examples ├── consumer │ ├── .gitignore │ ├── dub.json │ └── source │ │ └── app.d ├── httplog │ ├── .gitignore │ ├── dub.json │ └── source │ │ └── app.d ├── producer │ ├── .gitignore │ ├── dub.json │ └── source │ │ └── app.d ├── consumer_vibed │ ├── .gitignore │ ├── global.conf │ ├── dub.json │ └── source │ │ └── app.d └── producer_vibed │ ├── .gitignore │ ├── global.conf │ ├── dub.json │ └── source │ └── app.d ├── README.md ├── dub.json ├── .gitignore └── source └── rdkafkad ├── metadata.d ├── topic.d ├── handler ├── producer.d ├── simple_consumer.d ├── kafka_consumer.d └── package.d ├── config.d └── package.d /examples/consumer/.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | docs.json 3 | __dummy.html 4 | *.o 5 | *.obj 6 | -------------------------------------------------------------------------------- /examples/httplog/.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | docs.json 3 | __dummy.html 4 | *.o 5 | *.obj 6 | -------------------------------------------------------------------------------- /examples/producer/.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | docs.json 3 | __dummy.html 4 | *.o 5 | *.obj 6 | -------------------------------------------------------------------------------- /examples/consumer_vibed/.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | docs.json 3 | __dummy.html 4 | *.o 5 | *.obj 6 | -------------------------------------------------------------------------------- /examples/producer_vibed/.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | docs.json 3 | __dummy.html 4 | *.o 5 | *.obj 6 | -------------------------------------------------------------------------------- /examples/producer_vibed/global.conf: -------------------------------------------------------------------------------- 1 | # Global configuration 2 | metadata.broker.list = localhost 3 | group.id = rdkafkad 4 | -------------------------------------------------------------------------------- /examples/consumer_vibed/global.conf: -------------------------------------------------------------------------------- 1 | # Global configuration 2 | metadata.broker.list = localhost 3 | group.id = kafka2s3 4 | api.version.request = true 5 | enable.auto.commit = false 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # librdkafka-d 2 | 3 | Object oriented wrapper on top of [librdkafka](https://github.com/edenhill/librdkafka). 4 | 5 | #### vibe.d 6 | The wrapper does not depend on vibe.d but supports its event based engine. 7 | -------------------------------------------------------------------------------- /examples/producer/dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "producer", 3 | "authors": [ 4 | "Ilya Yaroshenko" 5 | ], 6 | "description": "Producer example", 7 | "copyright": "Copyright © 2016, Tamedia AG", 8 | "license": "BSL-1.0", 9 | "dependencies":{ 10 | "librdkafka-d": {"path" : "../../"} 11 | } 12 | } -------------------------------------------------------------------------------- /examples/consumer/dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "consumer", 3 | "authors": [ 4 | "Ilya Yaroshenko" 5 | ], 6 | "description": "KafkaConsumer example", 7 | "copyright": "Copyright © 2016, Tamedia AG", 8 | "license": "BSL-1.0", 9 | "dependencies":{ 10 | "librdkafka-d": {"path" : "../../"} 11 | } 12 | } -------------------------------------------------------------------------------- /dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "authors": [ 3 | "Ilya Yaroshenko (Converted from the CPP wrapper)" 4 | ], 5 | "name": "librdkafka-d", 6 | "license": "BSL-1.0", 7 | "copyright": "Copyright © 2016, Tamedia AG", 8 | "description": "OOP wrapper for librdkafka", 9 | "dependencies": { 10 | "librdkafka": "~>0.3.1" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /examples/producer_vibed/dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "producer_vibed", 3 | "authors": [ 4 | "Ilya Yaroshenko" 5 | ], 6 | "description": "Producer example", 7 | "copyright": "Copyright © 2016, Tamedia AG", 8 | "license": "BSL-1.0", 9 | "dependencies":{ 10 | "vibe-d": "~>0.7.30", 11 | "librdkafka-d": {"path" : "../../"}, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /examples/httplog/dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "httplog", 3 | "authors": [ 4 | "Ilya Yaroshenko" 5 | ], 6 | "description": "HTTP log example", 7 | "copyright": "Copyright © 2016, Tamedia AG", 8 | "license": "BSL-1.0", 9 | "dependencies":{ 10 | "vibe-d": "~>0.7.30", 11 | "librdkafka-d": {"path" : "../../"} 12 | }, 13 | "versions": ["VibeDefaultMain"] 14 | } 15 | -------------------------------------------------------------------------------- /examples/consumer_vibed/dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "consumer_vibed", 3 | "authors": [ 4 | "Ilya Yaroshenko" 5 | ], 6 | "description": "KafkaConsumer example", 7 | "copyright": "Copyright © 2016, Tamedia AG", 8 | "license": "BSL-1.0", 9 | "dependencies":{ 10 | "vibe-d": "~>0.8.0-beta.5", 11 | "librdkafka-d": {"path" : "../../"} 12 | }, 13 | "subConfigurations": { 14 | "vibe-d": "vibe-core" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .dub 2 | docs.json 3 | __dummy.html 4 | *.o 5 | *.obj 6 | 7 | dub.selections.json 8 | 9 | *.a 10 | 11 | *.sublime-project 12 | 13 | librdkafka-d-test-library 14 | 15 | examples/httplog/httplog 16 | 17 | examples/producer/producer 18 | 19 | librdkafka-d-test-default 20 | 21 | untitled.sublime-workspace 22 | 23 | examples/producer_vibed/producer_vibed 24 | 25 | examples/consumer_vibed/consumer_vibed 26 | -------------------------------------------------------------------------------- /examples/consumer/source/app.d: -------------------------------------------------------------------------------- 1 | import std.stdio; 2 | import rdkafkad; 3 | 4 | void main() 5 | { 6 | /// @@@@@ Configuration 7 | // pointer to the conf should be preserved for because delegates are used (for closures). 8 | auto conf = new GlobalConf; 9 | KafkaConsumer consumer; 10 | try 11 | { 12 | conf["metadata.broker.list"] = "localhost"; 13 | conf["group.id"] = "rdkafkad"; 14 | consumer = new KafkaConsumer(conf); 15 | } 16 | catch(Exception e) 17 | { 18 | stderr.writeln(e.msg); 19 | return; 20 | } 21 | consumer.subscribe("httplog_topic", /+...+/); 22 | for (size_t c;;) 23 | { 24 | if(++c % 100 == 0) // use large number for fast I/O! 25 | consumer.commitSync(); 26 | Message msg; 27 | consumer.consume(msg, 6000); 28 | if(auto error = msg.err) 29 | { 30 | writeln("Error: ", error.err2str); 31 | continue; 32 | } 33 | with(msg) writefln("Topic %s[%d]->%d, key: %s", 34 | topicName, partition, offset, cast(const(char)[])key); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /examples/producer/source/app.d: -------------------------------------------------------------------------------- 1 | import std.stdio; 2 | import std.exception; 3 | import std.algorithm; 4 | import std.datetime; 5 | import rdkafkad; 6 | import core.thread; 7 | 8 | __gshared continuePoll = true; 9 | 10 | void main() 11 | { 12 | /// @@@@@ Configuration 13 | // pointer to the conf should be preserved for because delegates are used (for closures). 14 | auto conf = new GlobalConf; 15 | Producer producer; 16 | try 17 | { 18 | conf["metadata.broker.list"] = "localhost"; 19 | producer = new Producer(conf); 20 | } 21 | catch(Exception e) 22 | { 23 | stderr.writeln(e.msg); 24 | return; 25 | } 26 | auto topics = [ 27 | producer.newTopic("httplog_topic3"), 28 | ]; 29 | auto time = Clock.currTime(UTC()); 30 | /// @@@@@ Main loop :-) 31 | auto handler = new Thread({ while(continuePoll) producer.poll(10);}).start; 32 | for (size_t c;;) foreach(topic; topics) 33 | { 34 | import std.format; 35 | string key = "myKey"; 36 | string payload = format(`{"ts":%s, "myValue":%d}`, (time + minutes(c)).toUnixTime, c++); // use large payload for I/O benchmarks 37 | if(auto error = producer.produce(topic, Topic.unassignedPartition, cast(void[])payload, cast(const(void)[])key)) 38 | { 39 | if(error == ErrorCode.queue_full) 40 | { 41 | writeln(error.err2str); 42 | Thread.sleep(10.msecs); 43 | continue; 44 | } 45 | stderr.writeln(error.err2str); 46 | continuePoll = false; 47 | thread_joinAll; 48 | return; 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /examples/producer_vibed/source/app.d: -------------------------------------------------------------------------------- 1 | import std.stdio; 2 | import std.exception; 3 | import std.algorithm; 4 | import std.file; 5 | 6 | import rdkafkad; 7 | import vibe.d; 8 | 9 | __gshared continuePoll = true; 10 | 11 | void main() 12 | { 13 | /// @@@@@ Configuration 14 | // pointer to the conf should be preserved for because delegates are used (for closures). 15 | auto conf = new GlobalConf; 16 | Producer producer; 17 | try 18 | { 19 | conf.fromText("global.conf".readText); 20 | producer = new Producer(conf); 21 | } 22 | catch(Exception e) 23 | { 24 | stderr.writeln(e.msg); 25 | return; 26 | } 27 | auto topics = [ 28 | producer.newTopic("httplog_topic"), 29 | ]; 30 | 31 | auto condition = new TaskCondition(new TaskMutex); 32 | /// @@@@@ Main loop :-) 33 | runTask({ 34 | while(continuePoll) 35 | { 36 | producer.poll(); 37 | } 38 | condition.notify; 39 | }); 40 | for (size_t c;;) foreach(topic; topics) 41 | { 42 | import std.format; 43 | string key = "myKey"; 44 | string payload = format("myValue %d", c++); // use large payload for I/O benchmarks 45 | sleep(100.msecs); // comment for I/O benchmarks 46 | if(auto error = producer.produce(topic, Topic.unassignedPartition, cast(void[])payload, cast(const(void)[])key)) 47 | { 48 | if(error == ErrorCode.queue_full) 49 | { 50 | writeln(error.err2str); 51 | sleep(10.msecs); 52 | continue; 53 | } 54 | stderr.writeln(error.err2str); 55 | continuePoll = false; 56 | synchronized(condition.mutex) 57 | condition.wait; 58 | return; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /examples/httplog/source/app.d: -------------------------------------------------------------------------------- 1 | import std.stdio; 2 | import rdkafkad; 3 | import vibe.d; 4 | 5 | __gshared Task producerTask; 6 | __gshared continuePoll = true; 7 | // pointer to the conf should be preserved for because delegates are used (for closures). 8 | __gshared GlobalConf conf; 9 | __gshared Producer producer; 10 | 11 | shared static this() 12 | { 13 | debug setLogLevel(LogLevel.debug_); 14 | 15 | auto settings = new HTTPServerSettings; 16 | settings.port = 8080; 17 | 18 | listenHTTP(settings, &handleRequest); 19 | conf = new GlobalConf; 20 | try 21 | { 22 | conf["metadata.broker.list"] = "localhost"; 23 | conf["group.id"] = "rdkafkad"; 24 | producer = new Producer(conf); 25 | } 26 | catch(Exception e) 27 | { 28 | stderr.writeln(e.msg); 29 | return; 30 | } 31 | auto topic = producer.newTopic("httplog_topic"); 32 | /// @@@@@ Main loop 33 | runWorkerTask({ while(continuePoll) producer.poll(10);}); 34 | producerTask = runTask({for (;;){ 35 | receive((string msg) { 36 | if(auto error = producer.produce(topic, Topic.unassignedPartition, cast(void[])msg)) 37 | { 38 | if(error == ErrorCode.queue_full) 39 | { 40 | logInfo(error.err2str); 41 | vibe.core.core.yield; 42 | } 43 | stderr.writeln(error.err2str); 44 | continuePoll = false; 45 | exitEventLoop(true); 46 | } 47 | }); 48 | }}); 49 | 50 | setMaxMailboxSize(producerTask.tid, 100, OnCrowding.throwException); 51 | } 52 | 53 | void handleRequest(HTTPServerRequest req, 54 | HTTPServerResponse res) 55 | { 56 | if (req.path == "/") 57 | res.writeBody("Hello, World!", "text/plain"); 58 | 59 | try { 60 | send(producerTask, req.path); 61 | } catch (MailboxFull) { 62 | // message queue is full, this may indicate that producer is blocking, no connection, etc. 63 | logInfo("Couldn't log to kafka"); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /examples/consumer_vibed/source/app.d: -------------------------------------------------------------------------------- 1 | import std.file; 2 | 3 | import vibe.d; 4 | import vibe.stream.stdio; 5 | import rdkafkad; 6 | import std.datetime: SysTime; 7 | 8 | void hello(HTTPServerRequest req, HTTPServerResponse res) 9 | { 10 | res.writeBody("Hello, World!"); 11 | } 12 | 13 | void main() 14 | { 15 | auto stdout = new StdoutStream; 16 | /// @@@@@ Configuration 17 | // pointer to the conf should be preserved for because delegates are used (for closures). 18 | auto tconf = new TopicConf; 19 | tconf["auto.offset.reset"] = "earliest"; 20 | auto conf = new GlobalConf(tconf); 21 | KafkaConsumer consumer; 22 | try 23 | { 24 | conf.fromText("global.conf".readText); 25 | consumer = new KafkaConsumer(conf); 26 | } 27 | catch(Exception e) 28 | { 29 | import std.stdio; 30 | stderr.writeln(e.msg); 31 | return; 32 | } 33 | consumer.subscribe("bigdata", /+...+/); 34 | import vibe.stream.taskpipe; 35 | auto stream = new TaskPipe; 36 | ubyte[64*1024] buffer = void; 37 | runTask({ 38 | import std.stdio; 39 | auto fout = File("temp2", "w"); 40 | while (!stream.empty) { 41 | import std.algorithm.comparison; 42 | size_t chunk = min(stream.leastSize, buffer.length); 43 | assert(chunk > 0, "leastSize returned zero for non-empty stream."); 44 | //logTrace("read pipe chunk %d", chunk); 45 | stream.read(buffer[0 .. chunk]); 46 | 47 | fout.writeln(cast(string) buffer[0 .. chunk]); 48 | } 49 | }); 50 | runTask({ 51 | for (size_t c;;) 52 | { 53 | Message msg; 54 | consumer.consume(msg, 0); 55 | if(auto error = msg.err) 56 | { 57 | if (msg.err == ErrorCode.timed_out || msg.err == ErrorCode.msg_timed_out) 58 | { 59 | sleep(1.msecs); 60 | } 61 | else if (msg.err == ErrorCode.partition_eof) 62 | { 63 | exitEventLoop(true); 64 | break; 65 | } 66 | else 67 | { 68 | logError("Error: ", error.err2str); 69 | vibe.core.core.yield; 70 | } 71 | continue; 72 | } 73 | stream.write(cast(const(ubyte[])) msg.payload, IOMode.all); /// vibe streams :-) 74 | stream.write(cast(const(ubyte[])) "\n", IOMode.all); 75 | } 76 | stream.finalize; 77 | }); 78 | 79 | auto settings = new HTTPServerSettings; 80 | settings.port = 8080; 81 | settings.bindAddresses = ["::1", "127.0.0.1"]; 82 | listenHTTP(settings, &hello); 83 | 84 | runEventLoop; 85 | } 86 | -------------------------------------------------------------------------------- /source/rdkafkad/metadata.d: -------------------------------------------------------------------------------- 1 | /// 2 | module rdkafkad.metadata; 3 | import rdkafkad; 4 | 5 | /** 6 | * Metadata: Broker information 7 | */ 8 | struct BrokerMetadata 9 | { 10 | private const(rd_kafka_metadata_broker_t)* broker_metadata_; 11 | 12 | const @property nothrow @nogc: 13 | 14 | /** Broker id */ 15 | int id() const 16 | { 17 | return broker_metadata_.id; 18 | } 19 | 20 | /** Broker hostname */ 21 | const(char)[] host() const 22 | { 23 | return broker_metadata_.host.fromStringz; 24 | } 25 | 26 | /** Broker listening port */ 27 | int port() const 28 | { 29 | return broker_metadata_.port; 30 | } 31 | } 32 | 33 | /** 34 | * Metadata: Partition information 35 | */ 36 | struct PartitionMetadata 37 | { 38 | private const (rd_kafka_metadata_partition_t)* partition_metadata_; 39 | 40 | const @property nothrow @nogc: 41 | 42 | /** Partition id */ 43 | int id() 44 | { 45 | return partition_metadata_.id; 46 | } 47 | 48 | /** Partition error reported by broker */ 49 | ErrorCode err() 50 | { 51 | return cast(ErrorCode) partition_metadata_.err; 52 | } 53 | 54 | /** Leader broker (id) for partition */ 55 | int leader() 56 | { 57 | return partition_metadata_.leader; 58 | } 59 | 60 | /** Replica brokers */ 61 | auto replicas() 62 | { 63 | static struct Replicas 64 | { 65 | nothrow @nogc: 66 | private const(rd_kafka_metadata_partition_t)* partition_metadata_; 67 | private size_t _i; 68 | auto empty() @property 69 | { 70 | return _i >= partition_metadata_.replica_cnt; 71 | } 72 | 73 | auto front() @property 74 | { 75 | return partition_metadata_.replicas[_i]; 76 | } 77 | 78 | auto popFront() 79 | { 80 | _i++; 81 | } 82 | 83 | auto length() @property 84 | { 85 | return partition_metadata_.replica_cnt - _i; 86 | } 87 | } 88 | 89 | return Replicas(partition_metadata_); 90 | } 91 | 92 | /** In-Sync-Replica brokers 93 | * Warning: The broker may return a cached/outdated list of ISRs. 94 | */ 95 | auto isrs() 96 | { 97 | static struct Isrs 98 | { 99 | nothrow @nogc: 100 | private const(rd_kafka_metadata_partition_t)* partition_metadata_; 101 | private size_t _i; 102 | auto empty() @property 103 | { 104 | return _i >= partition_metadata_.isr_cnt; 105 | } 106 | 107 | auto front() @property 108 | { 109 | return partition_metadata_.isrs[_i]; 110 | } 111 | 112 | auto popFront() 113 | { 114 | _i++; 115 | } 116 | 117 | auto length() @property 118 | { 119 | return partition_metadata_.isr_cnt - _i; 120 | } 121 | } 122 | 123 | return Isrs(partition_metadata_); 124 | } 125 | } 126 | 127 | /** 128 | * Metadata: Topic information 129 | */ 130 | struct TopicMetadata 131 | { 132 | private const(rd_kafka_metadata_topic_t)* topic_metadata_; 133 | 134 | const @property nothrow @nogc: 135 | 136 | /** Topic name */ 137 | const(char)[] topic() 138 | { 139 | return topic_metadata_.topic.fromStringz; 140 | } 141 | 142 | /** Partition list */ 143 | auto partitions() 144 | { 145 | static struct Partitions 146 | { 147 | nothrow @nogc: 148 | private const(rd_kafka_metadata_topic_t)* topic_metadata_; 149 | private size_t _i; 150 | auto empty() @property 151 | { 152 | return _i >= topic_metadata_.partition_cnt; 153 | } 154 | 155 | auto front() @property 156 | { 157 | return PartitionMetadata(&topic_metadata_.partitions[_i]); 158 | } 159 | 160 | auto popFront() 161 | { 162 | _i++; 163 | } 164 | 165 | auto length() @property 166 | { 167 | return topic_metadata_.partition_cnt - _i; 168 | } 169 | } 170 | 171 | return Partitions(topic_metadata_); 172 | } 173 | 174 | /** Topic error reported by broker */ 175 | ErrorCode err() 176 | { 177 | return cast(ErrorCode)(topic_metadata_.err); 178 | } 179 | } 180 | 181 | /** 182 | * Metadata container 183 | */ 184 | final class Metadata 185 | { 186 | private const(rd_kafka_metadata_t)* metadata_; 187 | 188 | nothrow @nogc: 189 | 190 | this(const(rd_kafka_metadata_t)* metadata) 191 | { 192 | metadata_ = metadata; 193 | } 194 | 195 | ~this() 196 | { 197 | rd_kafka_metadata_destroy(metadata_); 198 | } 199 | 200 | const @property: 201 | 202 | /** Broker list */ 203 | auto brokers() 204 | { 205 | static struct Brokers 206 | { 207 | nothrow @nogc: 208 | private const(rd_kafka_metadata_t)* metadata_; 209 | private size_t _i; 210 | auto empty() @property 211 | { 212 | return _i >= metadata_.broker_cnt; 213 | } 214 | 215 | auto front() @property 216 | { 217 | return BrokerMetadata(&metadata_.brokers[_i]); 218 | } 219 | 220 | auto popFront() 221 | { 222 | _i++; 223 | } 224 | 225 | auto length() @property 226 | { 227 | return metadata_.broker_cnt - _i; 228 | } 229 | } 230 | assert(metadata_); 231 | return Brokers(metadata_); 232 | } 233 | 234 | /** Topic list */ 235 | auto topics() 236 | { 237 | static struct Topics 238 | { 239 | nothrow @nogc: 240 | private const(rd_kafka_metadata_t)* metadata_; 241 | private size_t _i; 242 | auto empty() @property 243 | { 244 | return _i >= metadata_.topic_cnt; 245 | } 246 | 247 | auto front() @property 248 | { 249 | return TopicMetadata(&metadata_.topics[_i]); 250 | } 251 | 252 | auto popFront() 253 | { 254 | _i++; 255 | } 256 | 257 | auto length() @property 258 | { 259 | return metadata_.topic_cnt - _i; 260 | } 261 | } 262 | assert(metadata_); 263 | return Topics(metadata_); 264 | } 265 | 266 | /** Broker (id) originating this metadata */ 267 | int origBrokerId() 268 | { 269 | return metadata_.orig_broker_id; 270 | } 271 | 272 | /** Broker (name) originating this metadata */ 273 | const(char)[] origBrokerName() 274 | { 275 | return metadata_.orig_broker_name.fromStringz; 276 | } 277 | } 278 | -------------------------------------------------------------------------------- /source/rdkafkad/topic.d: -------------------------------------------------------------------------------- 1 | /// 2 | module rdkafkad.topic; 3 | import rdkafkad; 4 | 5 | /** 6 | * Topic+Partition 7 | * 8 | * This is a generic type to hold a single partition and various 9 | * information about it. 10 | * 11 | * Is typically used with std::vector to provide 12 | * a list of partitions for different operations. 13 | */ 14 | struct TopicPartition 15 | { 16 | package const(char)* topic_; 17 | package int partition_; 18 | package long offset_; 19 | package ErrorCode err_; 20 | 21 | void toString(in void delegate(const(char)[]) sink) const 22 | { 23 | sink(topic); 24 | import std.format; 25 | sink.formattedWrite("[%s]", partition_); 26 | } 27 | 28 | /** 29 | * Create topic+partition object for \p topic and \p partition. 30 | * 31 | * Use \c delete to deconstruct. 32 | */ 33 | 34 | nothrow: 35 | 36 | /// ditto 37 | this(const(char)[] topic, int partition) 38 | { 39 | this(topic.toStringz, partition); 40 | } 41 | 42 | @nogc: 43 | 44 | this(const(char)* topic, int partition) 45 | { 46 | topic_ = topic; 47 | partition_ = partition; 48 | offset_ = Offset.invalid; 49 | err_ = ErrorCode.no_error; 50 | } 51 | 52 | 53 | package this(const rd_kafka_topic_partition_t* c_part) 54 | { 55 | topic_ = c_part.topic; 56 | partition_ = c_part.partition; 57 | offset_ = c_part.offset; 58 | err_ = cast(ErrorCode) c_part.err; 59 | // FIXME: metadata 60 | } 61 | 62 | /** partition id */ 63 | int partition() @property 64 | { 65 | return partition_; 66 | } 67 | /** topic name */ 68 | const(char)[] topic() const @property 69 | { 70 | return topic_.fromStringz; 71 | } 72 | 73 | /** offset (if applicable) */ 74 | long offset() @property 75 | { 76 | return offset_; 77 | } 78 | 79 | /** Set offset */ 80 | void offset(long offset) @property 81 | { 82 | offset_ = offset; 83 | } 84 | 85 | /** error code (if applicable) */ 86 | ErrorCode err() @property 87 | { 88 | return err_; 89 | } 90 | } 91 | 92 | /** Special offsets */ 93 | enum Offset 94 | { 95 | beginning = -2, /**< Consume from beginning */ 96 | end = -1, /**< Consume from end */ 97 | stored = -1000, /**< Use offset storage */ 98 | invalid = -1001, /**< Invalid offset */ 99 | } 100 | 101 | /** 102 | * Topic handle 103 | * 104 | */ 105 | class Topic 106 | { 107 | /** 108 | * Creates a new topic handle for topic named \p topic_str 109 | * 110 | * \p conf is an optional configuration for the topic that will be used 111 | * instead of the default topic configuration. 112 | * The \p conf object is reusable after this call. 113 | * 114 | * the new topic handle or null on error (see \p errstr). 115 | */ 116 | this(Handle base, const(char)[] topic_str, TopicConf conf) 117 | { 118 | rd_kafka_topic_conf_t* rkt_conf; 119 | 120 | if (!conf) 121 | rkt_conf = rd_kafka_topic_conf_new(); 122 | else /* Make a copy of conf struct to allow Conf reuse. */ 123 | rkt_conf = rd_kafka_topic_conf_dup(conf.rkt_conf_); 124 | 125 | /* Set topic opaque to the topic so that we can reach our topic object 126 | * from whatever callbacks get registered. 127 | * The application itself will not need these opaques since their 128 | * callbacks are class based. */ 129 | rd_kafka_topic_conf_set_opaque(rkt_conf, cast(void*) this); 130 | 131 | if (conf) 132 | { 133 | if (conf.partitioner_cb_) 134 | { 135 | rd_kafka_topic_conf_set_partitioner_cb(rkt_conf, &partitioner_cb_trampoline); 136 | this.partitioner_cb_ = conf.partitioner_cb_; 137 | } 138 | else if (conf.partitioner_kp_cb_) 139 | { 140 | rd_kafka_topic_conf_set_partitioner_cb(rkt_conf, &partitioner_kp_cb_trampoline); 141 | this.partitioner_kp_cb_ = conf.partitioner_kp_cb_; 142 | } 143 | } 144 | auto str0 = topic_str.toStringz(); 145 | rkt_ = rd_kafka_topic_new(base.rk_, str0, rkt_conf); 146 | if (!rkt_) 147 | { 148 | auto msg = err2str(cast(ErrorCode)rd_kafka_errno2err(errno)); 149 | rd_kafka_topic_conf_destroy(rkt_conf); 150 | throw new Exception(msg); 151 | } 152 | } 153 | 154 | nothrow @nogc: 155 | 156 | ~this() 157 | { 158 | if (rkt_) 159 | { 160 | rd_kafka_topic_destroy(rkt_); 161 | } 162 | } 163 | 164 | package rd_kafka_topic_t* rkt_; 165 | package PartitionerCb partitioner_cb_; 166 | package PartitionerKeyPointerCb partitioner_kp_cb_; 167 | 168 | /** 169 | * Unassigned partition. 170 | * 171 | * The unassigned partition is used by the producer API for messages 172 | * that should be partitioned using the configured or default partitioner. 173 | */ 174 | enum int unassignedPartition = -1; 175 | 176 | package static nothrow @nogc int partitioner_cb_trampoline( 177 | const rd_kafka_topic_t* rkt, const(void)* keydata, size_t keylen, 178 | int partition_cnt, void* rkt_opaque, void* msg_opaque) 179 | { 180 | auto topic = cast(Topic) rkt_opaque; 181 | auto key = (cast(const(char)*) keydata)[0 .. keylen]; 182 | return topic.partitioner_cb_(topic, key, partition_cnt, msg_opaque); 183 | } 184 | 185 | package static nothrow @nogc int partitioner_kp_cb_trampoline( 186 | const rd_kafka_topic_t* rkt, const(void)* keydata, size_t keylen, 187 | int partition_cnt, void* rkt_opaque, void* msg_opaque) 188 | { 189 | auto topic = cast(Topic) rkt_opaque; 190 | return topic.partitioner_kp_cb_(topic, keydata, keylen, partition_cnt, msg_opaque); 191 | } 192 | 193 | /** the topic name */ 194 | final const(char)[] name() const 195 | { 196 | return rd_kafka_topic_name(rkt_).fromStringz; 197 | } 198 | 199 | /** 200 | * true if \p partition is available for the topic (has leader). 201 | * Warning: \b MUST \b ONLY be called from within a 202 | * PartitionerCb callback. 203 | */ 204 | final bool partitionAvailable(int partition) const 205 | { 206 | typeof(return) ret; 207 | ret = cast(bool) rd_kafka_topic_partition_available(rkt_, partition); 208 | return ret; 209 | } 210 | 211 | /** 212 | * Store offset \p offset for topic partition \p partition. 213 | * The offset will be committed (written) to the offset store according 214 | * to \p auto.commit.interval.ms. 215 | * 216 | * Note: This API should only be used with the simple Consumer, 217 | * not the high-level KafkaConsumer. 218 | * Note: \c auto.commit.enable must be set to \c false when using this API. 219 | * 220 | * ErrorCodde.no_error on success or an error code on error. 221 | */ 222 | final ErrorCode offsetStore(int partition, long offset) 223 | { 224 | typeof(return) ret; 225 | ret = cast(ErrorCode) rd_kafka_offset_store(rkt_, partition, offset); 226 | return ret; 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /source/rdkafkad/handler/producer.d: -------------------------------------------------------------------------------- 1 | /// 2 | module rdkafkad.handler.producer; 3 | import rdkafkad; 4 | 5 | import std.datetime: Clock, UTC; 6 | 7 | /** 8 | * Producer 9 | */ 10 | class Producer : Handle 11 | { 12 | package GlobalConf _conf; 13 | /** 14 | * Creates a new Kafka producer handle. 15 | * 16 | * \p conf is an optional object that will be used instead of the default 17 | * configuration. 18 | * The \p conf object is reusable after this call. 19 | */ 20 | this(GlobalConf conf) 21 | { 22 | _conf = conf; 23 | char[512] errbuf = void; 24 | rd_kafka_conf_t* rk_conf = null; 25 | 26 | if (conf) 27 | { 28 | if (!conf.rk_conf_) 29 | { 30 | throw new Exception("Requires Conf::CONF_GLOBAL object"); 31 | } 32 | 33 | this.setCommonConfig(conf); 34 | 35 | rk_conf = rd_kafka_conf_dup(conf.rk_conf_); 36 | 37 | if (conf.dr_cb_) 38 | { 39 | rd_kafka_conf_set_dr_msg_cb(rk_conf, &dr_msg_cb_trampoline); 40 | this.dr_cb_ = conf.dr_cb_; 41 | } 42 | } 43 | 44 | if (null is(rk_ = rd_kafka_new(rd_kafka_type_t.RD_KAFKA_PRODUCER, 45 | rk_conf, errbuf.ptr, errbuf.sizeof))) 46 | { 47 | throw new Exception(errbuf.ptr.fromStringz.idup); 48 | } 49 | } 50 | 51 | /++ 52 | Returns: new topic for this producer 53 | Params: 54 | topic = topic name 55 | topicConf = TopicConf, if null `defaultTopicConf` should be setted to the global configuration. 56 | +/ 57 | Topic newTopic(const(char)[] topic, TopicConf topicConf = null) 58 | { 59 | if(!topicConf) 60 | topicConf = _conf.defaultTopicConf; 61 | assert(topicConf); 62 | return new Topic(this, topic, topicConf); 63 | } 64 | 65 | nothrow @nogc: 66 | 67 | ~this() 68 | { 69 | if (rk_) 70 | { 71 | rd_kafka_destroy(rk_); 72 | } 73 | } 74 | 75 | /** 76 | * Producer::produce() \p msgflags 77 | * 78 | * These flags are optional and mutually exclusive. 79 | */ 80 | enum MsgOpt 81 | { 82 | free = 0x1, /**< rdkafka will free(3) \p payload 83 | * when it is done with it. */ 84 | copy = 0x2, /**< the \p payload data will be copied 85 | * and the \p payload pointer will not 86 | * be used by rdkafka after the 87 | * call returns. */ 88 | block = 0x4, /**< Block produce*() on message queue 89 | * full. 90 | * WARNING: 91 | * If a delivery report callback 92 | * is used the application MUST 93 | * call rd_kafka_poll() (or equiv.) 94 | * to make sure delivered messages 95 | * are drained from the internal 96 | * delivery report queue. 97 | * Failure to do so will result 98 | * in indefinately blocking on 99 | * the produce() call when the 100 | * message queue is full. 101 | */ 102 | } 103 | 104 | nothrow @nogc private static void dr_msg_cb_trampoline(rd_kafka_t* rk, 105 | const rd_kafka_message_t* rkmessage, void* opaque) 106 | { 107 | auto handle = cast(Handle) opaque; 108 | auto message = Message(null, rkmessage); 109 | handle.dr_cb_(message); 110 | message.destroy; 111 | } 112 | 113 | /** 114 | * Produce and send a single message to broker. 115 | * 116 | * This is an asynch non-blocking API. 117 | * 118 | * \p partition is the target partition, either: 119 | * - Topic::PARTITION_UA (unassigned) for 120 | * automatic partitioning using the topic's partitioner function, or 121 | * - a fixed partition (0..N) 122 | * 123 | * \p msgflags is zero or more of the following flags OR:ed together: 124 | * block - block \p produce*() call if 125 | * \p queue.buffering.max.messages or 126 | * \p queue.buffering.max.kbytes are exceeded. 127 | * Messages are considered in-queue from the point they 128 | * are accepted by produce() until their corresponding 129 | * delivery report callback/event returns. 130 | * It is thus a requirement to call 131 | * poll() (or equiv.) from a separate 132 | * thread when block is used. 133 | * See WARNING on \c block above. 134 | * free - rdkafka will free(3) \p payload when it is done with it. 135 | * copy - the \p payload data will be copied and the \p payload 136 | * pointer will not be used by rdkafka after the 137 | * call returns. 138 | * 139 | * NOTE: free and copy are mutually exclusive. 140 | * 141 | * If the function returns -1 and free was specified, then 142 | * the memory associated with the payload is still the caller's 143 | * responsibility. 144 | * 145 | * \p payload is the message payload of size \p len bytes. 146 | * 147 | * \p key is an optional message key, if non-null it 148 | * will be passed to the topic partitioner as well as be sent with the 149 | * message to the broker and passed on to the consumer. 150 | * 151 | * \p msg_opaque is an optional application-provided per-message opaque 152 | * pointer that will provided in the delivery report callback (\p dr_cb) for 153 | * referencing this message. 154 | * 155 | * an ErrorCode to indicate success or failure: 156 | * - _QUEUE_FULL - maximum number of outstanding messages has been 157 | * reached: \c queue.buffering.max.message 158 | * 159 | * - MSG_SIZE_TOO_LARGE - message is larger than configured max size: 160 | * \c messages.max.bytes 161 | * 162 | * - _UNKNOWN_PARTITION - requested \p partition is unknown in the 163 | * Kafka cluster. 164 | * 165 | * - _UNKNOWN_TOPIC - topic is unknown in the Kafka cluster. 166 | */ 167 | ErrorCode produce(Topic topic, int partition, void[] payload, 168 | const(void)[] key = null, 169 | long timestamp = Clock.currTime(UTC()).toUnixTime!long, 170 | int msgflags = MsgOpt.copy, 171 | void* msg_opaque = null) 172 | { 173 | int err; 174 | //with(rd_kafka_vtype_t) 175 | //err = rd_kafka_producev( 176 | // rk_, 177 | // RD_KAFKA_VTYPE_RKT, 178 | // topic.rkt_, 179 | // RD_KAFKA_VTYPE_PARTITION, 180 | // partition, 181 | // RD_KAFKA_VTYPE_MSGFLAGS, 182 | // msgflags, 183 | // RD_KAFKA_VTYPE_VALUE, 184 | // payload.ptr, 185 | // payload.length, 186 | // RD_KAFKA_VTYPE_KEY, 187 | // key.ptr, 188 | // key.length, 189 | // //RD_KAFKA_VTYPE_OPAQUE, 190 | // //msg_opaque, 191 | // RD_KAFKA_VTYPE_END, 192 | // ); 193 | err = rd_kafka_produce(topic.rkt_, partition, msgflags, payload.ptr, 194 | payload.length, key.ptr, key.length, msg_opaque); 195 | if (err == -1) 196 | return cast(ErrorCode) rd_kafka_errno2err(errno); 197 | return ErrorCode.no_error; 198 | } 199 | 200 | /** 201 | * Wait until all outstanding produce requests, et.al, are completed. 202 | * This should typically be done prior to destroying a producer instance 203 | * to make sure all queued and in-flight produce requests are completed 204 | * before terminating. 205 | * 206 | * Note: This function will call poll() and thus trigger callbacks. 207 | * 208 | * timed_out if \p timeout_ms was reached before all 209 | * outstanding requests were completed, else ErrorCode.no_error 210 | */ 211 | ErrorCode flush(int timeout_ms = 60_000) 212 | { 213 | typeof(return) ret; 214 | ret = cast(ErrorCode) rd_kafka_flush(rk_, timeout_ms); 215 | return ret; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /source/rdkafkad/config.d: -------------------------------------------------------------------------------- 1 | /// 2 | module rdkafkad.config; 3 | import rdkafkad; 4 | 5 | /** 6 | * \b Portability: OpenCb callback class 7 | * 8 | */ 9 | /** 10 | * Open callback 11 | * The open callback is responsible for opening the file specified by 12 | * \p pathname, using \p flags and \p mode. 13 | * The file shall be opened with \c CLOEXEC set in a racefree fashion, if 14 | * possible. 15 | * 16 | * It is typically not required to register an alternative open implementation 17 | * 18 | * Note: Not currently available on native Win32 19 | */ 20 | alias OpenCb = int function(const(char)[] path, int flags, int mode) nothrow @nogc; 21 | 22 | /// 23 | class ConfException : Exception 24 | { 25 | /** 26 | * Conf::Set() result code 27 | */ 28 | enum Result 29 | { 30 | UNKNOWN = -2, /**< Unknown configuration property */ 31 | INVALID = -1, /**< Invalid configuration value */ 32 | OK /**< Configuration property was succesfully set */ 33 | } 34 | /// 35 | Result result; 36 | /// 37 | this(Result result, string msg, string file = __FILE__, 38 | uint line = cast(uint) __LINE__, Throwable next = null) pure nothrow @nogc @safe 39 | { 40 | super(msg, file, line, next); 41 | this.result = result; 42 | } 43 | } 44 | 45 | /** 46 | * Configuration interface 47 | * 48 | * Holds either global or topic configuration that are passed to 49 | * Consumer::create(), Producer::create(), 50 | * create(), etc. 51 | * 52 | * See_also: CONFIGURATION.md for the full list of supported properties. 53 | */ 54 | abstract class Conf 55 | { 56 | /** 57 | * Set configuration property \p name to value \p value. 58 | * OK on success, else writes a human readable error 59 | * description to \p errstr on error. 60 | */ 61 | void opIndexAssign(in char[] value, in char[] name); 62 | 63 | /** Query single configuration value 64 | * OK if the property was set previously set and 65 | * returns the value in \p value. */ 66 | string opIndex(in char[] name) const; 67 | 68 | /// 69 | string[string] dump(); 70 | 71 | private static string[string] dumpImpl(const (char)** arrc, size_t cnt) 72 | { 73 | assert(cnt % 2 == 0); 74 | string[string] aa; 75 | foreach (size_t i; 0 .. cnt / 2) 76 | aa[arrc[i * 2].fromStringz.idup] = arrc[i * 2 + 1].fromStringz.idup; 77 | rd_kafka_conf_dump_free(arrc, cnt); 78 | return aa; 79 | } 80 | 81 | /// 82 | void fromText(string text) 83 | { 84 | import std.algorithm; 85 | import std.string; 86 | import std.format; 87 | import std.conv: ConvException; 88 | size_t i; 89 | foreach(line; text.lineSplitter) 90 | { 91 | i++; 92 | line = line.findSplit("#")[0].strip; 93 | if (line.empty) 94 | continue; 95 | auto t = line.findSplit("="); 96 | if (t[1].empty) 97 | throw new ConvException(format("failed to parse configuraiton at line %s", i)); 98 | auto key = t[0].stripRight; 99 | auto value = t[2].stripLeft; 100 | this[key] = value; 101 | } 102 | } 103 | } 104 | 105 | 106 | /// 107 | class GlobalConf : Conf 108 | { 109 | /// 110 | this(TopicConf defaultTopicConf = new TopicConf) 111 | { 112 | rk_conf_ = rd_kafka_conf_new; 113 | this.defaultTopicConf = defaultTopicConf; 114 | } 115 | 116 | private TopicConf _defaultTopicConf; 117 | 118 | /** Dump configuration names and values to list containing 119 | * name,value tuples */ 120 | override string[string] dump() 121 | { 122 | size_t cnt; 123 | auto arrc = rd_kafka_conf_dump(rk_conf_, &cnt); 124 | return dumpImpl(arrc, cnt); 125 | } 126 | 127 | /** 128 | * Set configuration property \p name to value \p value. 129 | * OK on success, else writes a human readable error 130 | * description to \p errstr on error. 131 | */ 132 | override void opIndexAssign(in char[] value, in char[] name) 133 | { 134 | char[512] errbuf = void; 135 | 136 | auto res = rd_kafka_conf_set(this.rk_conf_, name.toStringz(), 137 | value.toStringz(), errbuf.ptr, errbuf.sizeof); 138 | if (res != rd_kafka_conf_res_t.RD_KAFKA_CONF_OK) 139 | throw new ConfException(cast(ConfException.Result) res, 140 | errbuf.ptr.fromStringz.idup); 141 | } 142 | 143 | /** Query single configuration value 144 | * OK if the property was set previously set and 145 | * returns the value in \p value. */ 146 | override string opIndex(in char[] name) const 147 | { 148 | size_t size; 149 | 150 | rd_kafka_conf_res_t res = rd_kafka_conf_res_t.RD_KAFKA_CONF_OK; 151 | res = rd_kafka_conf_get(rk_conf_, name.toStringz(), null, &size); 152 | if (res != rd_kafka_conf_res_t.RD_KAFKA_CONF_OK) 153 | throw new ConfException(cast(ConfException.Result) res, "can not get config"); 154 | 155 | char[] value = new char[size]; 156 | res = rd_kafka_conf_get(rk_conf_, name.toStringz(), value.ptr, &size); 157 | if (res != rd_kafka_conf_res_t.RD_KAFKA_CONF_OK) 158 | throw new ConfException(cast(ConfException.Result) res, "can not get config"); 159 | return cast(string) value; 160 | } 161 | 162 | nothrow @nogc: 163 | 164 | ~this() nothrow @nogc 165 | { 166 | if (rk_conf_) 167 | rd_kafka_conf_destroy(rk_conf_); 168 | } 169 | 170 | package DeliveryReportCb dr_cb_; 171 | package EventCb event_cb_; 172 | package SocketCb socket_cb_; 173 | package OpenCb open_cb_; 174 | package RebalanceCb rebalance_cb_; 175 | package OffsetCommitCb offset_commit_cb_; 176 | package rd_kafka_conf_t* rk_conf_; 177 | 178 | /++ 179 | +/ 180 | void drCb(DeliveryReportCb cb) @property 181 | { 182 | dr_cb_ = cb; 183 | } 184 | /++ 185 | +/ 186 | void eventCb(EventCb cb) @property 187 | { 188 | event_cb_ = cb; 189 | } 190 | /++ 191 | +/ 192 | void socketCb(SocketCb cb) @property 193 | { 194 | socket_cb_ = cb; 195 | } 196 | /++ 197 | +/ 198 | void openCb(OpenCb cb) @property 199 | { 200 | open_cb_ = cb; 201 | } 202 | /++ 203 | +/ 204 | void rebalanceCb(RebalanceCb cb) @property 205 | { 206 | rebalance_cb_ = cb; 207 | } 208 | /++ 209 | +/ 210 | void offsetCommitCb(OffsetCommitCb cb) @property 211 | { 212 | offset_commit_cb_ = cb; 213 | } 214 | 215 | /** Use with \p name = \c \"default_topic_conf\" 216 | * 217 | * Sets the default topic configuration to use for for automatically 218 | * subscribed topics and for Topic construction with producer. 219 | * 220 | * See_also: subscribe() 221 | */ 222 | void defaultTopicConf(TopicConf topic_conf) @property 223 | { 224 | _defaultTopicConf = topic_conf; 225 | rd_kafka_conf_set_default_topic_conf(rk_conf_, 226 | rd_kafka_topic_conf_dup(topic_conf.rkt_conf_)); 227 | } 228 | 229 | ///ditto 230 | TopicConf defaultTopicConf() @property 231 | { 232 | return _defaultTopicConf; 233 | } 234 | 235 | } 236 | 237 | class TopicConf : Conf 238 | { 239 | /** 240 | * Set configuration property \p name to value \p value. 241 | * OK on success, else writes a human readable error 242 | * description to \p errstr on error. 243 | */ 244 | override void opIndexAssign(in char[] value, in char[] name) 245 | { 246 | rd_kafka_conf_res_t res; 247 | char[512] errbuf = void; 248 | 249 | res = rd_kafka_topic_conf_set(this.rkt_conf_, name.toStringz(), 250 | value.toStringz(), errbuf.ptr, errbuf.sizeof); 251 | 252 | if (res != rd_kafka_conf_res_t.RD_KAFKA_CONF_OK) 253 | throw new ConfException(cast(ConfException.Result) res, 254 | errbuf.ptr.fromStringz.idup); 255 | } 256 | 257 | /** Query single configuration value 258 | * OK if the property was set previously set and 259 | * returns the value in \p value. */ 260 | override string opIndex(in char[] name) const 261 | { 262 | size_t size; 263 | rd_kafka_conf_res_t res = rd_kafka_conf_res_t.RD_KAFKA_CONF_OK; 264 | res = rd_kafka_topic_conf_get(rkt_conf_, name.toStringz(), null, &size); 265 | if (res != rd_kafka_conf_res_t.RD_KAFKA_CONF_OK) 266 | throw new ConfException(cast(ConfException.Result) res, "can not get config"); 267 | char[] value = new char[size]; 268 | res = rd_kafka_topic_conf_get(rkt_conf_, name.toStringz(), value.ptr, &size); 269 | if (res != rd_kafka_conf_res_t.RD_KAFKA_CONF_OK) 270 | throw new ConfException(cast(ConfException.Result) res, "can not get config"); 271 | return cast(string) value; 272 | } 273 | 274 | /** Dump configuration names and values to list containing 275 | * name,value tuples */ 276 | override string[string] dump() 277 | { 278 | size_t cnt; 279 | auto arrc = rd_kafka_topic_conf_dump(rkt_conf_, &cnt); 280 | return dumpImpl(arrc, cnt); 281 | } 282 | 283 | nothrow @nogc: 284 | 285 | ~this() 286 | { 287 | if (rkt_conf_) 288 | rd_kafka_topic_conf_destroy(rkt_conf_); 289 | } 290 | 291 | /** 292 | * Create configuration object 293 | */ 294 | this() 295 | { 296 | rkt_conf_ = rd_kafka_topic_conf_new(); 297 | } 298 | 299 | package PartitionerCb partitioner_cb_; 300 | package PartitionerKeyPointerCb partitioner_kp_cb_; 301 | package rd_kafka_topic_conf_t* rkt_conf_; 302 | 303 | /++ 304 | +/ 305 | void partitionerCb(PartitionerCb cb) @property 306 | { 307 | partitioner_cb_ = cb; 308 | } 309 | /++ 310 | +/ 311 | void partitionerKpCb(PartitionerKeyPointerCb cb) @property 312 | { 313 | partitioner_kp_cb_ = cb; 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /source/rdkafkad/handler/simple_consumer.d: -------------------------------------------------------------------------------- 1 | /// 2 | module rdkafkad.handler.simple_consumer; 3 | import rdkafkad; 4 | 5 | /** 6 | * Simple Consumer (legacy) 7 | * 8 | * A simple non-balanced, non-group-aware, consumer. 9 | */ 10 | class Consumer : Handle 11 | { 12 | /** 13 | * Creates a new Kafka consumer handle. 14 | * 15 | * \p conf is an optional object that will be used instead of the default 16 | * configuration. 17 | * The \p conf object is reusable after this call. 18 | * 19 | * the new handle on success or null on error in which case 20 | * \p errstr is set to a human readable error message. 21 | */ 22 | this(GlobalConf conf) 23 | { 24 | char[512] errbuf = void; 25 | rd_kafka_conf_t* rk_conf = null; 26 | 27 | if (conf) 28 | { 29 | if (!conf.rk_conf_) 30 | { 31 | throw new Exception("Requires Conf::CONF_GLOBAL object"); 32 | } 33 | 34 | this.setCommonConfig(conf); 35 | 36 | rk_conf = rd_kafka_conf_dup(conf.rk_conf_); 37 | } 38 | 39 | rk_ = rd_kafka_new(rd_kafka_type_t.RD_KAFKA_CONSUMER, 40 | rk_conf, errbuf.ptr, errbuf.sizeof); 41 | if (null is rk_) 42 | { 43 | throw new Exception(errbuf.ptr.fromStringz.idup); 44 | } 45 | } 46 | 47 | nothrow @nogc: 48 | 49 | ~this() 50 | { 51 | rd_kafka_destroy(rk_); 52 | } 53 | 54 | static: 55 | 56 | /** 57 | * Start consuming messages for topic and \p partition 58 | * at offset \p offset which may either be a proper offset (0..N) 59 | * or one of the the special offsets: \p OFFSET_BEGINNING or \p OFFSET_END. 60 | * 61 | * rdkafka will attempt to keep \p queued.min.messages (config property) 62 | * messages in the local queue by repeatedly fetching batches of messages 63 | * from the broker until the threshold is reached. 64 | * 65 | * The application shall use one of the \p ...consume*() functions 66 | * to consume messages from the local queue, each kafka message being 67 | * represented as a `Message ` object. 68 | * 69 | * \p ...start() must not be called multiple times for the same 70 | * topic and partition without stopping consumption first with 71 | * \p ...stop(). 72 | * 73 | * an ErrorCode to indicate success or failure. 74 | */ 75 | ErrorCode start(Topic topic, int partition, long offset, Queue queue) 76 | { 77 | int err; 78 | err = rd_kafka_consume_start(topic.rkt_, partition, offset); 79 | if (err == -1) 80 | return cast(ErrorCode)(rd_kafka_errno2err(errno)); 81 | return ErrorCode.no_error; 82 | } 83 | 84 | /** 85 | * Stop consuming messages for topic and \p partition, purging 86 | * all messages currently in the local queue. 87 | * 88 | * The application needs to be stop all consumers before destroying 89 | * the Consumer handle. 90 | * 91 | * an ErrorCode to indicate success or failure. 92 | */ 93 | ErrorCode stop(Topic topic, int partition) 94 | { 95 | int err; 96 | err = rd_kafka_consume_stop(topic.rkt_, partition); 97 | if (err == -1) 98 | return cast(ErrorCode)(rd_kafka_errno2err(errno)); 99 | return ErrorCode.no_error; 100 | } 101 | 102 | /** 103 | * Seek consumer for topic+partition to \p offset which is either an 104 | * absolute or logical offset. 105 | * 106 | * If \p timeout_ms is not 0 the call will wait this long for the 107 | * seek to be performed. If the timeout is reached the internal state 108 | * will be unknown and this function returns `timed_out`. 109 | * If \p timeout_ms is 0 it will initiate the seek but return 110 | * immediately without any error reporting (e.g., async). 111 | * 112 | * This call triggers a fetch queue barrier flush. 113 | * 114 | * an ErrorCode to indicate success or failure. 115 | */ 116 | ErrorCode seek(Topic topic, int partition, long offset, int timeout_ms) 117 | { 118 | int err; 119 | err = rd_kafka_seek(topic.rkt_, partition, offset, timeout_ms); 120 | if (err == -1) 121 | return cast(ErrorCode)(rd_kafka_errno2err(errno)); 122 | return ErrorCode.no_error; 123 | } 124 | 125 | /** 126 | * Consume a single message from \p topic and \p partition. 127 | * 128 | * \p timeout_ms is maximum amount of time to wait for a message to be 129 | * received. 130 | * Consumer must have been previously started with \p ...start(). 131 | * 132 | * a Message object, the application needs to check if message 133 | * is an error or a proper message Message::err() and checking for 134 | * \p ErrorCode.no_error. 135 | * 136 | * The message object must be destroyed when the application is done with it. 137 | * 138 | * Errors (in Message::err()): 139 | * - timed_out - \p timeout_ms was reached with no new messages fetched. 140 | * - PARTITION_EOF - End of partition reached, not an error. 141 | */ 142 | void consume(Topic topic, int partition, int timeout_ms, ref Message msg) 143 | { 144 | rd_kafka_message_t* rkmessage; 145 | 146 | rkmessage = rd_kafka_consume(topic.rkt_, partition, timeout_ms); 147 | if (!rkmessage) 148 | msg = Message(topic, cast(ErrorCode) rd_kafka_errno2err(errno)); 149 | 150 | msg = Message(topic, rkmessage); 151 | } 152 | 153 | /** 154 | * Consume a single message from the specified queue. 155 | * 156 | * \p timeout_ms is maximum amount of time to wait for a message to be 157 | * received. 158 | * Consumer must have been previously started on the queue with 159 | * \p ...start(). 160 | * 161 | * a Message object, the application needs to check if message 162 | * is an error or a proper message \p Message.err() and checking for 163 | * \p ErrorCode.no_error. 164 | * 165 | * The message object must be destroyed when the application is done with it. 166 | * 167 | * Errors (in Message::err()): 168 | * - timed_out - \p timeout_ms was reached with no new messages fetched 169 | * 170 | * Note that Message.topic() may be nullptr after certain kinds of 171 | * errors, so applications should check that it isn't null before 172 | * dereferencing it. 173 | */ 174 | void consume(Queue queue, int timeout_ms, ref Message msg) 175 | { 176 | rd_kafka_message_t* rkmessage; 177 | rkmessage = rd_kafka_consume_queue(queue.queue_, timeout_ms); 178 | if (!rkmessage) 179 | msg = Message(null, cast(ErrorCode) rd_kafka_errno2err(errno)); 180 | /* 181 | * Recover our Topic from the topic conf's opaque field, which we 182 | * set in Topic::create() for just this kind of situation. 183 | */ 184 | void* opaque = rd_kafka_topic_opaque(rkmessage.rkt); 185 | Topic topic = cast(Topic)(opaque); 186 | 187 | msg = Message(topic, rkmessage); 188 | } 189 | 190 | /* Helper struct for `consume_callback'. 191 | * Encapsulates the values we need in order to call `rd_kafka_consume_callback' 192 | * and keep track of the C++ callback function and `opaque' value. 193 | */ 194 | private static struct ConsumerCallback 195 | { 196 | /* This function is the one we give to `rd_kafka_consume_callback', with 197 | * the `opaque' pointer pointing to an instance of this struct, in which 198 | * we can find the C++ callback and `cb_data'. 199 | */ 200 | static nothrow @nogc void consume_cb_trampoline(rd_kafka_message_t* msg, void* opaque) 201 | { 202 | ConsumerCallback* instance = cast(ConsumerCallback*) opaque; 203 | Message message = Message(instance.topic, msg, false /*don't free*/ ); 204 | instance.cb_cls(message); 205 | } 206 | 207 | Topic topic; 208 | ConsumeCb cb_cls; 209 | } 210 | 211 | /** 212 | * Consumes messages from \p topic and \p partition, calling 213 | * the provided callback for each consumed messsage. 214 | * 215 | * \p consumeCallback() provides higher throughput performance 216 | * than \p consume(). 217 | * 218 | * \p timeout_ms is the maximum amount of time to wait for one or 219 | * more messages to arrive. 220 | * 221 | * The provided \p consume_cb instance has its \p consume_cb function 222 | * called for every message received. 223 | * 224 | * The \p opaque argument is passed to the \p consume_cb as \p opaque. 225 | * 226 | * the number of messages processed or -1 on error. 227 | * 228 | * See_also: Consumer::consume() 229 | */ 230 | int consumeCallback(Topic topic, int partition, int timeout_ms, ConsumeCb consume_cb) 231 | { 232 | auto context = ConsumerCallback(topic, consume_cb); 233 | int ret; 234 | ret = rd_kafka_consume_callback(topic.rkt_, partition, timeout_ms, 235 | &ConsumerCallback.consume_cb_trampoline, &context); 236 | return ret; 237 | } 238 | 239 | /* Helper struct for `consume_callback' with a Queue. 240 | * Encapsulates the values we need in order to call `rd_kafka_consume_callback' 241 | * and keep track of the C++ callback function and `opaque' value. 242 | */ 243 | private static struct ConsumerQueueCallback 244 | { 245 | /* This function is the one we give to `rd_kafka_consume_callback', with 246 | * the `opaque' pointer pointing to an instance of this struct, in which 247 | * we can find the C++ callback and `cb_data'. 248 | */ 249 | static nothrow @nogc void consume_cb_trampoline(rd_kafka_message_t* msg, void* opaque) 250 | { 251 | ConsumerQueueCallback* instance = cast(ConsumerQueueCallback*) opaque; 252 | /* 253 | * Recover our Topic from the topic conf's opaque field, which we 254 | * set in Topic::create() for just this kind of situation. 255 | */ 256 | void* topic_opaque = rd_kafka_topic_opaque(msg.rkt); 257 | Topic topic = cast(Topic) topic_opaque; 258 | Message message = Message(topic, msg, false /*don't free*/ ); 259 | instance.cb_cls(message); 260 | } 261 | 262 | ConsumeCb cb_cls; 263 | } 264 | 265 | /** 266 | * Consumes messages from \p queue, calling the provided callback for 267 | * each consumed messsage. 268 | * 269 | * See_also: Consumer::consumeCallback() 270 | */ 271 | 272 | int consumeCallback(Queue queue, int timeout_ms, ConsumeCb consume_cb) 273 | { 274 | auto context = ConsumerQueueCallback(consume_cb); 275 | int ret; 276 | ret = rd_kafka_consume_callback_queue(queue.queue_, timeout_ms, 277 | &ConsumerQueueCallback.consume_cb_trampoline, &context); 278 | return ret; 279 | } 280 | 281 | /** 282 | * Converts an offset into the logical offset from the tail of a topic. 283 | * 284 | * \p offset is the (positive) number of items from the end. 285 | * 286 | * the logical offset for message \p offset from the tail, this value 287 | * may be passed to Consumer::start, et.al. 288 | * Note: The returned logical offset is specific to librdkafka. 289 | */ 290 | long offsetTail(long offset) 291 | { 292 | return RD_KAFKA_OFFSET_TAIL(offset); 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /source/rdkafkad/handler/kafka_consumer.d: -------------------------------------------------------------------------------- 1 | /// 2 | module rdkafkad.handler.kafka_consumer; 3 | import rdkafkad; 4 | 5 | /** 6 | * High-level KafkaConsumer (for brokers 0.9 and later) 7 | * 8 | * Note: Requires Apache Kafka >= 0.9.0 brokers 9 | * 10 | * Currently supports the \c range and \c roundrobin partition assignment 11 | * strategies (see \c partition.assignment.strategy) 12 | */ 13 | class KafkaConsumer : Handle 14 | { 15 | /** 16 | * Creates a KafkaConsumer. 17 | * 18 | * The \p conf object must have \c group.id set to the consumer group to join. 19 | * 20 | * Use close() to shut down the consumer. 21 | * 22 | * See_also: RebalanceCb 23 | * See_also: CONFIGURATION.md for \c group.id, \c session.timeout.ms, 24 | * \c partition.assignment.strategy, etc. 25 | */ 26 | this(GlobalConf conf) 27 | { 28 | char[512] errbuf = void; 29 | rd_kafka_conf_t* rk_conf = null; 30 | size_t grlen; 31 | 32 | if (!conf.rk_conf_) 33 | { 34 | throw new Exception("Requires Conf::CONF_GLOBAL object"); 35 | } 36 | 37 | if (rd_kafka_conf_get(conf.rk_conf_, "group.id", null, 38 | &grlen) != rd_kafka_conf_res_t.RD_KAFKA_CONF_OK || grlen <= 1 /* terminating null only */ ) 39 | { 40 | throw new Exception("\"group.id\" must be configured"); 41 | } 42 | 43 | this.setCommonConfig(conf); 44 | 45 | rk_conf = rd_kafka_conf_dup(conf.rk_conf_); 46 | 47 | rd_kafka_t* rk; 48 | if (null is(rk = rd_kafka_new(rd_kafka_type_t.RD_KAFKA_CONSUMER, rk_conf, 49 | errbuf.ptr, errbuf.sizeof))) 50 | { 51 | throw new Exception(errbuf.ptr.fromStringz.idup); 52 | } 53 | 54 | this.rk_ = rk; 55 | 56 | /* Redirect handle queue to cgrp's queue to provide a single queue point */ 57 | rd_kafka_poll_set_consumer(rk); 58 | } 59 | 60 | nothrow: 61 | 62 | /** Returns the current partition assignment as set by 63 | * assign() */ 64 | ErrorCode assignment(ref TopicPartition[] partitions) 65 | { 66 | rd_kafka_topic_partition_list_t* c_parts; 67 | rd_kafka_resp_err_t err; 68 | 69 | err = rd_kafka_assignment(rk_, &c_parts); 70 | if (err) 71 | return cast(ErrorCode) err; 72 | 73 | partitions.length = c_parts.cnt; 74 | 75 | foreach (i, ref p; partitions) 76 | p = TopicPartition(&c_parts.elems[i]); 77 | 78 | rd_kafka_topic_partition_list_destroy(c_parts); 79 | 80 | return ErrorCode.no_error; 81 | 82 | } 83 | 84 | @nogc: 85 | 86 | /** 87 | * Update the subscription set to \p topics. 88 | * 89 | * Any previous subscription will be unassigned and unsubscribed first. 90 | * 91 | * The subscription set denotes the desired topics to consume and this 92 | * set is provided to the partition assignor (one of the elected group 93 | * members) for all clients which then uses the configured 94 | * \c partition.assignment.strategy to assign the subscription sets's 95 | * topics's partitions to the consumers, depending on their subscription. 96 | * 97 | * The result of such an assignment is a rebalancing which is either 98 | * handled automatically in librdkafka or can be overriden by the application 99 | * by providing a RebalanceCb. 100 | * 101 | * The rebalancing passes the assigned partition set to 102 | * assign() to update what partitions are actually 103 | * being fetched by the KafkaConsumer. 104 | * 105 | * Regex pattern matching automatically performed for topics prefixed 106 | * with \c \"^\" (e.g. \c \"^myPfx[0-9]_.*\" 107 | */ 108 | ErrorCode subscribe(const(char)*[] topics...) 109 | { 110 | rd_kafka_topic_partition_list_t* c_topics; 111 | rd_kafka_resp_err_t err; 112 | 113 | c_topics = rd_kafka_topic_partition_list_new(cast(int) topics.length); 114 | 115 | foreach (t; topics) 116 | rd_kafka_topic_partition_list_add(c_topics, t, RD_KAFKA_PARTITION_UA); 117 | 118 | err = rd_kafka_subscribe(rk_, c_topics); 119 | 120 | rd_kafka_topic_partition_list_destroy(c_topics); 121 | 122 | return cast(ErrorCode) err; 123 | } 124 | /** Unsubscribe from the current subscription set. */ 125 | auto unsubscribe() 126 | { 127 | ErrorCode ret; 128 | ret = cast(ErrorCode)(rd_kafka_unsubscribe(this.rk_)); 129 | return ret; 130 | } 131 | 132 | /** 133 | * Consume message or get error event, triggers callbacks. 134 | * 135 | * Will automatically call registered callbacks for any such queued events, 136 | * including RebalanceCb, EventCb, OffsetCommitCb, 137 | * etc. 138 | * 139 | * Note: An application should make sure to call consume() at regular 140 | * intervals, even if no messages are expected, to serve any 141 | * queued callbacks waiting to be called. This is especially 142 | * important when a RebalanceCb has been registered as it needs 143 | * to be called and handled properly to synchronize internal 144 | * consumer state. 145 | * 146 | * Note: Application MUST NOT call \p poll() on KafkaConsumer objects. 147 | * 148 | * One of: 149 | * - proper message (Message::err() is ErrorCode.no_error) 150 | * - error event (Message::err() is != ErrorCode.no_error) 151 | * - timeout due to no message or event in \p timeout_ms 152 | * (Message::err() is timed_out) 153 | */ 154 | /++ 155 | Params: 156 | msg = message to fill. Use `msg.err` to check errors. 157 | timeout_ms = time to to wait if no incomming msgs in queue. 158 | +/ 159 | auto consume(ref Message msg, int timeout_ms = 10) 160 | { 161 | rd_kafka_message_t* rkmessage; 162 | 163 | rkmessage = rd_kafka_consumer_poll(this.rk_, timeout_ms); 164 | 165 | if (!rkmessage) 166 | { 167 | msg = Message(null, ErrorCode.timed_out); 168 | return; 169 | } 170 | 171 | msg = Message(rkmessage); 172 | } 173 | 174 | /** 175 | * Update the assignment set to \p partitions. 176 | * 177 | * The assignment set is the set of partitions actually being consumed 178 | * by the KafkaConsumer. 179 | */ 180 | ErrorCode assign(const TopicPartition[] partitions) 181 | { 182 | rd_kafka_topic_partition_list_t* c_parts; 183 | rd_kafka_resp_err_t err; 184 | 185 | c_parts = partitions_to_c_parts(partitions); 186 | 187 | err = rd_kafka_assign(rk_, c_parts); 188 | 189 | rd_kafka_topic_partition_list_destroy(c_parts); 190 | return cast(ErrorCode) err; 191 | } 192 | 193 | /** 194 | * Stop consumption and remove the current assignment. 195 | */ 196 | ErrorCode unassign() 197 | { 198 | typeof(return) ret; 199 | ret = cast(ErrorCode) rd_kafka_assign(rk_, null); 200 | return ret; 201 | } 202 | 203 | /** 204 | * Commit offsets for the current assignment. 205 | * 206 | * Note: This is the synchronous variant that blocks until offsets 207 | * are committed or the commit fails (see return value). 208 | * 209 | * Note: If a OffsetCommitCb callback is registered it will 210 | * be called with commit details on a future call to 211 | * consume() 212 | * 213 | * ErrorCode.no_error or error code. 214 | */ 215 | ErrorCode commitSync() 216 | { 217 | typeof(return) ret; 218 | ret = cast(ErrorCode) rd_kafka_commit(rk_, null, 0 /*sync*/ ); 219 | return ret; 220 | } 221 | 222 | /** 223 | * Asynchronous version of CommitSync() 224 | * 225 | * See_also: KafkaConsummer::commitSync() 226 | */ 227 | ErrorCode commitAsync() 228 | { 229 | typeof(return) ret; 230 | ret = cast(ErrorCode) rd_kafka_commit(rk_, null, 1 /*async*/ ); 231 | return ret; 232 | } 233 | 234 | /** 235 | * Commit offset for a single topic+partition based on \p message 236 | * 237 | * Note: This is the synchronous variant. 238 | * 239 | * See_also: KafkaConsummer::commitSync() 240 | */ 241 | ErrorCode commitSync(ref Message message) 242 | { 243 | typeof(return) ret; 244 | ret = cast(ErrorCode) rd_kafka_commit_message(rk_, message.rkmessage_, 0 /*sync*/ ); 245 | return ret; 246 | } 247 | 248 | /** 249 | * Commit offset for a single topic+partition based on \p message 250 | * 251 | * Note: This is the asynchronous variant. 252 | * 253 | * See_also: KafkaConsummer::commitSync() 254 | */ 255 | ErrorCode commitAsync(ref Message message) 256 | { 257 | typeof(return) ret; 258 | ret = cast(ErrorCode) rd_kafka_commit_message(rk_, message.rkmessage_, 1 /*async*/ ); 259 | return ret; 260 | } 261 | 262 | /** 263 | * Commit offsets for the provided list of partitions. 264 | * 265 | * Note: This is the synchronous variant. 266 | */ 267 | ErrorCode commitSync(TopicPartition[] offsets) 268 | { 269 | rd_kafka_topic_partition_list_t* c_parts = partitions_to_c_parts(offsets); 270 | rd_kafka_resp_err_t err; 271 | err = rd_kafka_commit(rk_, c_parts, 0); 272 | if (!err) 273 | update_partitions_from_c_parts(offsets, c_parts); 274 | rd_kafka_topic_partition_list_destroy(c_parts); 275 | return cast(ErrorCode) err; 276 | } 277 | 278 | /** 279 | * Commit offset for the provided list of partitions. 280 | * 281 | * Note: This is the asynchronous variant. 282 | */ 283 | ErrorCode commitAsync(const TopicPartition[] offsets) 284 | { 285 | rd_kafka_topic_partition_list_t* c_parts = partitions_to_c_parts(offsets); 286 | rd_kafka_resp_err_t err; 287 | err = rd_kafka_commit(rk_, c_parts, 1); 288 | rd_kafka_topic_partition_list_destroy(c_parts); 289 | return cast(ErrorCode) err; 290 | } 291 | 292 | /** 293 | * Retrieve committed offsets for topics+partitions. 294 | * 295 | * RD_KAFKA_RESP_ErrorCode.no_error on success in which case the 296 | * \p offset or \p err field of each \p partitions' element is filled 297 | * in with the stored offset, or a partition specific error. 298 | * Else returns an error code. 299 | */ 300 | ErrorCode committed(TopicPartition[] partitions, int timeout_ms) 301 | { 302 | rd_kafka_topic_partition_list_t* c_parts; 303 | rd_kafka_resp_err_t err; 304 | 305 | c_parts = partitions_to_c_parts(partitions); 306 | 307 | err = rd_kafka_committed(rk_, c_parts, timeout_ms); 308 | 309 | if (!err) 310 | { 311 | update_partitions_from_c_parts(partitions, c_parts); 312 | } 313 | 314 | rd_kafka_topic_partition_list_destroy(c_parts); 315 | 316 | return cast(ErrorCode) err; 317 | } 318 | 319 | /** 320 | * Retrieve current positions (offsets) for topics+partitions. 321 | * 322 | * RD_KAFKA_RESP_ErrorCode.no_error on success in which case the 323 | * \p offset or \p err field of each \p partitions' element is filled 324 | * in with the stored offset, or a partition specific error. 325 | * Else returns an error code. 326 | */ 327 | ErrorCode position(TopicPartition[] partitions) 328 | { 329 | rd_kafka_topic_partition_list_t* c_parts; 330 | rd_kafka_resp_err_t err; 331 | 332 | c_parts = partitions_to_c_parts(partitions); 333 | 334 | err = rd_kafka_position(rk_, c_parts); 335 | 336 | if (!err) 337 | { 338 | update_partitions_from_c_parts(partitions, c_parts); 339 | } 340 | 341 | rd_kafka_topic_partition_list_destroy(c_parts); 342 | 343 | return cast(ErrorCode) err; 344 | 345 | } 346 | 347 | /** 348 | * Close and shut down the proper. 349 | * 350 | * This call will block until the following operations are finished: 351 | * - Trigger a local rebalance to void the current assignment 352 | * - Stop consumption for current assignment 353 | * - Commit offsets 354 | * - Leave group 355 | * 356 | * The maximum blocking time is roughly limited to session.timeout.ms. 357 | * 358 | * Note: Callbacks, such as RebalanceCb and 359 | * OffsetCommitCb, etc, may be called. 360 | * 361 | * Note: The consumer object must later be freed with \c delete 362 | */ 363 | ErrorCode close() 364 | { 365 | rd_kafka_resp_err_t err; 366 | err = rd_kafka_consumer_close(rk_); 367 | if (err) 368 | return cast(ErrorCode) err; 369 | 370 | while (rd_kafka_outq_len(rk_) > 0) 371 | rd_kafka_poll(rk_, 10); 372 | rd_kafka_destroy(rk_); 373 | 374 | return cast(ErrorCode) err; 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /source/rdkafkad/handler/package.d: -------------------------------------------------------------------------------- 1 | /// 2 | module rdkafkad.handler; 3 | import rdkafkad; 4 | 5 | /** 6 | * Base handle, super class for specific clients. 7 | */ 8 | class Handle 9 | { 10 | 11 | /** 12 | * Returns the client's broker-assigned group member id 13 | * 14 | * Note: This currently requires the high-level KafkaConsumer 15 | * 16 | * Last assigned member id, or empty string if not currently 17 | * a group member. 18 | */ 19 | auto memberid() const 20 | { 21 | char* str; 22 | str = rd_kafka_memberid(rk_); 23 | string memberid = str.fromStringz.idup; 24 | if (str) 25 | rd_kafka_mem_free(cast(rd_kafka_s*) rk_, str); 26 | return memberid; 27 | } 28 | 29 | /** 30 | * Request Metadata from broker. 31 | * 32 | * Parameters: 33 | * \p all_topics - if non-zero: request info about all topics in cluster, 34 | * if zero: only request info about locally known topics. 35 | * \p only_rkt - only request info about this topic 36 | * \p metadatap - pointer to hold metadata result. 37 | * The \p *metadatap pointer must be released with \c delete. 38 | * \p timeout_ms - maximum response time before failing. 39 | * 40 | * ErrorCode.no_error on success (in which case \p *metadatap 41 | * will be set), else timed_out on timeout or 42 | * other error code on error. 43 | */ 44 | Metadata metadata(bool all_topics, const Topic only_rkt, int timeout_ms = 60_000) 45 | { 46 | const rd_kafka_metadata_t* cmetadatap = null; 47 | 48 | rd_kafka_topic_t* topic = only_rkt ? cast(rd_kafka_topic_t*) only_rkt.rkt_ : null; 49 | 50 | ErrorCode error; 51 | error = cast(ErrorCode)rd_kafka_metadata(rk_, all_topics, topic, &cmetadatap, timeout_ms); 52 | if (error) 53 | { 54 | throw new Exception(error.err2str); 55 | } 56 | return new Metadata(cmetadatap); 57 | } 58 | 59 | @nogc nothrow: 60 | 61 | package void setCommonConfig(GlobalConf conf) 62 | { 63 | rd_kafka_conf_set_opaque(conf.rk_conf_, cast(void*) this); 64 | 65 | if (conf.event_cb_) 66 | { 67 | rd_kafka_conf_set_log_cb(conf.rk_conf_, &log_cb_trampoline); 68 | rd_kafka_conf_set_error_cb(conf.rk_conf_, &error_cb_trampoline); 69 | rd_kafka_conf_set_throttle_cb(conf.rk_conf_, &throttle_cb_trampoline); 70 | rd_kafka_conf_set_stats_cb(conf.rk_conf_, &stats_cb_trampoline); 71 | event_cb_ = conf.event_cb_; 72 | } 73 | 74 | if (conf.socket_cb_) 75 | { 76 | rd_kafka_conf_set_socket_cb(conf.rk_conf_, &socket_cb_trampoline); 77 | socket_cb_ = conf.socket_cb_; 78 | } 79 | 80 | version (Windows) 81 | { 82 | } 83 | else if (conf.open_cb_) 84 | { 85 | rd_kafka_conf_set_open_cb(conf.rk_conf_, &open_cb_trampoline); 86 | open_cb_ = conf.open_cb_; 87 | } 88 | 89 | if (conf.rebalance_cb_) 90 | { 91 | rd_kafka_conf_set_rebalance_cb(conf.rk_conf_, &rebalance_cb_trampoline); 92 | rebalance_cb_ = conf.rebalance_cb_; 93 | } 94 | 95 | if (conf.offset_commit_cb_) 96 | { 97 | rd_kafka_conf_set_offset_commit_cb(conf.rk_conf_, &offset_commit_cb_trampoline); 98 | offset_commit_cb_ = conf.offset_commit_cb_; 99 | } 100 | } 101 | 102 | /** 103 | * Query broker for low (oldest/beginning) 104 | * and high (newest/end) offsets for partition. 105 | * 106 | * Offsets are returned in \p *low and \p *high respectively. 107 | * 108 | * ErrorCode.no_error on success or an error code on failure. 109 | */ 110 | auto queryWatermarkOffsets(const(char)* topic, int partition, 111 | ref long low, ref long high, int timeout_ms) 112 | { 113 | ErrorCode ret; 114 | ret = cast(ErrorCode) rd_kafka_query_watermark_offsets(rk_, topic, 115 | partition, &low, &high, timeout_ms); 116 | return ret; 117 | } 118 | 119 | package(rdkafkad) rd_kafka_t* rk_; 120 | /* All Producer and Consumer callbacks must reside in HandleImpl and 121 | * the opaque provided to rdkafka must be a pointer to HandleImpl, since 122 | * ProducerImpl and ConsumerImpl classes cannot be safely directly cast to 123 | * HandleImpl due to the skewed diamond inheritance. */ 124 | package(rdkafkad) EventCb event_cb_; 125 | package(rdkafkad) SocketCb socket_cb_; 126 | package(rdkafkad) OpenCb open_cb_; 127 | package(rdkafkad) DeliveryReportCb dr_cb_; 128 | package(rdkafkad) PartitionerCb partitioner_cb_; 129 | package(rdkafkad) PartitionerKeyPointerCb partitioner_kp_cb_; 130 | package(rdkafkad) RebalanceCb rebalance_cb_; 131 | package(rdkafkad) OffsetCommitCb offset_commit_cb_; 132 | 133 | /** the name of the handle */ 134 | const(char)[] name() const 135 | { 136 | return rd_kafka_name(rk_).fromStringz; 137 | } 138 | 139 | /** 140 | * Polls the provided kafka handle for events. 141 | * 142 | * Events will trigger application provided callbacks to be called. 143 | * 144 | * The \p timeout_ms argument specifies the maximum amount of time 145 | * (in milliseconds) that the call will block waiting for events. 146 | * For non-blocking calls, provide 0 as \p timeout_ms. 147 | * To wait indefinately for events, provide -1. 148 | * 149 | * Events: 150 | * - delivery report callbacks (if an DeliveryCb is configured) [producer] 151 | * - event callbacks (if an EventCb is configured) [producer & consumer] 152 | * 153 | * Note: An application should make sure to call poll() at regular 154 | * intervals to serve any queued callbacks waiting to be called. 155 | * 156 | * Warning: This method MUST NOT be used with the KafkaConsumer, 157 | * use its consume() instead. 158 | * 159 | * the number of events served. 160 | */ 161 | int poll(int timeout_ms = 10) 162 | { 163 | typeof(return) ret; 164 | ret = rd_kafka_poll(rk_, timeout_ms); 165 | return ret; 166 | } 167 | 168 | /** 169 | * Returns the current out queue length 170 | * 171 | * The out queue contains messages and requests waiting to be sent to, 172 | * or acknowledged by, the broker. 173 | */ 174 | int outqLen() 175 | { 176 | typeof(return) ret; 177 | ret = rd_kafka_outq_len(rk_); 178 | return ret; 179 | } 180 | 181 | /** 182 | * Convert a list of C partitions to C++ partitions 183 | */ 184 | nothrow @nogc package static TopicPartition[] c_parts_to_partitions( 185 | const rd_kafka_topic_partition_list_t* c_parts) 186 | { 187 | auto partitions = (cast(TopicPartition*) malloc(c_parts.cnt * TopicPartition.sizeof))[0 188 | .. c_parts.cnt]; 189 | foreach (i, ref p; partitions) 190 | partitions[i] = TopicPartition(&c_parts.elems[i]); 191 | return partitions; 192 | } 193 | 194 | nothrow @nogc static void free_partition_vector(ref TopicPartition[] v) 195 | { 196 | foreach (ref p; v) 197 | p.destroy; 198 | free(v.ptr); 199 | v = null; 200 | } 201 | 202 | nothrow @nogc package static rd_kafka_topic_partition_list_t* partitions_to_c_parts(const TopicPartition[] partitions) 203 | { 204 | rd_kafka_topic_partition_list_t* c_parts = rd_kafka_topic_partition_list_new(cast(int) partitions.length); 205 | 206 | foreach (ref tpi; partitions) 207 | { 208 | rd_kafka_topic_partition_t* rktpar = rd_kafka_topic_partition_list_add(c_parts, tpi.topic_, tpi.partition_); 209 | rktpar.offset = tpi.offset_; 210 | } 211 | 212 | return c_parts; 213 | } 214 | 215 | /** 216 | * @brief Update the application provided 'partitions' with info from 'c_parts' 217 | */ 218 | nothrow @nogc package static void update_partitions_from_c_parts(TopicPartition[] partitions, 219 | const rd_kafka_topic_partition_list_t* c_parts) 220 | { 221 | foreach (i; 0 .. c_parts.cnt) 222 | { 223 | const rd_kafka_topic_partition_t* p = &c_parts.elems[i]; 224 | 225 | /* Find corresponding C++ entry */ 226 | foreach (pp; partitions) 227 | { 228 | if (!strcmp(p.topic, pp.topic_) && p.partition == pp.partition_) 229 | { 230 | pp.offset_ = p.offset; 231 | pp.err_ = cast(ErrorCode) p.err; 232 | } 233 | } 234 | } 235 | } 236 | 237 | nothrow @nogc package static void log_cb_trampoline(const rd_kafka_t* rk, int level, 238 | const(char)* fac, const(char)* buf) 239 | { 240 | if (!rk) 241 | { 242 | rd_kafka_log_print(rk, level, fac, buf); 243 | return; 244 | } 245 | 246 | void* opaque = rd_kafka_opaque(rk); 247 | Handle handle = cast(Handle)(opaque); 248 | 249 | if (!handle.event_cb_) 250 | { 251 | rd_kafka_log_print(rk, level, fac, buf); 252 | return; 253 | } 254 | 255 | auto event = Event(Event.Type.log, ErrorCode.no_error, 256 | cast(Event.Severity)(level), fac, buf); 257 | 258 | handle.event_cb_(event); 259 | } 260 | 261 | nothrow @nogc package static void error_cb_trampoline(rd_kafka_t* rk, int err, 262 | const(char)* reason, void* opaque) 263 | { 264 | Handle handle = cast(Handle)(opaque); 265 | 266 | auto event = Event(Event.Type.error, cast(ErrorCode) err, 267 | Event.Severity.error, null, reason); 268 | 269 | handle.event_cb_(event); 270 | } 271 | 272 | nothrow @nogc package static void throttle_cb_trampoline(rd_kafka_t* rk, 273 | const(char)* broker_name, int broker_id, int throttle_time_ms, void* opaque) 274 | { 275 | Handle handle = cast(Handle)(opaque); 276 | 277 | auto event = Event(Event.Type.throttle); 278 | event.str_ = broker_name.fromStringz; 279 | event.id_ = broker_id; 280 | event.throttle_time_ = throttle_time_ms; 281 | 282 | handle.event_cb_(event); 283 | } 284 | 285 | nothrow @nogc package static int stats_cb_trampoline(rd_kafka_t* rk, char* json, 286 | size_t json_len, void* opaque) 287 | { 288 | Handle handle = cast(Handle)(opaque); 289 | auto event = Event(Event.Type.stats, ErrorCode.no_error, Event.Severity.info, 290 | null, json); 291 | 292 | handle.event_cb_(event); 293 | 294 | return 0; 295 | } 296 | 297 | nothrow @nogc package static int socket_cb_trampoline(int domain, int type, int protocol, void* opaque) 298 | { 299 | Handle handle = cast(Handle)(opaque); 300 | 301 | return handle.socket_cb_(domain, type, protocol); 302 | } 303 | 304 | nothrow @nogc package static int open_cb_trampoline(const(char)* pathname, int flags, 305 | mode_t mode, void* opaque) 306 | { 307 | Handle handle = cast(Handle)(opaque); 308 | 309 | return handle.open_cb_(pathname.fromStringz, flags, cast(int)(mode)); 310 | } 311 | 312 | nothrow @nogc package static void rebalance_cb_trampoline(rd_kafka_t* rk, 313 | rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* c_partitions, void* opaque) 314 | { 315 | auto handle = cast(KafkaConsumer)(opaque); 316 | TopicPartition[] partitions = c_parts_to_partitions(c_partitions); 317 | 318 | handle.rebalance_cb_(handle, cast(ErrorCode) err, partitions); 319 | 320 | free_partition_vector(partitions); 321 | } 322 | 323 | nothrow @nogc package static void offset_commit_cb_trampoline(rd_kafka_t* rk, 324 | rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t* c_offsets, void* opaque) 325 | { 326 | Handle handle = cast(Handle)(opaque); 327 | TopicPartition[] offsets; 328 | 329 | if (c_offsets) 330 | offsets = c_parts_to_partitions(c_offsets); 331 | 332 | handle.offset_commit_cb_(cast(ErrorCode) err, offsets); 333 | 334 | free_partition_vector(offsets); 335 | } 336 | 337 | /** 338 | * Pause producing or consumption for the provided list of partitions. 339 | * 340 | * Success or error is returned per-partition in the \p partitions list. 341 | * 342 | * ErrorCode::no_error 343 | * 344 | * See_also: resume() 345 | */ 346 | ErrorCode pause(TopicPartition[] partitions) 347 | { 348 | rd_kafka_topic_partition_list_t* c_parts; 349 | rd_kafka_resp_err_t err; 350 | 351 | c_parts = partitions_to_c_parts(partitions); 352 | 353 | err = rd_kafka_pause_partitions(rk_, c_parts); 354 | 355 | if (!err) 356 | update_partitions_from_c_parts(partitions, c_parts); 357 | 358 | rd_kafka_topic_partition_list_destroy(c_parts); 359 | 360 | return cast(ErrorCode) err; 361 | 362 | } 363 | 364 | /** 365 | * Resume producing or consumption for the provided list of partitions. 366 | * 367 | * Success or error is returned per-partition in the \p partitions list. 368 | * 369 | * ErrorCode::no_error 370 | * 371 | * See_also: pause() 372 | */ 373 | ErrorCode resume(TopicPartition[] partitions) 374 | { 375 | rd_kafka_topic_partition_list_t* c_parts; 376 | rd_kafka_resp_err_t err; 377 | 378 | c_parts = partitions_to_c_parts(partitions); 379 | 380 | err = rd_kafka_resume_partitions(rk_, c_parts); 381 | 382 | if (!err) 383 | update_partitions_from_c_parts(partitions, c_parts); 384 | 385 | rd_kafka_topic_partition_list_destroy(c_parts); 386 | 387 | return cast(ErrorCode) err; 388 | } 389 | 390 | /** 391 | * Get last known low (oldest/beginning) 392 | * and high (newest/end) offsets for partition. 393 | * 394 | * The low offset is updated periodically (if statistics.interval.ms is set) 395 | * while the high offset is updated on each fetched message set from the 396 | * broker. 397 | * 398 | * If there is no cached offset (either low or high, or both) then 399 | * OFFSET_INVALID will be returned for the respective offset. 400 | * 401 | * Offsets are returned in \p *low and \p *high respectively. 402 | * 403 | * ErrorCode.no_error on success or an error code on failure. 404 | * 405 | * Note: Shall only be used with an active consumer instance. 406 | */ 407 | ErrorCode getWatermarkOffsets(const(char)* topic, int partition, ref long low, 408 | ref long high) 409 | { 410 | return cast(ErrorCode) rd_kafka_get_watermark_offsets(rk_, topic, 411 | partition, &low, &high); 412 | } 413 | } 414 | 415 | /** 416 | * Queue interface 417 | * 418 | * Create a new message queue. Message queues allows the application 419 | * to re-route consumed messages from multiple topic+partitions into 420 | * one single queue point. This queue point, containing messages from 421 | * a number of topic+partitions, may then be served by a single 422 | * consume() method, rather than one per topic+partition combination. 423 | * 424 | * See the Consumer::start(), Consumer::consume(), and 425 | * Consumer::consumeCallback() methods that take a queue as the first 426 | * parameter for more information. 427 | */ 428 | class Queue 429 | { 430 | nothrow @nogc: 431 | /** 432 | * Create Queue object 433 | */ 434 | this(Handle handle) 435 | { 436 | queue_ = rd_kafka_queue_new(handle.rk_); 437 | } 438 | 439 | ~this() 440 | { 441 | rd_kafka_queue_destroy(queue_); 442 | } 443 | 444 | package: 445 | rd_kafka_queue_t* queue_; 446 | } 447 | -------------------------------------------------------------------------------- /source/rdkafkad/package.d: -------------------------------------------------------------------------------- 1 | /* 2 | * librdkafka - Apache Kafka C/C++ library 3 | * 4 | * Copyright (c) 2014 Magnus Edenhill 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * 1. Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 2. Redistributions in binary form must reproduce the above copyright notice, 13 | * this list of conditions and the following disclaimer in the documentation 14 | * and/or other materials provided with the distribution. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | * POSSIBILITY OF SUCH DAMAGE. 27 | */ 28 | 29 | /** 30 | * Apache Kafka C/C++ consumer and producer client library. 31 | * 32 | * rdkafkacpp.h contains the public C++ API for librdkafka. 33 | * The API is documented in this file as comments prefixing the class, 34 | * function, type, enum, define, etc. 35 | * For more information, see the C interface in rdkafka.h and read the 36 | * manual in INTRODUCTION.md. 37 | * The C++ interface is STD C++ '03 compliant and adheres to the 38 | * Google C++ Style Guide. 39 | 40 | * See_also: For the C interface see rdkafka.h 41 | * 42 | */ 43 | module rdkafkad; 44 | 45 | package import std.string : fromStringz, toStringz; 46 | package import core.stdc.errno; 47 | package import core.stdc.string; 48 | package import core.stdc.stdlib; 49 | package import core.stdc.ctype; 50 | package import core.sys.posix.sys.types; 51 | package import deimos.rdkafka; 52 | 53 | public import rdkafkad.config; 54 | public import rdkafkad.handler; 55 | public import rdkafkad.handler.simple_consumer; 56 | public import rdkafkad.handler.kafka_consumer; 57 | public import rdkafkad.handler.producer; 58 | public import rdkafkad.metadata; 59 | public import rdkafkad.topic; 60 | 61 | /** 62 | * librdkafka version 63 | * 64 | * Interpreted as hex \c MM.mm.rr.xx: 65 | * - MM = Major 66 | * - mm = minor 67 | * - rr = revision 68 | * - xx = pre-release id (0xff is the final release) 69 | * 70 | * E.g.: \c 0x000801ff.8.1 71 | * 72 | * Note: This value should only be used during compile time, 73 | * for runtime checks of version use version() 74 | */ 75 | enum RD_KAFKA_VERSION = 0x00090200; 76 | 77 | /** 78 | * Returns the librdkafka version as integer. 79 | * 80 | * See_also: See RD_KAFKA_VERSION for how to parse the integer format. 81 | */ 82 | int version_() nothrow @nogc 83 | { 84 | return rd_kafka_version(); 85 | } 86 | 87 | /** 88 | * Returns the librdkafka version as string. 89 | */ 90 | const(char)[] versionStr() nothrow @nogc 91 | { 92 | return fromStringz(rd_kafka_version_str); 93 | } 94 | 95 | /** 96 | * Returns a CSV list of the supported debug contexts 97 | * for use with Conf::Set("debug", ..). 98 | */ 99 | const(char)[] getDebugContexts() nothrow @nogc 100 | { 101 | return rd_kafka_get_debug_contexts().fromStringz; 102 | } 103 | 104 | /** 105 | * Wait for all rd_kafka_t objects to be destroyed. 106 | * 107 | * 0 if all kafka objects are now destroyed, or -1 if the 108 | * timeout was reached. 109 | * Since RdKafka handle deletion is an asynch operation the 110 | * \p wait_destroyed() function can be used for applications where 111 | * a clean shutdown is required. 112 | */ 113 | auto waitDestroyed(int timeout_ms) 114 | { 115 | int ret; 116 | ret = rd_kafka_wait_destroyed(timeout_ms); 117 | return ret; 118 | } 119 | 120 | /** 121 | * Error codes. 122 | * 123 | * The negative error codes delimited by two underscores 124 | * (\c _..) denotes errors internal to librdkafka and are 125 | * displayed as \c \"Local: \\", while the error codes 126 | * delimited by a single underscore (\c ERR_..) denote broker 127 | * errors and are displayed as \c \"Broker: \\". 128 | * 129 | * See_also: Use err2str() to translate an error code a human readable string 130 | */ 131 | enum ErrorCode 132 | { 133 | /* Internal errors to rdkafka: */ 134 | /** Begin internal error codes */ 135 | begin = -200, 136 | /** Received message is incorrect */ 137 | bad_msg = -199, 138 | /** Bad/unknown compression */ 139 | bad_compression = -198, 140 | /** Broker is going away */ 141 | destroy = -197, 142 | /** Generic failure */ 143 | fail = -196, 144 | /** Broker transport failure */ 145 | transport = -195, 146 | /** Critical system resource */ 147 | crit_sys_resource = -194, 148 | /** Failed to resolve broker */ 149 | resolve = -193, 150 | /** Produced message timed out*/ 151 | msg_timed_out = -192, 152 | /** Reached the end of the topic+partition queue on 153 | * the broker. Not really an error. */ 154 | partition_eof = -191, 155 | /** Permanent: Partition does not exist in cluster. */ 156 | unknown_partition = -190, 157 | /** File or filesystem error */ 158 | fs = -189, 159 | /** Permanent: Topic does not exist in cluster. */ 160 | unknown_topic = -188, 161 | /** All broker connections are down. */ 162 | all_brokers_down = -187, 163 | /** Invalid argument, or invalid configuration */ 164 | invalid_arg = -186, 165 | /** Operation timed out */ 166 | timed_out = -185, 167 | /** Queue is full */ 168 | queue_full = -184, 169 | /** ISR count < required.acks */ 170 | isr_insuff = -183, 171 | /** Broker node update */ 172 | node_update = -182, 173 | /** SSL error */ 174 | ssl = -181, 175 | /** Waiting for coordinator to become available. */ 176 | wait_coord = -180, 177 | /** Unknown client group */ 178 | unknown_group = -179, 179 | /** Operation in progress */ 180 | in_progress = -178, 181 | /** Previous operation in progress, wait for it to finish. */ 182 | prev_in_progress = -177, 183 | /** This operation would interfere with an existing subscription */ 184 | existing_subscription = -176, 185 | /** Assigned partitions (rebalance_cb) */ 186 | assign_partitions = -175, 187 | /** Revoked partitions (rebalance_cb) */ 188 | revoke_partitions = -174, 189 | /** Conflicting use */ 190 | conflict = -173, 191 | /** Wrong state */ 192 | state = -172, 193 | /** Unknown protocol */ 194 | unknown_protocol = -171, 195 | /** Not implemented */ 196 | not_implemented = -170, 197 | /** Authentication failure*/ 198 | authentication = -169, 199 | /** No stored offset */ 200 | no_offset = -168, 201 | /** Outdated */ 202 | outdated = -167, 203 | /** Timed out in queue */ 204 | timed_out_queue = -166, 205 | 206 | /** End internal error codes */ 207 | end = -100, 208 | 209 | /* Kafka broker errors: */ 210 | /** Unknown broker error */ 211 | unknown = -1, 212 | /** Success */ 213 | no_error, 214 | /** Offset out of range */ 215 | offset_out_of_range = 1, 216 | /** Invalid message */ 217 | invalid_msg = 2, 218 | /** Unknown topic or partition */ 219 | unknown_topic_or_part = 3, 220 | /** Invalid message size */ 221 | invalid_msg_size = 4, 222 | /** Leader not available */ 223 | leader_not_available = 5, 224 | /** Not leader for partition */ 225 | not_leader_for_partition = 6, 226 | /** Request timed out */ 227 | request_timed_out = 7, 228 | /** Broker not available */ 229 | broker_not_available = 8, 230 | /** Replica not available */ 231 | replica_not_available = 9, 232 | /** Message size too large */ 233 | msg_size_too_large = 10, 234 | /** StaleControllerEpochCode */ 235 | stale_ctrl_epoch = 11, 236 | /** Offset metadata string too large */ 237 | offset_metadata_too_large = 12, 238 | /** Broker disconnected before response received */ 239 | network_exception = 13, 240 | /** Group coordinator load in progress */ 241 | group_load_in_progress = 14, 242 | /** Group coordinator not available */ 243 | group_coordinator_not_available = 15, 244 | /** Not coordinator for group */ 245 | not_coordinator_for_group = 16, 246 | /** Invalid topic */ 247 | topic_exception = 17, 248 | /** Message batch larger than configured server segment size */ 249 | record_list_too_large = 18, 250 | /** Not enough in-sync replicas */ 251 | not_enough_replicas = 19, 252 | /** Message(s) written to insufficient number of in-sync replicas */ 253 | not_enough_replicas_after_append = 20, 254 | /** Invalid required acks value */ 255 | invalid_required_acks = 21, 256 | /** Specified group generation id is not valid */ 257 | illegal_generation = 22, 258 | /** Inconsistent group protocol */ 259 | inconsistent_group_protocol = 23, 260 | /** Invalid group.id */ 261 | invalid_group_id = 24, 262 | /** Unknown member */ 263 | unknown_member_id = 25, 264 | /** Invalid session timeout */ 265 | invalid_session_timeout = 26, 266 | /** Group rebalance in progress */ 267 | rebalance_in_progress = 27, 268 | /** Commit offset data size is not valid */ 269 | invalid_commit_offset_size = 28, 270 | /** Topic authorization failed */ 271 | topic_authorization_failed = 29, 272 | /** Group authorization failed */ 273 | group_authorization_failed = 30, 274 | /** Cluster authorization failed */ 275 | cluster_authorization_failed = 31 276 | } 277 | 278 | /** 279 | * Returns a human readable representation of a kafka error. 280 | */ 281 | 282 | string err2str(ErrorCode err) nothrow @nogc 283 | { 284 | return cast(string)rd_kafka_err2str(cast(rd_kafka_resp_err_t) err).fromStringz; 285 | } 286 | 287 | /** 288 | * Delivery Report callback class 289 | * 290 | * The delivery report callback will be called once for each message 291 | * accepted by Producer::produce() (et.al) with 292 | * Message::err() set to indicate the result of the produce request. 293 | * 294 | * The callback is called when a message is succesfully produced or 295 | * if librdkafka encountered a permanent failure, or the retry counter for 296 | * temporary errors has been exhausted. 297 | * 298 | * An application must call poll() at regular intervals to 299 | * serve queued delivery report callbacks. 300 | 301 | */ 302 | alias DeliveryReportCb = void delegate(ref Message message) nothrow @nogc; 303 | 304 | /** 305 | * Partitioner callback class 306 | * 307 | * Generic partitioner callback class for implementing custom partitioners. 308 | * 309 | * See_also: Conf::set() \c "partitioner_cb" 310 | */ 311 | /** 312 | * Partitioner callback 313 | * 314 | * Return the partition to use for \p key in \p topic. 315 | * 316 | * The \p msg_opaque is the same \p msg_opaque provided in the 317 | * Producer::produce() call. 318 | * 319 | * Note: \p key may be null or the empty. 320 | * 321 | * Must return a value between 0 and \p partition_cnt (non-inclusive). 322 | * May return RD_KAFKA_PARTITION_UA (-1) if partitioning failed. 323 | * 324 | * See_also: The callback may use Topic::partition_available() to check 325 | * if a partition has an active leader broker. 326 | */ 327 | alias PartitionerCb = int delegate(const Topic topic, const(void)[] key, 328 | int partition_cnt, void* msg_opaque) nothrow @nogc; 329 | 330 | /** 331 | * Variant partitioner with key pointer 332 | * 333 | */ 334 | alias PartitionerKeyPointerCb = int delegate(const Topic topic, const(void)* key, 335 | size_t key_len, int partition_cnt, void* msg_opaque) nothrow @nogc; 336 | 337 | /** 338 | * Event callback class 339 | * 340 | * Events are a generic interface for propagating errors, statistics, logs, etc 341 | * from librdkafka to the application. 342 | * 343 | * See_also: Event 344 | */ 345 | alias EventCb = void delegate(ref Event event) nothrow @nogc; 346 | 347 | /** 348 | * Event object class as passed to the EventCb callback. 349 | */ 350 | struct Event 351 | { 352 | nothrow @nogc: 353 | /** Event type */ 354 | enum Type 355 | { 356 | error, /**< Event is an error condition */ 357 | stats, /**< Event is a statistics JSON document */ 358 | log, /**< Event is a log message */ 359 | throttle /**< Event is a throttle level signaling from the broker */ 360 | } 361 | 362 | /** LOG severities (conforms to syslog(3) severities) */ 363 | enum Severity 364 | { 365 | emerg, 366 | alert = 1, 367 | critical = 2, 368 | error = 3, 369 | warning = 4, 370 | notice = 5, 371 | info = 6, 372 | debug_ = 7 373 | } 374 | 375 | package this(Type type, ErrorCode err, Severity severity, const char* fac, const char* str) 376 | { 377 | type_ = type; 378 | err_ = err; 379 | severity_ = severity; 380 | fac_ = fac.fromStringz; 381 | str_ = str.fromStringz; 382 | } 383 | 384 | package this(Type type) 385 | { 386 | type_ = type; 387 | err_ = ErrorCode.no_error; 388 | severity_ = Severity.emerg; 389 | fac_ = ""; 390 | str_ = ""; 391 | } 392 | 393 | package Type type_; 394 | package ErrorCode err_; 395 | package Severity severity_; 396 | package const(char)[] fac_; 397 | package const(char)[] str_; /* reused for throttle broker_name */ 398 | package int id_; 399 | package int throttle_time_; 400 | 401 | @property: 402 | 403 | /* 404 | * Event Accessor methods 405 | */ 406 | 407 | /** 408 | * The event type 409 | * Note: Applies to all event types 410 | */ 411 | Type type() const 412 | { 413 | return type_; 414 | } 415 | 416 | /** 417 | * Event error, if any. 418 | * Note: Applies to all event types except throttle 419 | */ 420 | ErrorCode err() const 421 | { 422 | return err_; 423 | } 424 | 425 | /** 426 | * Log severity level. 427 | * Note: Applies to LOG event type. 428 | */ 429 | Severity severity() const 430 | { 431 | return severity_; 432 | } 433 | 434 | /** 435 | * Log facility string. 436 | * Note: Applies to LOG event type. 437 | */ 438 | const(char)[] fac() const 439 | { 440 | return fac_; 441 | } 442 | 443 | /** 444 | * Log message string. 445 | * 446 | * \c LOG: Log message string. 447 | * \c STATS: JSON object (as string). 448 | * 449 | * Note: Applies to LOG event type. 450 | */ 451 | const(char)[] str() const 452 | { 453 | return str_; 454 | } 455 | 456 | /** 457 | * throttle time in milliseconds. 458 | * Note: Applies to throttle event type. 459 | */ 460 | int throttleTime() const 461 | { 462 | return throttle_time_; 463 | } 464 | 465 | /** 466 | * Throttling broker's name. 467 | * Note: Applies to throttle event type. 468 | */ 469 | const(char)[] brokerName() const 470 | { 471 | if (type_ == Type.throttle) 472 | return str_; 473 | else 474 | return ""; 475 | } 476 | 477 | /** 478 | * Throttling broker's id. 479 | * Note: Applies to throttle event type. 480 | */ 481 | int brokerId() const 482 | { 483 | return id_; 484 | } 485 | } 486 | 487 | /** 488 | * Consume callback class 489 | */ 490 | /** 491 | * The consume callback is used with 492 | * Consumer::consumeCallback() 493 | * methods and will be called for each consumed \p message. 494 | * 495 | * The callback interface is optional but provides increased performance. 496 | */ 497 | alias ConsumeCb = void delegate(ref Message message) nothrow @nogc; 498 | 499 | /** 500 | * \b KafkaConsunmer: Rebalance callback class 501 | */ 502 | /** 503 | * Group rebalance callback for use with KafkaConsunmer 504 | * 505 | * Registering a \p rebalance_cb turns off librdkafka's automatic 506 | * partition assignment/revocation and instead delegates that responsibility 507 | * to the application's \p rebalance_cb. 508 | * 509 | * The rebalance callback is responsible for updating librdkafka's 510 | * assignment set based on the two events: ASSIGN_PARTITIONS 511 | * and REVOKE_PARTITIONS but should also be able to handle 512 | * arbitrary rebalancing failures where \p err is neither of those. 513 | * Note: In this latter case (arbitrary error), the application must 514 | * call unassign() to synchronize state. 515 | * 516 | * Without a rebalance callback this is done automatically by librdkafka 517 | * but registering a rebalance callback gives the application flexibility 518 | * in performing other operations along with the assinging/revocation, 519 | * such as fetching offsets from an alternate location (on assign) 520 | * or manually committing offsets (on revoke). 521 | */ 522 | alias RebalanceCb = void delegate(KafkaConsumer consumer, ErrorCode err, 523 | ref TopicPartition[] partitions) nothrow @nogc; 524 | 525 | /** 526 | * Offset Commit callback class 527 | */ 528 | /** 529 | * Set offset commit callback for use with consumer groups 530 | * 531 | * The results of automatic or manual offset commits will be scheduled 532 | * for this callback and is served by consume() 533 | * or commitSync() 534 | * 535 | * If no partitions had valid offsets to commit this callback will be called 536 | * with \p err == NO_OFFSET which is not to be considered an error. 537 | * 538 | * The \p offsets list contains per-partition information: 539 | * - \c topic The topic committed 540 | * - \c partition The partition committed 541 | * - \c offset: Committed offset (attempted) 542 | * - \c err: Commit error 543 | */ 544 | 545 | alias OffsetCommitCb = void delegate(ErrorCode err, ref TopicPartition[] offsets) nothrow @nogc; 546 | 547 | /** 548 | * \b Portability: SocketCb callback class 549 | * 550 | */ 551 | /** 552 | * Socket callback 553 | * 554 | * The socket callback is responsible for opening a socket 555 | * according to the supplied \p domain, \p type and \p protocol. 556 | * The socket shall be created with \c CLOEXEC set in a racefree fashion, if 557 | * possible. 558 | * 559 | * It is typically not required to register an alternative socket 560 | * implementation 561 | * 562 | * The socket file descriptor or -1 on error (\c errno must be set) 563 | */ 564 | alias SocketCb = int function(int domain, int type, int protocol) nothrow @nogc; 565 | 566 | 567 | /** 568 | * Message object 569 | * 570 | * This object represents either a single consumed or produced message, 571 | * or an event (\p err() is set). 572 | * 573 | * An application must check Message::err() to see if the 574 | * object is a proper message (error is ErrorCode.no_error) or a 575 | * an error event. 576 | * 577 | */ 578 | struct Message 579 | { 580 | /** 581 | * Message timestamp object 582 | * 583 | * Represents the number of milliseconds since the epoch (UTC). 584 | * 585 | * The Type dictates the timestamp type or origin. 586 | * 587 | * Note: Requires Apache Kafka broker version >= 0.10.0 588 | * 589 | */ 590 | static struct Timestamp 591 | { 592 | enum Type 593 | { 594 | not_available, /**< Timestamp not available */ 595 | create_time, /**< Message creation time (source) */ 596 | log_append_time /**< Message log append time (broker) */ 597 | } 598 | 599 | long timestamp; /**< Milliseconds since epoch (UTC). */ 600 | Type type; /**< Timestamp type */ 601 | } 602 | 603 | 604 | @disable this(this); 605 | nothrow @nogc: 606 | 607 | bool isNull() @safe pure 608 | { 609 | return rkmessage_ is null; 610 | } 611 | 612 | ~this() 613 | { 614 | if (free_rkmessage_ && rkmessage_) 615 | rd_kafka_message_destroy(cast(rd_kafka_message_t*) rkmessage_); 616 | } 617 | 618 | this(Topic topic, rd_kafka_message_t* rkmessage, bool dofree = true) 619 | { 620 | topic_ = topic; 621 | rkmessage_ = rkmessage; 622 | free_rkmessage_ = dofree; 623 | } 624 | 625 | this(Topic topic, const rd_kafka_message_t* rkmessage) 626 | { 627 | assert(rkmessage); 628 | topic_ = topic; 629 | rkmessage_ = rkmessage; 630 | err_ = cast(ErrorCode) rkmessage.err; 631 | } 632 | 633 | this(rd_kafka_message_t* rkmessage) 634 | { 635 | assert(rkmessage); 636 | rkmessage_ = rkmessage; 637 | free_rkmessage_ = true; 638 | err_ = cast(ErrorCode) rkmessage.err; 639 | if (rkmessage.rkt) 640 | { 641 | /* Possibly null */ 642 | topic_ = cast(Topic) rd_kafka_topic_opaque(rkmessage.rkt); 643 | } 644 | } 645 | 646 | /* Create errored message */ 647 | this(Topic topic, ErrorCode err) 648 | { 649 | topic_ = topic; 650 | err_ = err; 651 | } 652 | 653 | /** The error string if object represent an error event, 654 | * else an empty string. */ 655 | string errstr() const 656 | { 657 | /* FIXME: If there is an error string in payload (for consume_cb) 658 | * it wont be shown since 'payload' is reused for errstr 659 | * and we cant distinguish between consumer and producer. 660 | * For the producer case the payload needs to be the original 661 | * payload pointer. */ 662 | return err2str(err_); 663 | } 664 | 665 | /** The error code if object represents an error event, else 0. */ 666 | ErrorCode err() const 667 | { 668 | return err_; 669 | } 670 | 671 | /** the Topic object for a message (if applicable), 672 | * or null if a corresponding Topic object has not been 673 | * explicitly created with Topic::create(). 674 | * In this case use topic_name() instead. */ 675 | const(Topic) topic() const 676 | { 677 | return topic_; 678 | } 679 | /** Topic name (if applicable, else empty string) */ 680 | const(char)[] topicName() const 681 | { 682 | if (rkmessage_.rkt) 683 | return rd_kafka_topic_name(rkmessage_.rkt).fromStringz; 684 | else 685 | return ""; 686 | } 687 | /** Partition (if applicable) */ 688 | int partition() const 689 | { 690 | return rkmessage_.partition; 691 | } 692 | /** Message payload (if applicable) */ 693 | const(void)[] payload() const 694 | { 695 | return rkmessage_.payload[0 .. rkmessage_.len]; 696 | } 697 | 698 | /** Message key as string (if applicable) */ 699 | const(void)[] key() const 700 | { 701 | return (cast(const(char)*) rkmessage_.key)[0 .. rkmessage_.key_len]; 702 | } 703 | 704 | /** Message or error offset (if applicable) */ 705 | long offset() const 706 | { 707 | return rkmessage_.offset; 708 | } 709 | 710 | /** Message timestamp (if applicable) */ 711 | Timestamp timestamp() const 712 | { 713 | Timestamp ts; 714 | rd_kafka_timestamp_type_t tstype; 715 | ts.timestamp = rd_kafka_message_timestamp(rkmessage_, &tstype); 716 | ts.type = cast(Timestamp.Type) tstype; 717 | return ts; 718 | } 719 | 720 | /** The \p msg_opaque as provided to Producer::produce() */ 721 | const(void)* msgOpaque() const 722 | { 723 | return rkmessage_._private; 724 | } 725 | 726 | package: 727 | Topic topic_; 728 | const (rd_kafka_message_t)* rkmessage_; 729 | bool free_rkmessage_; 730 | ErrorCode err_; 731 | } 732 | --------------------------------------------------------------------------------