├── LICENSE ├── README.md ├── consumer.py └── producer.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 CloudKarafka 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apache Kafka example for Python 2 | 3 | 4 | ## Getting started 5 | 6 | Setup your free Apache Kafka instance here: https://www.cloudkarafka.com 7 | 8 | Configuration 9 | 10 | * `export CLOUDKARAFKA_BROKERS="host1:9094,host2:9094,host3:9094"` 11 | Hostnames can be found in the Details view in for your CloudKarafka instance. 12 | * `export CLOUDKARAFKA_USERNAME="username"` 13 | Username can be found in the Details view in for your CloudKarafka instance. 14 | * `export CLOUDKARAFKA_PASSWORD="password"` 15 | Password can be found in the Details view in for your CloudKarafka instance. 16 | * `export CLOUDKARAFKA_TOPIC="username-topic"` 17 | Topic should be the same as your username followed by a dash before the topic. 18 | 19 | These export commands must be run in both of the terminal windows below. 20 | 21 | ``` 22 | git clone https://github.com/CloudKarafka/python-kafka-example.git 23 | cd python-kafka-example` 24 | pip install confluent_kafka 25 | python consumer.py 26 | ``` 27 | 28 | Open another terminal window and `cd` into same directory and run `python producer.py`. 29 | Send your messages by pressing your system's EOF key sequence. (ctrl-d in bash) 30 | 31 | ## Adding a Root CA 32 | 33 | In some cases the CloudKarafka Root CA may need to be manually added to the example, particularly if you are seeing the error: 34 | ``` 35 | Failed to verify broker certificate: unable to get local issuer certificate 36 | ``` 37 | returned when you run the example. If this is the case you will need to download the [CloudKarakfa Root CA from our FAQ page](https://www.cloudkarafka.com/docs/faq.html) and place it in the python-kafka-example directory, then add the following line into the `conf {...}` section: 38 | ``` 39 | 'ssl.ca.location': 'cloudkarafka.ca' 40 | ``` 41 | This should resolve the error and allow for successful connection to the server. 42 | -------------------------------------------------------------------------------- /consumer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | from confluent_kafka import Consumer, KafkaException, KafkaError 5 | 6 | if __name__ == '__main__': 7 | topics = os.environ['CLOUDKARAFKA_TOPIC'].split(",") 8 | 9 | # Consumer configuration 10 | # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md 11 | conf = { 12 | 'bootstrap.servers': os.environ['CLOUDKARAFKA_BROKERS'], 13 | 'group.id': "%s-consumer" % os.environ['CLOUDKARAFKA_USERNAME'], 14 | 'session.timeout.ms': 6000, 15 | 'default.topic.config': {'auto.offset.reset': 'smallest'}, 16 | 'security.protocol': 'SASL_SSL', 17 | 'sasl.mechanisms': 'SCRAM-SHA-256', 18 | 'sasl.username': os.environ['CLOUDKARAFKA_USERNAME'], 19 | 'sasl.password': os.environ['CLOUDKARAFKA_PASSWORD'] 20 | } 21 | 22 | c = Consumer(**conf) 23 | c.subscribe(topics) 24 | try: 25 | while True: 26 | msg = c.poll(timeout=1.0) 27 | if msg is None: 28 | continue 29 | if msg.error(): 30 | # Error or event 31 | if msg.error().code() == KafkaError._PARTITION_EOF: 32 | # End of partition event 33 | sys.stderr.write('%% %s [%d] reached end at offset %d\n' % 34 | (msg.topic(), msg.partition(), msg.offset())) 35 | elif msg.error(): 36 | # Error 37 | raise KafkaException(msg.error()) 38 | else: 39 | # Proper message 40 | sys.stderr.write('%% %s [%d] at offset %d with key %s:\n' % 41 | (msg.topic(), msg.partition(), msg.offset(), 42 | str(msg.key()))) 43 | print(msg.value()) 44 | 45 | except KeyboardInterrupt: 46 | sys.stderr.write('%% Aborted by user\n') 47 | 48 | # Close down consumer to commit final offsets. 49 | c.close() 50 | -------------------------------------------------------------------------------- /producer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | from confluent_kafka import Producer 5 | 6 | if __name__ == '__main__': 7 | topic = os.environ['CLOUDKARAFKA_TOPIC'].split(",")[0] 8 | 9 | # Consumer configuration 10 | # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md 11 | conf = { 12 | 'bootstrap.servers': os.environ['CLOUDKARAFKA_BROKERS'], 13 | 'session.timeout.ms': 6000, 14 | 'default.topic.config': {'auto.offset.reset': 'smallest'}, 15 | 'security.protocol': 'SASL_SSL', 16 | 'sasl.mechanisms': 'SCRAM-SHA-256', 17 | 'sasl.username': os.environ['CLOUDKARAFKA_USERNAME'], 18 | 'sasl.password': os.environ['CLOUDKARAFKA_PASSWORD'] 19 | } 20 | 21 | p = Producer(**conf) 22 | 23 | def delivery_callback(err, msg): 24 | if err: 25 | sys.stderr.write('%% Message failed delivery: %s\n' % err) 26 | else: 27 | sys.stderr.write('%% Message delivered to %s [%d]\n' % 28 | (msg.topic(), msg.partition())) 29 | 30 | for line in sys.stdin: 31 | try: 32 | p.produce(topic, line.rstrip(), callback=delivery_callback) 33 | except BufferError as e: 34 | sys.stderr.write('%% Local producer queue is full (%d messages awaiting delivery): try again\n' % 35 | len(p)) 36 | p.poll(0) 37 | 38 | sys.stderr.write('%% Waiting for %d deliveries\n' % len(p)) 39 | p.flush() 40 | --------------------------------------------------------------------------------