├── debian ├── docs ├── compat ├── source │ └── format ├── changelog ├── copyright ├── control └── rules ├── docs ├── requirements.txt ├── autoreload.py ├── index.rst ├── operations.rst ├── README.md └── monitoring.rst ├── .gitignore ├── .arcconfig ├── licenses ├── NOTICE.kafka.txt ├── NOTICE.log4j.txt ├── NOTICE.commons-cli.txt ├── NOTICE.zookeeper.txt ├── NOTICE.velocity.txt ├── NOTICE.joda-time.txt ├── NOTICE.avro.txt ├── LICENSE.jackson-annotations.txt ├── LICENSE.jackson-databind.txt ├── LICENSE.jackson-core.txt ├── LICENSE.jackson-jaxrs-providers.txt ├── NOTICE.commons-compress.txt ├── NOTICE.metrics.txt ├── netty │ └── licenses │ │ ├── LICENSE.compress-lzf.txt │ │ ├── LICENSE.abstractnodequeue.txt │ │ ├── LICENSE.jfastlz.txt │ │ ├── LICENSE.libdivsufsort.txt │ │ ├── LICENSE.jbzip2.txt │ │ ├── LICENSE.bouncycastle.txt │ │ ├── LICENSE.slf4j.txt │ │ ├── LICENSE.jzlib.txt │ │ ├── LICENSE.snappy.txt │ │ ├── LICENSE.base64.txt │ │ ├── LICENSE.deque.txt │ │ ├── LICENSE.jsr166y.txt │ │ ├── LICENSE.webbit.txt │ │ └── LICENSE.protobuf.txt ├── NOTICE.snappy-java.txt ├── COPYRIGHT.hibernate.txt ├── NOTICE.jackson-core.txt ├── NOTICE.jackson-databind.txt ├── NOTICE.jackson-jaxrs-providers.txt ├── NOTICE.confluent-common.txt ├── LICENSE.mit.txt ├── NOTICE.lz4-java.txt ├── LICENSE.bsd.txt ├── NOTICE.jetty.txt └── NOTICE.confluent-rest-utils.txt ├── config ├── log4j.properties └── kafka-rest.properties ├── src ├── main │ ├── resources │ │ └── log4j.properties │ └── java │ │ └── io │ │ └── confluent │ │ └── kafkarest │ │ ├── entities │ │ ├── ProduceRecord.java │ │ ├── SpoolChannel.java │ │ ├── SpoolMode.java │ │ ├── AvroConsumerRecord.java │ │ ├── TopicProduceRequest.java │ │ ├── PartitionProduceRequest.java │ │ ├── SpoolShard.java │ │ ├── EntityUtils.java │ │ ├── EmbeddedFormat.java │ │ ├── TopicProduceRecord.java │ │ ├── ProduceResponse.java │ │ ├── CreateConsumerInstanceResponse.java │ │ ├── ProduceRecordBase.java │ │ ├── SpoolMessage.java │ │ ├── BrokerList.java │ │ ├── AvroProduceRecord.java │ │ ├── SchemaHolder.java │ │ ├── AvroTopicProduceRecord.java │ │ ├── Broker.java │ │ ├── PartitionReplica.java │ │ ├── BinaryTopicProduceRecord.java │ │ ├── BinaryProduceRecord.java │ │ ├── Partition.java │ │ ├── Topic.java │ │ ├── BinaryConsumerRecord.java │ │ ├── TopicPartitionOffset.java │ │ ├── ConsumerRecord.java │ │ ├── PartitionOffset.java │ │ └── ConsumerInstanceConfig.java │ │ ├── SystemTime.java │ │ ├── ConsumerWorkerReadCallback.java │ │ ├── Time.java │ │ ├── exceptions │ │ ├── SpoolException.java │ │ └── ZkExceptionMapper.java │ │ ├── converters │ │ └── ConversionException.java │ │ ├── RecordMetadataOrException.java │ │ ├── ConsumerRecordAndSize.java │ │ ├── RestProducer.java │ │ ├── UriUtils.java │ │ ├── Main.java │ │ ├── Context.java │ │ ├── ConsumerInstanceId.java │ │ ├── resources │ │ ├── RootResource.java │ │ ├── BrokersResource.java │ │ └── SpoolResource.java │ │ ├── BinaryRestProducer.java │ │ ├── BinaryConsumerState.java │ │ ├── ConsumerTopicState.java │ │ ├── AvroConsumerState.java │ │ ├── ProduceTask.java │ │ ├── Versions.java │ │ ├── KafkaRestApplication.java │ │ ├── ConsumerWorker.java │ │ ├── AvroRestProducer.java │ │ └── SpoolRecord.java ├── test │ ├── resources │ │ └── log4j.properties │ └── java │ │ └── io │ │ └── confluent │ │ └── kafkarest │ │ ├── mock │ │ └── MockTime.java │ │ ├── integration │ │ ├── TestKafkaRestApplication.java │ │ ├── ProducerTopicAutoCreationTest.java │ │ ├── AbstractProducerTest.java │ │ └── ConsumerTimeoutTest.java │ │ └── unit │ │ ├── BrokersResourceTest.java │ │ ├── UriUtilsTest.java │ │ └── RootResourceTest.java └── assembly │ ├── development.xml │ ├── standalone.xml │ └── package.xml ├── bin ├── kafka-rest-stop ├── kafka-rest-start ├── kafka-rest-stop-service └── kafka-rest-run-class └── NOTICE /debian/docs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 1.0 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | sphinx_rtd_theme 3 | sphinxcontrib-httpdomain 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Build products 3 | target/ 4 | docs/_build/ 5 | 6 | # IntelliJ data 7 | *.iml 8 | .idea/ 9 | .ipr 10 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | kafka-rest (0) unstable; urgency=low 2 | 3 | * fork kafka-rest 4 | 5 | -- Norbert Hu Fri, 13 Mar 2015 00:00:00 +0000 6 | -------------------------------------------------------------------------------- /.arcconfig: -------------------------------------------------------------------------------- 1 | { 2 | "project_id": "kafka-rest", 3 | "conduit_uri": "https://code.uberinternal.com/", 4 | "git.default-relative-commit": "origin/master" 5 | } 6 | -------------------------------------------------------------------------------- /docs/autoreload.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from livereload import Server, shell 3 | 4 | server = Server() 5 | server.watch('*.rst', shell('make html')) 6 | server.serve() 7 | -------------------------------------------------------------------------------- /licenses/NOTICE.kafka.txt: -------------------------------------------------------------------------------- 1 | Apache Kafka 2 | Copyright 2012 The Apache Software Foundation. 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /licenses/NOTICE.log4j.txt: -------------------------------------------------------------------------------- 1 | Apache log4j 2 | Copyright 2010 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /licenses/NOTICE.commons-cli.txt: -------------------------------------------------------------------------------- 1 | Apache Commons CLI 2 | Copyright 2001-2014 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /licenses/NOTICE.zookeeper.txt: -------------------------------------------------------------------------------- 1 | Apache ZooKeeper 2 | Copyright 2009-2014 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /licenses/NOTICE.velocity.txt: -------------------------------------------------------------------------------- 1 | Apache Velocity 2 | 3 | Copyright (C) 2000-2007 The Apache Software Foundation 4 | 5 | This product includes software developed at 6 | The Apache Software Foundation (http://www.apache.org/). 7 | -------------------------------------------------------------------------------- /config/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n -------------------------------------------------------------------------------- /licenses/NOTICE.joda-time.txt: -------------------------------------------------------------------------------- 1 | ============================================================================= 2 | = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = 3 | ============================================================================= 4 | This product includes software developed by 5 | Joda.org (http://www.joda.org/). 6 | -------------------------------------------------------------------------------- /licenses/NOTICE.avro.txt: -------------------------------------------------------------------------------- 1 | Apache Avro 2 | Copyright 2010 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | C JSON parsing provided by Jansson and 8 | written by Petri Lehtinen. The original software is 9 | available from http://www.digip.org/jansson/. 10 | -------------------------------------------------------------------------------- /licenses/LICENSE.jackson-annotations.txt: -------------------------------------------------------------------------------- 1 | This copy of Jackson JSON processor annotations is licensed under the 2 | Apache (Software) License, version 2.0 ("the License"). 3 | See the License for details about distribution rights, and the 4 | specific rights regarding derivate works. 5 | 6 | You may obtain a copy of the License at: 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | -------------------------------------------------------------------------------- /licenses/LICENSE.jackson-databind.txt: -------------------------------------------------------------------------------- 1 | This copy of Jackson JSON processor databind module is licensed under the 2 | Apache (Software) License, version 2.0 ("the License"). 3 | See the License for details about distribution rights, and the 4 | specific rights regarding derivate works. 5 | 6 | You may obtain a copy of the License at: 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | -------------------------------------------------------------------------------- /licenses/LICENSE.jackson-core.txt: -------------------------------------------------------------------------------- 1 | This copy of Jackson JSON processor streaming parser/generator is licensed under the 2 | Apache (Software) License, version 2.0 ("the License"). 3 | See the License for details about distribution rights, and the 4 | specific rights regarding derivate works. 5 | 6 | You may obtain a copy of the License at: 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | -------------------------------------------------------------------------------- /licenses/LICENSE.jackson-jaxrs-providers.txt: -------------------------------------------------------------------------------- 1 | This copy of Jackson JSON processor databind module is licensed under the 2 | Apache (Software) License, version 2.0 ("the License"). 3 | See the License for details about distribution rights, and the 4 | specific rights regarding derivate works. 5 | 6 | You may obtain a copy of the License at: 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Kafka REST Proxy documentation master file, created by 2 | sphinx-quickstart on Wed Dec 17 14:17:15 2014. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Kafka REST Proxy 7 | ================ 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 3 13 | 14 | intro 15 | api 16 | config 17 | operations 18 | -------------------------------------------------------------------------------- /licenses/NOTICE.commons-compress.txt: -------------------------------------------------------------------------------- 1 | Apache Commons Compress 2 | Copyright 2002-2015 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | The files in the package org.apache.commons.compress.archivers.sevenz 8 | were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), 9 | which has been placed in the public domain: 10 | 11 | "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) 12 | -------------------------------------------------------------------------------- /licenses/NOTICE.metrics.txt: -------------------------------------------------------------------------------- 1 | Metrics 2 | Copyright 2010-2012 Coda Hale and Yammer, Inc. 3 | 4 | This product includes software developed by Coda Hale and Yammer, Inc. 5 | 6 | This product includes code derived from the JSR-166 project (ThreadLocalRandom), which was released 7 | with the following comments: 8 | 9 | Written by Doug Lea with assistance from members of JCP JSR-166 10 | Expert Group and released to the public domain, as explained at 11 | http://creativecommons.org/publicdomain/zero/1.0/ 12 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=OFF, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n 6 | 7 | log4j.logger.kafka=ERROR 8 | log4j.logger.org.apache.kafka=ERROR 9 | 10 | # zkclient can be verbose, during debugging it is common to adjust is separately 11 | log4j.logger.org.I0Itec.zkclient.ZkClient=ERROR 12 | log4j.logger.org.apache.zookeeper=ERROR 13 | -------------------------------------------------------------------------------- /docs/operations.rst: -------------------------------------------------------------------------------- 1 | .. _kafkarest_operations: 2 | 3 | Operations 4 | ========== 5 | 6 | In this document, you will learn how to manage a REST Proxy cluster. 7 | 8 | Here, we will cover two main topics: 9 | 10 | * Deploying your cluster to production, including best practices and important configurations that should (or should not!) be changed 11 | * Monitoring your cluster’s vital statistics, understanding which behaviors are normal and which should be cause for alarm, and interpreting various stats 12 | 13 | .. toctree:: 14 | :maxdepth: 3 15 | 16 | deployment 17 | monitoring 18 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.compress-lzf.txt: -------------------------------------------------------------------------------- 1 | Copyright 2009-2010 Ning, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | use this file except in compliance with the License. You may obtain a copy of 5 | the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS,WITHOUT 9 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | License for the specific language governing permissions and limitations under 11 | the License. 12 | -------------------------------------------------------------------------------- /licenses/NOTICE.snappy-java.txt: -------------------------------------------------------------------------------- 1 | This product includes software developed by Google 2 | Snappy: http://code.google.com/p/snappy/ (New BSD License) 3 | 4 | This product includes software developed by Apache 5 | PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ 6 | (Apache 2.0 license) 7 | 8 | This library containd statically linked libstdc++. This inclusion is allowed by 9 | "GCC RUntime Library Exception" 10 | http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html 11 | 12 | == Contributors == 13 | * Tatu Saloranta 14 | * Providing benchmark suite 15 | * Alec Wysoker 16 | * Performance and memory usage improvement 17 | -------------------------------------------------------------------------------- /licenses/COPYRIGHT.hibernate.txt: -------------------------------------------------------------------------------- 1 | Adam Stawicki 2 | Alaa Nassef 3 | Andrey Rodionov 4 | Brent Douglas 5 | Carlos Vara 6 | Dag Hovland 7 | Davide Marchignoli 8 | Doug Lea 9 | Emmanuel Bernard 10 | Efthymis Sarbanis 11 | Federico 12 | Federico Mancini 13 | Gavin King 14 | George Gastaldi 15 | Gerhard Petracek 16 | Gunnar Morling 17 | Hardy Ferentschik 18 | Henno Vermeulen 19 | Jason T. Greene 20 | Juraci Krohling 21 | Justin Nauman 22 | Kevin Pollet 23 | Khalid Alqinyah 24 | Leonardo Loch Zanivan 25 | Mark Hobson 26 | Paolo Perrotta 27 | Pete Muir 28 | Sanne Grinovero 29 | Shane Bryzak 30 | Shelly McGowan 31 | Steve Ebersole 32 | Strong Liu 33 | Victor Rezende dos Santos 34 | -------------------------------------------------------------------------------- /bin/kafka-rest-stop: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Confluent Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | exec $(dirname $0)/kafka-rest-stop-service 'kafkarest\.Main' 18 | -------------------------------------------------------------------------------- /bin/kafka-rest-start: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Confluent Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | exec $(dirname $0)/kafka-rest-run-class io.confluent.kafkarest.Main "$@" 18 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.abstractnodequeue.txt: -------------------------------------------------------------------------------- 1 | This software is licensed under the Apache 2 license, quoted below. 2 | 3 | Copyright 2009-2013 Typesafe Inc. [http://www.typesafe.com] 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | use this file except in compliance with the License. You may obtain a copy of 7 | the License at 8 | 9 | [http://www.apache.org/licenses/LICENSE-2.0] 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | License for the specific language governing permissions and limitations under 15 | the License. -------------------------------------------------------------------------------- /config/kafka-rest.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright 2015 Confluent Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | ## 16 | 17 | #id=kafka-rest-test-server 18 | #schema.registry.url=http://localhost:8081 19 | #zookeeper.connect=localhost:2181 20 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: kafka-rest 3 | Source: http://confluent.io/downloads/ 4 | 5 | Files: * 6 | Copyright: The project is licensed under the Apache 2 license. 7 | License: Apache License, Version 2.0 8 | You may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 10 | . 11 | Unless required by applicable law or agreed to in writing, 12 | software distributed under the License is distributed on an 13 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations 16 | under the License. 17 | -------------------------------------------------------------------------------- /src/assembly/development.xml: -------------------------------------------------------------------------------- 1 | 5 | 7 | development 8 | 9 | dir 10 | 11 | false 12 | 13 | 14 | share/java/kafka-rest/ 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | This documentation is built using [Sphinx](http://sphinx-doc.org). It also uses some extensions for theming and REST API 2 | documentation support. 3 | 4 | Start by installing the requirements: 5 | 6 | pip install -r requirements.txt 7 | 8 | Then you can generate the HTML version of the docs: 9 | 10 | make html 11 | 12 | The root of the documentation will be at `_build/html/index.html` 13 | 14 | While editing the documentation, you can get a live preview using python-livepreview. Install the Python library: 15 | 16 | pip install livereload 17 | 18 | Then run the monitoring script in the background: 19 | 20 | python autoreload.py & 21 | 22 | If you install the [browser extensions](http://livereload.com/) then everything should update every time any files are 23 | saved without any manual steps on your part. -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: kafka-rest 2 | Section: database 3 | Priority: extra 4 | Maintainer: Norbert Hu 5 | Build-Depends: debhelper (>= 8.0.0), openjdk-7-jdk, maven 6 | Standards-Version: 3.9.3 7 | Homepage: http://confluent.io/docs/current/index.html 8 | 9 | Package: kafka-rest 10 | Architecture: all 11 | Depends: openjdk-7-jre-headless 12 | Description: The Kafka REST Proxy provides a RESTful interface to a 13 | Kafka cluster. It makes it easy to produce and consume messages, view 14 | the state of the cluster, and perform administrative actions without 15 | using the native Kafka protocol or clients. Examples of use cases 16 | include reporting data to Kafka from any frontend app built in any 17 | language, ingesting messages into a stream processing framework that 18 | doesn’t yet support Kafka, and scripting administrative actions. -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/ProduceRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | public interface ProduceRecord { 20 | 21 | public K getKey(); 22 | 23 | public V getValue(); 24 | } -------------------------------------------------------------------------------- /licenses/NOTICE.jackson-core.txt: -------------------------------------------------------------------------------- 1 | # Jackson JSON processor 2 | 3 | Jackson is a high-performance, Free/Open Source JSON processing library. 4 | It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has 5 | been in development since 2007. 6 | It is currently developed by a community of developers, as well as supported 7 | commercially by FasterXML.com. 8 | 9 | ## Licensing 10 | 11 | Jackson core and extension components may licensed under different licenses. 12 | To find the details that apply to this artifact see the accompanying LICENSE file. 13 | For more information, including possible other licensing options, contact 14 | FasterXML.com (http://fasterxml.com). 15 | 16 | ## Credits 17 | 18 | A list of contributors may be found from CREDITS file, which is included 19 | in some artifacts (usually source distributions); but is always available 20 | from the source code management (SCM) system project uses. 21 | -------------------------------------------------------------------------------- /licenses/NOTICE.jackson-databind.txt: -------------------------------------------------------------------------------- 1 | # Jackson JSON processor 2 | 3 | Jackson is a high-performance, Free/Open Source JSON processing library. 4 | It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has 5 | been in development since 2007. 6 | It is currently developed by a community of developers, as well as supported 7 | commercially by FasterXML.com. 8 | 9 | ## Licensing 10 | 11 | Jackson core and extension components may be licensed under different licenses. 12 | To find the details that apply to this artifact see the accompanying LICENSE file. 13 | For more information, including possible other licensing options, contact 14 | FasterXML.com (http://fasterxml.com). 15 | 16 | ## Credits 17 | 18 | A list of contributors may be found from CREDITS file, which is included 19 | in some artifacts (usually source distributions); but is always available 20 | from the source code management (SCM) system project uses. 21 | -------------------------------------------------------------------------------- /licenses/NOTICE.jackson-jaxrs-providers.txt: -------------------------------------------------------------------------------- 1 | # Jackson JSON processor 2 | 3 | Jackson is a high-performance, Free/Open Source JSON processing library. 4 | It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has 5 | been in development since 2007. 6 | It is currently developed by a community of developers, as well as supported 7 | commercially by FasterXML.com. 8 | 9 | ## Licensing 10 | 11 | Jackson core and extension components may be licensed under different licenses. 12 | To find the details that apply to this artifact see the accompanying LICENSE file. 13 | For more information, including possible other licensing options, contact 14 | FasterXML.com (http://fasterxml.com). 15 | 16 | ## Credits 17 | 18 | A list of contributors may be found from CREDITS file, which is included 19 | in some artifacts (usually source distributions); but is always available 20 | from the source code management (SCM) system project uses. 21 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/SystemTime.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | public class SystemTime extends io.confluent.common.utils.SystemTime implements Time { 19 | 20 | @Override 21 | public void waitOn(Object on, long ms) throws InterruptedException { 22 | on.wait(ms); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/ConsumerWorkerReadCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import java.util.List; 20 | 21 | import io.confluent.kafkarest.entities.ConsumerRecord; 22 | 23 | public interface ConsumerWorkerReadCallback { 24 | 25 | public void onCompletion(List> records, Exception e); 26 | } 27 | -------------------------------------------------------------------------------- /licenses/NOTICE.confluent-common.txt: -------------------------------------------------------------------------------- 1 | The following libraries are included in packaged versions of this project: 2 | 3 | * Apache log4j 4 | * COPYRIGHT: Copyright 2010 The Apache Software Foundation 5 | * LICENSE: licenses/LICENSE.apache2.txt 6 | * NOTICE: licenses/NOTICE.log4j.txt 7 | * HOMEPAGE: http://logging.apache.org/log4j/1.2/ 8 | 9 | * Apache ZooKeeper 10 | * COPYRIGHT: Copyright 2009-2014 The Apache Software Foundation 11 | * LICENSE: licenses/LICENSE.apache2.txt 12 | * NOTICE: licenses/NOTICE.zookeeper.txt 13 | * HOMEPAGE: http://zookeeper.apache.org/ 14 | 15 | * jline 16 | * COPYRIGHT: Copyright (c) 2002-2006, Marc Prud'hommeaux 17 | * LICENSE: licenses/LICENSE.bsd.txt 18 | * HOMEPAGE: http://jline.sourceforge.net/ 19 | 20 | * SLF4J 21 | * COPYRIGHT: Copyright (c) 2004-2013 QOS.ch 22 | * LICENSE: licenses/LICENSE.mit.txt 23 | * HOMEPAGE: http://www.slf4j.org/ 24 | 25 | * ZkClient 26 | * LICENSE: licenses/LICENSE.apache2.txt 27 | * HOMEPAGE: https://github.com/sgroschupf/zkclient 28 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/SpoolChannel.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonFormat; 20 | 21 | /** 22 | * List of FileChannel dirs for SpoolProducer. 23 | */ 24 | @JsonFormat(shape = JsonFormat.Shape.STRING) 25 | public enum SpoolChannel { 26 | queue, 27 | retry, 28 | error 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/Time.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | public interface Time extends io.confluent.common.utils.Time { 20 | 21 | public long milliseconds(); 22 | 23 | public long nanoseconds(); 24 | 25 | public void sleep(long ms); 26 | 27 | // Equivalent to on.wait(ms) 28 | public void waitOn(Object on, long ms) throws InterruptedException; 29 | } -------------------------------------------------------------------------------- /licenses/LICENSE.mit.txt: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy of 2 | this software and associated documentation files (the "Software"), to deal in 3 | the Software without restriction, including without limitation the rights to 4 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 5 | the Software, and to permit persons to whom the Software is furnished to do so, 6 | subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all 9 | copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/SpoolMode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonFormat; 20 | 21 | /** 22 | * Options for SpoolProducer to dictate how messages are being buffered in the local FileChannel. 23 | */ 24 | @JsonFormat(shape = JsonFormat.Shape.STRING) 25 | public enum SpoolMode { 26 | DISABLED, // synchronous (DEFAULT) 27 | SYNC, // synchronous with spool fallback 28 | ASYNC // asynchronous via spool 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/exceptions/SpoolException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.exceptions; 18 | 19 | import org.apache.kafka.common.errors.RetriableException; 20 | 21 | public class SpoolException extends RetriableException { 22 | public final int failures; 23 | 24 | public SpoolException(int failures) { 25 | this.failures = failures; 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return "Back pressuring after " + failures + " consecutive failures"; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.jfastlz.txt: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2009 William Kinney 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without 8 | restriction, including without limitation the rights to use, 9 | copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the 11 | Software is furnished to do so, subject to the following 12 | conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | OTHER DEALINGS IN THE SOFTWARE. 25 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.libdivsufsort.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.jbzip2.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010-2011 Matthew J. Francis and Contributors of the jbzip2 Project 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/AvroConsumerRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import com.fasterxml.jackson.databind.JsonNode; 21 | 22 | public class AvroConsumerRecord extends ConsumerRecord { 23 | 24 | public AvroConsumerRecord( 25 | @JsonProperty("key") JsonNode key, @JsonProperty("value") JsonNode value, 26 | @JsonProperty("partition") int partition, @JsonProperty("offset") long offset 27 | ) { 28 | super(key, value, partition, offset); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/converters/ConversionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.converters; 18 | 19 | import javax.validation.ConstraintViolationException; 20 | 21 | /** 22 | * Exception thrown when conversion fails. Since this should generally be converted into a 422 HTTP 23 | * status, this class extends ConstraintViolationException so you get the expected behavior if you 24 | * don't catch the exception. 25 | */ 26 | public class ConversionException extends ConstraintViolationException { 27 | 28 | ConversionException(String msg) { 29 | super(msg, null); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.bouncycastle.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2000 - 2013 The Legion of the Bouncy Castle Inc. 4 | (http://www.bouncycastle.org) 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /bin/kafka-rest-stop-service: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | TARGET=`ps ax | grep -i "$1" | grep java | grep -v grep | awk '{print $1}'` 17 | if [ "x$TARGET" = "x" ]; then 18 | >&2 echo "No running instance found." 19 | exit 1 20 | fi 21 | 22 | kill "$TARGET" 23 | for i in `seq 20`; do 24 | sleep 0.25 25 | ps ax | grep -i "$1" | grep "$TARGET" > /dev/null 26 | if [ $? -eq 0 ]; then 27 | exit 0 28 | fi 29 | done 30 | 31 | >&2 echo "Tried to kill $TARGET but never saw it die" 32 | exit 1 -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.slf4j.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2004-2007 QOS.ch 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining 6 | * a copy of this software and associated documentation files (the 7 | * "Software"), to deal in the Software without restriction, including 8 | * without limitation the rights to use, copy, modify, merge, publish, 9 | * distribute, sublicense, and/or sell copies of the Software, and to 10 | * permit persons to whom the Software is furnished to do so, subject to 11 | * the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | */ 24 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/TopicProduceRequest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import org.hibernate.validator.constraints.NotEmpty; 21 | 22 | import java.util.List; 23 | 24 | public class TopicProduceRequest extends SchemaHolder { 25 | 26 | @NotEmpty 27 | private List records; 28 | 29 | @JsonProperty 30 | public List getRecords() { 31 | return records; 32 | } 33 | 34 | @JsonProperty 35 | public void setRecords(List records) { 36 | this.records = records; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/PartitionProduceRequest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import org.hibernate.validator.constraints.NotEmpty; 21 | 22 | import java.util.List; 23 | 24 | public class PartitionProduceRequest extends SchemaHolder { 25 | 26 | @NotEmpty 27 | private List records; 28 | 29 | @JsonProperty 30 | public List getRecords() { 31 | return records; 32 | } 33 | 34 | @JsonProperty 35 | public void setRecords(List records) { 36 | this.records = records; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/RecordMetadataOrException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | 20 | import org.apache.kafka.clients.producer.RecordMetadata; 21 | 22 | public class RecordMetadataOrException { 23 | 24 | private final RecordMetadata recordMetadata; 25 | private final Exception exception; 26 | 27 | public RecordMetadataOrException(RecordMetadata recordMetadata, Exception exception) { 28 | this.recordMetadata = recordMetadata; 29 | this.exception = exception; 30 | } 31 | 32 | public RecordMetadata getRecordMetadata() { 33 | return recordMetadata; 34 | } 35 | 36 | public Exception getException() { 37 | return exception; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/assembly/standalone.xml: -------------------------------------------------------------------------------- 1 | 5 | 7 | standalone 8 | 9 | jar 10 | 11 | false 12 | 13 | 14 | ${project.basedir} 15 | / 16 | 17 | README* 18 | LICENSE* 19 | NOTICE* 20 | 21 | 22 | 23 | 24 | 25 | / 26 | true 27 | true 28 | runtime 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/ConsumerRecordAndSize.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import io.confluent.kafkarest.entities.ConsumerRecord; 20 | 21 | /** 22 | * Simple wrapper for a ConsumerRecord and an approximate serialized size. 23 | */ 24 | public class ConsumerRecordAndSize { 25 | 26 | private final ConsumerRecord record; 27 | private final long size; 28 | 29 | public ConsumerRecordAndSize(ConsumerRecord record, long size) { 30 | this.record = record; 31 | this.size = size; 32 | } 33 | 34 | public ConsumerRecord getRecord() { 35 | return record; 36 | } 37 | 38 | public long getSize() { 39 | return size; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/mock/MockTime.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.mock; 17 | 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import io.confluent.kafkarest.Time; 21 | 22 | public class MockTime implements Time { 23 | 24 | volatile long currentMs = 0; 25 | 26 | @Override 27 | public long milliseconds() { 28 | return currentMs; 29 | } 30 | 31 | @Override 32 | public long nanoseconds() { 33 | return TimeUnit.NANOSECONDS.convert(currentMs, TimeUnit.MILLISECONDS); 34 | } 35 | 36 | @Override 37 | public void sleep(long ms) { 38 | currentMs += ms; 39 | } 40 | 41 | @Override 42 | public void waitOn(Object on, long ms) throws InterruptedException { 43 | currentMs += ms; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh $@ 5 | 6 | SLF4J_VERSION = 1.7.7 7 | SLF4J = slf4j-$(SLF4J_VERSION) 8 | 9 | override_dh_auto_build: 10 | mvn clean package 11 | 12 | # Do not install init script automatically 13 | override_dh_installinit: 14 | 15 | DESTDIR = debian/kafka-rest 16 | override_dh_auto_install: 17 | install -m 755 -d $(DESTDIR)/etc/kafka-rest 18 | install -m 644 config/*.properties $(DESTDIR)/etc/kafka-rest 19 | install -m 755 -d $(DESTDIR)/usr/lib/kafka-rest 20 | for i in `ls | grep -vE config\|debian`; do \ 21 | cp -r $$i $(DESTDIR)/usr/lib/kafka-rest || exit $$?; \ 22 | done 23 | find $(DESTDIR)/usr/lib/kafka-rest -type f -a \ 24 | \( -name \*.java -o -name \*.class -o \ 25 | -name \*.MF -o -name \*.html \) \ 26 | -print -delete 27 | for i in `seq 10`; do \ 28 | find $(DESTDIR) -type d -empty -print -exec rmdir '{}' ';' || :; \ 29 | done 30 | find $(DESTDIR)/usr/lib/kafka-rest -type f -a \ 31 | \( -name README\* -o -name LICENSE -o -name NOTICE -o -name HEADER \) \ 32 | -print -delete || : 33 | find $(DESTDIR)/usr/lib/kafka-rest -type d -a \ 34 | \( -name test -o -name src -o -name tmp \) \ 35 | -print -exec rm -rf '{}' ';' || : 36 | ln -s /etc/kafka-rest $(DESTDIR)/usr/lib/kafka-rest/config 37 | ln -s /var/log/kafka-rest $(DESTDIR)/usr/lib/kafka-rest/logs 38 | install -m 755 -d $(DESTDIR)/var/log/kafka-rest 39 | -------------------------------------------------------------------------------- /licenses/NOTICE.lz4-java.txt: -------------------------------------------------------------------------------- 1 | LZ4 Library 2 | Copyright (c) 2011-2014, Yann Collet 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, this 12 | list of conditions and the following disclaimer in the documentation and/or 13 | other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 19 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 22 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/SpoolShard.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonCreator; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | public class SpoolShard { 22 | private String path; 23 | private long timestamp; 24 | 25 | @JsonCreator 26 | public SpoolShard(@JsonProperty("spool_path") String path, 27 | @JsonProperty("resume_timestamp") long timestamp) { 28 | this.path = path; 29 | this.timestamp = timestamp; 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return "SpoolShard{" + 35 | "spool_path=" + path + 36 | ", resume_timestamp=" + timestamp + '\'' + 37 | '}'; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/EntityUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import javax.xml.bind.DatatypeConverter; 19 | 20 | public class EntityUtils { 21 | 22 | public static byte[] parseBase64Binary(String data) throws IllegalArgumentException { 23 | try { 24 | return DatatypeConverter.parseBase64Binary(data); 25 | } catch (ArrayIndexOutOfBoundsException e) { 26 | // Implementation can throw index error on invalid inputs, make sure all known parsing issues 27 | // get converted to illegal argument error 28 | throw new IllegalArgumentException(e); 29 | } 30 | } 31 | 32 | public static String encodeBase64Binary(byte[] data) { 33 | return DatatypeConverter.printBase64Binary(data); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/RestProducer.java: -------------------------------------------------------------------------------- 1 | package io.confluent.kafkarest; /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | import java.util.Collection; 18 | 19 | import io.confluent.kafkarest.entities.ProduceRecord; 20 | import io.confluent.kafkarest.entities.SpoolMode; 21 | 22 | /** 23 | * Wrapper for KafkaProducer that handles schemas. 24 | */ 25 | public interface RestProducer { 26 | 27 | /** 28 | * Produces messages to the topic, handling any conversion, schema lookups or other operations 29 | * that need to be performed before sending the messages. If schemas are looked up or registered, 30 | * the SchemaHolder is updated with the resulting IDs. 31 | */ 32 | public void produce(ProduceTask task, String topic, Integer partition, SpoolMode spoolMode, 33 | Collection> records); 34 | 35 | public void close(); 36 | } 37 | -------------------------------------------------------------------------------- /licenses/LICENSE.bsd.txt: -------------------------------------------------------------------------------- 1 | Redistribution and use in source and binary forms, with or without 2 | modification, are permitted provided that the following conditions 3 | are met: 4 | 1. Redistributions of source code must retain the above copyright 5 | notice, this list of conditions and the following disclaimer. 6 | 2. Redistributions in binary form must reproduce the above copyright 7 | notice, this list of conditions and the following disclaimer in the 8 | documentation and/or other materials provided with the distribution. 9 | 3. Neither the name of the copyright holders nor the names of its 10 | contributors may be used to endorse or promote products derived from 11 | this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 14 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 17 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 19 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 20 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 21 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 22 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 23 | THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.jzlib.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in 11 | the documentation and/or other materials provided with the distribution. 12 | 13 | 3. The names of the authors may not be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, 17 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 18 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, 19 | INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, 20 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, 22 | OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 24 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 25 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.snappy.txt: -------------------------------------------------------------------------------- 1 | Copyright 2011, Google Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/EmbeddedFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonFormat; 20 | 21 | /** 22 | * Permitted formats for ProduceRecords embedded in produce requests/consume responses, e.g. 23 | * base64-encoded binary, JSON-encoded Avro, etc. Each of these correspond to a content type, a 24 | * ProduceRecord implementation, a Producer in the ProducerPool (with corresponding Kafka 25 | * serializer), ConsumerRecord implementation, and a serializer for any instantiated consumers. 26 | * 27 | * Note that for each type, it's assumed that the key and value can be handled by the same 28 | * serializer. This means each serializer should handle both it's complex type (e.g. 29 | * Indexed/Generic/SpecificRecord for Avro) and boxed primitive types (Integer, Boolean, etc.). 30 | */ 31 | @JsonFormat(shape = JsonFormat.Shape.STRING) 32 | public enum EmbeddedFormat { 33 | BINARY, 34 | AVRO 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/UriUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import java.net.URI; 20 | 21 | import javax.ws.rs.core.UriBuilder; 22 | import javax.ws.rs.core.UriInfo; 23 | 24 | public class UriUtils { 25 | 26 | public static UriBuilder absoluteUriBuilder(KafkaRestConfig config, UriInfo uriInfo) { 27 | String hostname = config.getString(KafkaRestConfig.HOST_NAME_CONFIG); 28 | UriBuilder builder = uriInfo.getAbsolutePathBuilder(); 29 | if (hostname.length() > 0) { 30 | builder.host(hostname); 31 | // Resetting the hostname removes the scheme and port for some reason, so they may need to 32 | // be reset. 33 | URI origAbsoluteUri = uriInfo.getAbsolutePath(); 34 | builder.scheme(origAbsoluteUri.getScheme()); 35 | // Only reset the port if it was set in the original URI 36 | if (origAbsoluteUri.getPort() != -1) { 37 | builder.port(config.getInt(KafkaRestConfig.PORT_CONFIG)); 38 | } 39 | } 40 | return builder; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/exceptions/ZkExceptionMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.exceptions; 18 | 19 | import org.I0Itec.zkclient.exception.ZkException; 20 | 21 | import javax.ws.rs.core.Response; 22 | 23 | import io.confluent.kafkarest.Errors; 24 | import io.confluent.kafkarest.KafkaRestConfig; 25 | import io.confluent.rest.exceptions.DebuggableExceptionMapper; 26 | 27 | /** 28 | * Handles uncaught ZkExceptions and converts them into RestExceptions. Currently all ZkExceptions 29 | * use a single error code. 30 | */ 31 | public class ZkExceptionMapper extends DebuggableExceptionMapper { 32 | 33 | public ZkExceptionMapper(KafkaRestConfig restConfig) { 34 | super(restConfig); 35 | } 36 | 37 | @Override 38 | public Response toResponse(ZkException e) { 39 | return createResponse(e, Errors.ZOOKEEPER_ERROR_ERROR_CODE, 40 | Response.Status.INTERNAL_SERVER_ERROR, 41 | Errors.ZOOKEEPER_ERROR_MESSAGE + e.getMessage()).build(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.base64.txt: -------------------------------------------------------------------------------- 1 | The person or persons who have associated work with this document (the 2 | "Dedicator" or "Certifier") hereby either (a) certifies that, to the best of 3 | his knowledge, the work of authorship identified is in the public domain of 4 | the country from which the work is published, or (b) hereby dedicates whatever 5 | copyright the dedicators holds in the work of authorship identified below (the 6 | "Work") to the public domain. A certifier, moreover, dedicates any copyright 7 | interest he may have in the associated work, and for these purposes, is 8 | described as a "dedicator" below. 9 | 10 | A certifier has taken reasonable steps to verify the copyright status of this 11 | work. Certifier recognizes that his good faith efforts may not shield him from 12 | liability if in fact the work certified is not in the public domain. 13 | 14 | Dedicator makes this dedication for the benefit of the public at large and to 15 | the detriment of the Dedicator's heirs and successors. Dedicator intends this 16 | dedication to be an overt act of relinquishment in perpetuate of all present 17 | and future rights under copyright law, whether vested or contingent, in the 18 | Work. Dedicator understands that such relinquishment of all rights includes 19 | the relinquishment of all rights to enforce (by lawsuit or otherwise) those 20 | copyrights in the Work. 21 | 22 | Dedicator recognizes that, once placed in the public domain, the Work may be 23 | freely reproduced, distributed, transmitted, used, modified, built upon, or 24 | otherwise exploited by anyone for any purpose, commercial or non-commercial, 25 | and in any way, including by methods that have not yet been invented or 26 | conceived. 27 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.deque.txt: -------------------------------------------------------------------------------- 1 | The person or persons who have associated work with this document (the 2 | "Dedicator" or "Certifier") hereby either (a) certifies that, to the best of 3 | his knowledge, the work of authorship identified is in the public domain of 4 | the country from which the work is published, or (b) hereby dedicates whatever 5 | copyright the dedicators holds in the work of authorship identified below (the 6 | "Work") to the public domain. A certifier, moreover, dedicates any copyright 7 | interest he may have in the associated work, and for these purposes, is 8 | described as a "dedicator" below. 9 | 10 | A certifier has taken reasonable steps to verify the copyright status of this 11 | work. Certifier recognizes that his good faith efforts may not shield him from 12 | liability if in fact the work certified is not in the public domain. 13 | 14 | Dedicator makes this dedication for the benefit of the public at large and to 15 | the detriment of the Dedicator's heirs and successors. Dedicator intends this 16 | dedication to be an overt act of relinquishment in perpetuate of all present 17 | and future rights under copyright law, whether vested or contingent, in the 18 | Work. Dedicator understands that such relinquishment of all rights includes 19 | the relinquishment of all rights to enforce (by lawsuit or otherwise) those 20 | copyrights in the Work. 21 | 22 | Dedicator recognizes that, once placed in the public domain, the Work may be 23 | freely reproduced, distributed, transmitted, used, modified, built upon, or 24 | otherwise exploited by anyone for any purpose, commercial or non-commercial, 25 | and in any way, including by methods that have not yet been invented or 26 | conceived. 27 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.jsr166y.txt: -------------------------------------------------------------------------------- 1 | The person or persons who have associated work with this document (the 2 | "Dedicator" or "Certifier") hereby either (a) certifies that, to the best of 3 | his knowledge, the work of authorship identified is in the public domain of 4 | the country from which the work is published, or (b) hereby dedicates whatever 5 | copyright the dedicators holds in the work of authorship identified below (the 6 | "Work") to the public domain. A certifier, moreover, dedicates any copyright 7 | interest he may have in the associated work, and for these purposes, is 8 | described as a "dedicator" below. 9 | 10 | A certifier has taken reasonable steps to verify the copyright status of this 11 | work. Certifier recognizes that his good faith efforts may not shield him from 12 | liability if in fact the work certified is not in the public domain. 13 | 14 | Dedicator makes this dedication for the benefit of the public at large and to 15 | the detriment of the Dedicator's heirs and successors. Dedicator intends this 16 | dedication to be an overt act of relinquishment in perpetuity of all present 17 | and future rights under copyright law, whether vested or contingent, in the 18 | Work. Dedicator understands that such relinquishment of all rights includes 19 | the relinquishment of all rights to enforce (by lawsuit or otherwise) those 20 | copyrights in the Work. 21 | 22 | Dedicator recognizes that, once placed in the public domain, the Work may be 23 | freely reproduced, distributed, transmitted, used, modified, built upon, or 24 | otherwise exploited by anyone for any purpose, commercial or non-commercial, 25 | and in any way, including by methods that have not yet been invented or 26 | conceived. 27 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/Main.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.io.IOException; 22 | 23 | import io.confluent.rest.RestConfigException; 24 | 25 | public class Main { 26 | 27 | private static final Logger log = LoggerFactory.getLogger(Main.class); 28 | 29 | /** 30 | * Starts an embedded Jetty server running the REST server. 31 | */ 32 | public static void main(String[] args) throws IOException { 33 | try { 34 | KafkaRestConfig config = new KafkaRestConfig((args.length > 0 ? args[0] : null)); 35 | KafkaRestApplication app = new KafkaRestApplication(config); 36 | app.start(); 37 | log.info("Server started, listening for requests..."); 38 | app.join(); 39 | } catch (RestConfigException e) { 40 | log.error("Server configuration failed: ", e); 41 | System.exit(1); 42 | } catch (Exception e) { 43 | log.error("Server died unexpectedly: " + e); 44 | System.exit(1); 45 | } 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/TopicProduceRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | public interface TopicProduceRecord extends ProduceRecord { 20 | // It may seem odd that this is an interface when ProduceRecord is an abstract class. If 21 | // we used an abstract class here and included the (Integer partition) field and 22 | // getters/setters, then subclasses would have to inherit from this class, and reuse of the 23 | // implementations of ProduceRecord would have to be via composition. This should be fine, but 24 | // currently it seems to be impossible to get Jackson to behave properly during 25 | // deserialization when combining that complex type hierarchy with it's unwrapping feature, 26 | // which would be required to get the serialize (key,value,partition) values at the same level. 27 | // This means implementations have a bit of duplication to provide the partition part of the 28 | // interface. 29 | 30 | public K getKey(); 31 | 32 | public V getValue(); 33 | 34 | public Integer getPartition(); 35 | } 36 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.webbit.txt: -------------------------------------------------------------------------------- 1 | (BSD License: http://www.opensource.org/licenses/bsd-license) 2 | 3 | Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or 7 | without modification, are permitted provided that the 8 | following conditions are met: 9 | 10 | * Redistributions of source code must retain the above 11 | copyright notice, this list of conditions and the 12 | following disclaimer. 13 | 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the 16 | following disclaimer in the documentation and/or other 17 | materials provided with the distribution. 18 | 19 | * Neither the name of the Webbit nor the names of 20 | its contributors may be used to endorse or promote products 21 | derived from this software without specific prior written 22 | permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 25 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 26 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 27 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 29 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 30 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 31 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 32 | GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 33 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 34 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 36 | OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 | POSSIBILITY OF SUCH DAMAGE. 38 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/Context.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | /** 19 | * Shared, global state for the REST proxy server, including configuration and connection pools. 20 | */ 21 | public class Context { 22 | 23 | private final KafkaRestConfig config; 24 | private final MetadataObserver metadataObserver; 25 | private final ProducerPool producerPool; 26 | private final ConsumerManager consumerManager; 27 | 28 | public Context(KafkaRestConfig config, MetadataObserver metadataObserver, 29 | ProducerPool producerPool, ConsumerManager consumerManager) { 30 | this.config = config; 31 | this.metadataObserver = metadataObserver; 32 | this.producerPool = producerPool; 33 | this.consumerManager = consumerManager; 34 | } 35 | 36 | public KafkaRestConfig getConfig() { 37 | return config; 38 | } 39 | 40 | public MetadataObserver getMetadataObserver() { 41 | return metadataObserver; 42 | } 43 | 44 | public ProducerPool getProducerPool() { 45 | return producerPool; 46 | } 47 | 48 | public ConsumerManager getConsumerManager() { 49 | return consumerManager; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/ProduceResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import org.hibernate.validator.constraints.NotEmpty; 21 | 22 | import java.util.List; 23 | 24 | public class ProduceResponse { 25 | 26 | @NotEmpty 27 | private List offsets; 28 | 29 | private Integer keySchemaId; 30 | 31 | private Integer valueSchemaId; 32 | 33 | @JsonProperty 34 | public List getOffsets() { 35 | return offsets; 36 | } 37 | 38 | @JsonProperty 39 | public void setOffsets(List offsets) { 40 | this.offsets = offsets; 41 | } 42 | 43 | @JsonProperty("key_schema_id") 44 | public Integer getKeySchemaId() { 45 | return keySchemaId; 46 | } 47 | 48 | public void setKeySchemaId(Integer keySchemaId) { 49 | this.keySchemaId = keySchemaId; 50 | } 51 | 52 | @JsonProperty("value_schema_id") 53 | public Integer getValueSchemaId() { 54 | return valueSchemaId; 55 | } 56 | 57 | public void setValueSchemaId(Integer valueSchemaId) { 58 | this.valueSchemaId = valueSchemaId; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/CreateConsumerInstanceResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import org.hibernate.validator.constraints.NotBlank; 21 | import org.hibernate.validator.constraints.URL; 22 | 23 | public class CreateConsumerInstanceResponse { 24 | 25 | @NotBlank 26 | String instanceId; 27 | 28 | @NotBlank 29 | @URL 30 | String baseUri; 31 | 32 | public CreateConsumerInstanceResponse(@JsonProperty("instance_id") String instanceId, 33 | @JsonProperty("base_uri") String baseUri) { 34 | this.instanceId = instanceId; 35 | this.baseUri = baseUri; 36 | } 37 | 38 | @JsonProperty("instance_id") 39 | public String getInstanceId() { 40 | return instanceId; 41 | } 42 | 43 | @JsonProperty("instance_id") 44 | public void setInstanceId(String instanceId) { 45 | this.instanceId = instanceId; 46 | } 47 | 48 | @JsonProperty("base_uri") 49 | public String getBaseUri() { 50 | return baseUri; 51 | } 52 | 53 | @JsonProperty("base_uri") 54 | public void setBaseUri(String baseUri) { 55 | this.baseUri = baseUri; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /licenses/netty/licenses/LICENSE.protobuf.txt: -------------------------------------------------------------------------------- 1 | Copyright 2008, Google Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | Code generated by the Protocol Buffer compiler is owned by the owner 31 | of the input file used when generating it. This code is not 32 | standalone and requires a support library to be linked with it. This 33 | support library is itself covered by the above license. 34 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/ProduceRecordBase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnore; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | public abstract class ProduceRecordBase implements ProduceRecord { 23 | 24 | protected K key; 25 | protected V value; 26 | 27 | public ProduceRecordBase(@JsonProperty K key, @JsonProperty V value) { 28 | this.key = key; 29 | this.value = value; 30 | } 31 | 32 | @JsonIgnore 33 | public K getKey() { 34 | return key; 35 | } 36 | 37 | public void setKey(K key) { 38 | this.key = key; 39 | } 40 | 41 | @JsonIgnore 42 | public V getValue() { 43 | return value; 44 | } 45 | 46 | public void setValue(V value) { 47 | this.value = value; 48 | } 49 | 50 | /** 51 | * Return a JSON-serializable version of the key. This does not need to handle schemas. 52 | */ 53 | @JsonProperty("key") 54 | public Object getJsonKey() { 55 | return key; 56 | } 57 | 58 | /** 59 | * Return a JSON-serializable version of the value. This does not need to handle schemas. 60 | */ 61 | @JsonProperty("value") 62 | public Object getJsonValue() { 63 | return value; 64 | } 65 | } -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/SpoolMessage.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonCreator; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | public class SpoolMessage { 22 | private int attempt; 23 | private long timestamp; 24 | private String topic; 25 | private byte[] key; 26 | private byte[] value; 27 | 28 | @JsonCreator 29 | public SpoolMessage(@JsonProperty("attempt") int attempt, 30 | @JsonProperty("timestamp") long timestamp, 31 | @JsonProperty("topic") String topic, 32 | @JsonProperty("key") byte[] key, 33 | @JsonProperty("value") byte[] value) { 34 | this.attempt = attempt; 35 | this.timestamp = timestamp; 36 | this.topic = topic; 37 | this.key = key; 38 | this.value = value; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "SpoolMessage{" + 44 | "attempt=" + attempt + 45 | ", timestamp=" + timestamp + 46 | ", topic=" + topic + 47 | ", key=" + EntityUtils.encodeBase64Binary(key) + 48 | ", value=" + EntityUtils.encodeBase64Binary(value) + '\'' + 49 | '}'; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/BrokerList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonCreator; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import java.util.List; 22 | 23 | import javax.validation.constraints.NotNull; 24 | 25 | public class BrokerList { 26 | 27 | @NotNull 28 | private List brokers; 29 | 30 | @JsonCreator 31 | public BrokerList(@JsonProperty("brokers") List brokers) { 32 | this.brokers = brokers; 33 | } 34 | 35 | @JsonProperty 36 | public List getBrokers() { 37 | return brokers; 38 | } 39 | 40 | @JsonProperty 41 | public void setBrokers(List brokers) { 42 | this.brokers = brokers; 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return "BrokerList{" + 48 | "brokers=" + brokers + 49 | '}'; 50 | } 51 | 52 | @Override 53 | public boolean equals(Object o) { 54 | if (this == o) { 55 | return true; 56 | } 57 | if (!(o instanceof BrokerList)) { 58 | return false; 59 | } 60 | 61 | BrokerList that = (BrokerList) o; 62 | 63 | if (!brokers.equals(that.brokers)) { 64 | return false; 65 | } 66 | 67 | return true; 68 | } 69 | 70 | @Override 71 | public int hashCode() { 72 | return brokers.hashCode(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/ConsumerInstanceId.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | public class ConsumerInstanceId { 19 | 20 | private final String group; 21 | private final String instance; 22 | 23 | public ConsumerInstanceId(String group, String instance) { 24 | this.group = group; 25 | this.instance = instance; 26 | } 27 | 28 | public String getGroup() { 29 | return group; 30 | } 31 | 32 | public String getInstance() { 33 | return instance; 34 | } 35 | 36 | @Override 37 | public boolean equals(Object o) { 38 | if (this == o) { 39 | return true; 40 | } 41 | if (o == null || getClass() != o.getClass()) { 42 | return false; 43 | } 44 | 45 | ConsumerInstanceId that = (ConsumerInstanceId) o; 46 | 47 | if (group != null ? !group.equals(that.group) : that.group != null) { 48 | return false; 49 | } 50 | if (instance != null ? !instance.equals(that.instance) : that.instance != null) { 51 | return false; 52 | } 53 | 54 | return true; 55 | } 56 | 57 | @Override 58 | public int hashCode() { 59 | int result = group != null ? group.hashCode() : 0; 60 | result = 31 * result + (instance != null ? instance.hashCode() : 0); 61 | return result; 62 | } 63 | 64 | @Override 65 | public String toString() { 66 | return "ConsumerInstanceId{" + 67 | "group='" + group + '\'' + 68 | ", instance='" + instance + '\'' + 69 | '}'; 70 | } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/resources/RootResource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.resources; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | import javax.validation.Valid; 22 | import javax.ws.rs.Consumes; 23 | import javax.ws.rs.GET; 24 | import javax.ws.rs.POST; 25 | import javax.ws.rs.Path; 26 | import javax.ws.rs.Produces; 27 | 28 | import io.confluent.kafkarest.Versions; 29 | import io.confluent.rest.annotations.PerformanceMetric; 30 | 31 | @Path("/") 32 | @Produces({Versions.KAFKA_V1_JSON_WEIGHTED, Versions.KAFKA_DEFAULT_JSON_WEIGHTED, 33 | Versions.JSON_WEIGHTED}) 34 | @Consumes({Versions.KAFKA_V1_JSON, Versions.KAFKA_DEFAULT_JSON, Versions.JSON, 35 | Versions.GENERIC_REQUEST}) 36 | public class RootResource { 37 | 38 | @GET 39 | @PerformanceMetric("root.get") 40 | public Map get() { 41 | // Currently this just provides an endpoint that's a nop and can be used to check for liveness 42 | // and can be used for tests that need to test the server setup rather than the functionality 43 | // of a specific resource. Some APIs provide a listing of endpoints as their root resource; it 44 | // might be nice to provide that. 45 | return new HashMap(); 46 | } 47 | 48 | @POST 49 | @PerformanceMetric("root.post") 50 | public Map post(@Valid Map request) { 51 | // This version allows testing with posted entities 52 | return new HashMap(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/resources/BrokersResource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.resources; 17 | 18 | import javax.validation.Valid; 19 | import javax.ws.rs.Consumes; 20 | import javax.ws.rs.GET; 21 | import javax.ws.rs.Path; 22 | import javax.ws.rs.PathParam; 23 | import javax.ws.rs.Produces; 24 | 25 | import io.confluent.kafkarest.Context; 26 | import io.confluent.kafkarest.Errors; 27 | import io.confluent.kafkarest.Versions; 28 | import io.confluent.kafkarest.entities.Broker; 29 | import io.confluent.kafkarest.entities.BrokerList; 30 | import io.confluent.rest.annotations.PerformanceMetric; 31 | 32 | /** 33 | * Resource representing the collection of all available brokers. 34 | */ 35 | @Path("/brokers") 36 | @Produces({Versions.KAFKA_V1_JSON_WEIGHTED, Versions.KAFKA_DEFAULT_JSON_WEIGHTED, 37 | Versions.JSON_WEIGHTED}) 38 | @Consumes() 39 | public class BrokersResource { 40 | 41 | private final Context ctx; 42 | 43 | public BrokersResource(Context ctx) { 44 | this.ctx = ctx; 45 | } 46 | 47 | @GET 48 | @Valid 49 | @PerformanceMetric("brokers.list") 50 | public BrokerList list() { 51 | return new BrokerList(ctx.getMetadataObserver().getBrokerIds()); 52 | } 53 | 54 | @GET 55 | @Path("/{broker}") 56 | @PerformanceMetric("broker.get") 57 | public Broker getBroker(@PathParam("broker") int brokerId) { 58 | Broker broker = ctx.getMetadataObserver().getBroker(brokerId); 59 | if (broker == null) { 60 | throw Errors.brokerNotFoundException(); 61 | } 62 | return broker; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/assembly/package.xml: -------------------------------------------------------------------------------- 1 | 5 | 8 | package 9 | 10 | dir 11 | 12 | false 13 | 14 | 15 | ${project.basedir} 16 | share/doc/kafka-rest/ 17 | 18 | README* 19 | LICENSE* 20 | NOTICE* 21 | licenses/ 22 | 23 | 24 | 25 | ${project.basedir} 26 | / 27 | 28 | bin/* 29 | 30 | 31 | 32 | ${project.basedir}/config 33 | etc/kafka-rest 34 | 35 | * 36 | 37 | 38 | 39 | 40 | 41 | share/java/kafka-rest 42 | true 43 | true 44 | 45 | io.confluent:rest-utils 46 | io.confluent:rest-utils-test 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/AvroProduceRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonCreator; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import com.fasterxml.jackson.databind.JsonNode; 22 | 23 | public class AvroProduceRecord extends ProduceRecordBase { 24 | 25 | @JsonCreator 26 | public AvroProduceRecord(@JsonProperty("key") JsonNode key, 27 | @JsonProperty("value") JsonNode value) { 28 | super(key, value); 29 | } 30 | 31 | public AvroProduceRecord(JsonNode value) { 32 | this(null, value); 33 | } 34 | 35 | @Override 36 | public JsonNode getJsonKey() { 37 | return key; 38 | } 39 | 40 | @Override 41 | public JsonNode getJsonValue() { 42 | return value; 43 | } 44 | 45 | @Override 46 | public boolean equals(Object o) { 47 | if (this == o) { 48 | return true; 49 | } 50 | if (o == null || getClass() != o.getClass()) { 51 | return false; 52 | } 53 | 54 | AvroProduceRecord that = (AvroProduceRecord) o; 55 | 56 | if (key != null ? !key.equals(that.key) : that.key != null) { 57 | return false; 58 | } 59 | if (value != null ? !value.equals(that.value) : that.value != null) { 60 | return false; 61 | } 62 | 63 | return true; 64 | } 65 | 66 | @Override 67 | public int hashCode() { 68 | int result = key != null ? key.hashCode() : 0; 69 | result = 31 * result + (value != null ? value.hashCode() : 0); 70 | return result; 71 | } 72 | } -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/integration/TestKafkaRestApplication.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.integration; 18 | 19 | import org.I0Itec.zkclient.ZkClient; 20 | 21 | import javax.ws.rs.core.Configurable; 22 | 23 | import io.confluent.kafkarest.ConsumerManager; 24 | import io.confluent.kafkarest.KafkaRestApplication; 25 | import io.confluent.kafkarest.KafkaRestConfig; 26 | import io.confluent.kafkarest.MetadataObserver; 27 | import io.confluent.kafkarest.ProducerPool; 28 | 29 | /** 30 | * Test version of KakfaRestApplication that allows for dependency injection so components and be 31 | * tweaked for tests. 32 | */ 33 | public class TestKafkaRestApplication extends KafkaRestApplication { 34 | 35 | ZkClient zkClientInjected; 36 | MetadataObserver mdObserverInjected; 37 | ProducerPool producerPoolInjected; 38 | ConsumerManager consumerManagerInjected; 39 | 40 | public TestKafkaRestApplication(KafkaRestConfig config, ZkClient zkClient, 41 | MetadataObserver mdObserver, ProducerPool producerPool, 42 | ConsumerManager consumerManager) { 43 | super(config); 44 | zkClientInjected = zkClient; 45 | mdObserverInjected = mdObserver; 46 | producerPoolInjected = producerPool; 47 | consumerManagerInjected = consumerManager; 48 | } 49 | 50 | @Override 51 | public void setupResources(Configurable config, KafkaRestConfig appConfig) { 52 | setupInjectedResources(config, appConfig, zkClientInjected, mdObserverInjected, 53 | producerPoolInjected, consumerManagerInjected); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/BinaryRestProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import org.apache.kafka.clients.producer.KafkaProducer; 20 | import org.apache.kafka.clients.producer.ProducerRecord; 21 | import org.apache.kafka.common.serialization.Serializer; 22 | 23 | import java.util.Collection; 24 | 25 | import io.confluent.kafkarest.entities.ProduceRecord; 26 | import io.confluent.kafkarest.entities.SpoolMode; 27 | 28 | /** 29 | * Wrapper for KafkaProducers that handles schemas. 30 | */ 31 | public class BinaryRestProducer implements RestProducer { 32 | 33 | protected final SpoolProducer producer; 34 | protected final Serializer keySerializer; 35 | protected final Serializer valueSerializer; 36 | 37 | public BinaryRestProducer(SpoolProducer producer, 38 | Serializer keySerializer, 39 | Serializer valueSerializer) { 40 | this.producer = producer; 41 | this.keySerializer = keySerializer; 42 | this.valueSerializer = valueSerializer; 43 | } 44 | 45 | public void produce(ProduceTask task, String topic, Integer partition, SpoolMode spoolMode, 46 | Collection> records) { 47 | for (ProduceRecord record : records) { 48 | producer.send(spoolMode, 49 | new ProducerRecord(topic, partition, record.getKey(), record.getValue()), 50 | task.createCallback()); 51 | } 52 | } 53 | 54 | public void close() { 55 | producer.close(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/BinaryConsumerState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import io.confluent.kafkarest.entities.BinaryConsumerRecord; 20 | import kafka.javaapi.consumer.ConsumerConnector; 21 | import kafka.message.MessageAndMetadata; 22 | import kafka.serializer.Decoder; 23 | import kafka.serializer.DefaultDecoder; 24 | import kafka.utils.VerifiableProperties; 25 | 26 | /** 27 | * Binary implementation of ConsumerState that does no decoding, returning the raw bytes directly. 28 | */ 29 | public class BinaryConsumerState extends ConsumerState { 30 | 31 | private static final Decoder decoder = new DefaultDecoder(new VerifiableProperties()); 32 | 33 | public BinaryConsumerState(KafkaRestConfig config, 34 | ConsumerInstanceId instanceId, 35 | ConsumerConnector consumer) { 36 | super(config, instanceId, consumer); 37 | } 38 | 39 | @Override 40 | protected Decoder getKeyDecoder() { 41 | return decoder; 42 | } 43 | 44 | @Override 45 | protected Decoder getValueDecoder() { 46 | return decoder; 47 | } 48 | 49 | @Override 50 | public ConsumerRecordAndSize createConsumerRecord( 51 | MessageAndMetadata msg) { 52 | long approxSize = (msg.key() != null ? msg.key().length : 0) 53 | + (msg.message() != null ? msg.message().length : 0); 54 | return new ConsumerRecordAndSize( 55 | new BinaryConsumerRecord(msg.key(), msg.message(), msg.partition(), msg.offset()), 56 | approxSize); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /licenses/NOTICE.jetty.txt: -------------------------------------------------------------------------------- 1 | ============================================================== 2 | Jetty Web Container 3 | Copyright 1995-2015 Mort Bay Consulting Pty Ltd. 4 | ============================================================== 5 | 6 | The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd 7 | unless otherwise noted. 8 | 9 | Jetty is dual licensed under both 10 | 11 | * The Apache 2.0 License 12 | http://www.apache.org/licenses/LICENSE-2.0.html 13 | 14 | and 15 | 16 | * The Eclipse Public 1.0 License 17 | http://www.eclipse.org/legal/epl-v10.html 18 | 19 | Jetty may be distributed under either license. 20 | 21 | 22 | ------ 23 | Oracle 24 | 25 | The following artifacts are CDDL + GPLv2 with classpath exception. 26 | 27 | https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html 28 | 29 | javax.servlet:javax.servlet-api 30 | javax.servlet.jsp:javax.servlet.jsp-api 31 | 32 | 33 | ------ 34 | Apache 35 | 36 | The following artifacts are ASL2 licensed. 37 | 38 | org.apache.taglibs:taglibs-standard-spec 39 | org.apache.taglibs:taglibs-standard-impl 40 | 41 | 42 | ------ 43 | MortBay 44 | 45 | The following artifacts are ASL2 licensed. Based on selected classes from 46 | following Apache Tomcat jars, all ASL2 licensed. 47 | 48 | org.mortbay.jasper:apache-jsp 49 | org.apache.tomcat:tomcat-jasper 50 | org.apache.tomcat:tomcat-juli 51 | org.apache.tomcat:tomcat-jsp-api 52 | org.apache.tomcat:tomcat-el-api 53 | org.apache.tomcat:tomcat-jasper-el 54 | org.apache.tomcat:tomcat-api 55 | org.apache.tomcat:tomcat-util-scan 56 | org.apache.tomcat:tomcat-util 57 | 58 | org.mortbay.jasper:apache-el 59 | org.apache.tomcat:tomcat-jasper-el 60 | org.apache.tomcat:tomcat-el-api 61 | 62 | 63 | ------ 64 | Mortbay 65 | 66 | The following artifacts are CDDL + GPLv2 with classpath exception. 67 | 68 | https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html 69 | 70 | org.eclipse.jetty.toolchain:jetty-schemas 71 | 72 | ------ 73 | Assorted 74 | 75 | The UnixCrypt.java code implements the one way cryptography used by 76 | Unix systems for simple password protection. Copyright 1996 Aki Yoshida, 77 | modified April 2001 by Iris Van den Broeke, Daniel Deville. 78 | Permission to use, copy, modify and distribute UnixCrypt 79 | for non-commercial or commercial purposes and without fee is 80 | granted provided that the copyright notice appears in all copies. 81 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/SchemaHolder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | public class SchemaHolder { 22 | 23 | protected String keySchema; 24 | protected Integer keySchemaId; 25 | 26 | protected String valueSchema; 27 | protected Integer valueSchemaId; 28 | 29 | public SchemaHolder() { 30 | } 31 | 32 | public SchemaHolder(String keySchema, String valueSchema) { 33 | this(keySchema, null, valueSchema, null); 34 | } 35 | 36 | public SchemaHolder(String keySchema, Integer keySchemaId, 37 | String valueSchema, Integer valueSchemaId) { 38 | this.keySchema = keySchema; 39 | this.keySchemaId = keySchemaId; 40 | this.valueSchema = valueSchema; 41 | this.valueSchemaId = valueSchemaId; 42 | } 43 | 44 | @JsonProperty("key_schema") 45 | public String getKeySchema() { 46 | return keySchema; 47 | } 48 | 49 | public void setKeySchema(String keySchema) { 50 | this.keySchema = keySchema; 51 | } 52 | 53 | @JsonProperty("key_schema_id") 54 | public Integer getKeySchemaId() { 55 | return keySchemaId; 56 | } 57 | 58 | public void setKeySchemaId(Integer keySchemaId) { 59 | this.keySchemaId = keySchemaId; 60 | } 61 | 62 | @JsonProperty("value_schema") 63 | public String getValueSchema() { 64 | return valueSchema; 65 | } 66 | 67 | public void setValueSchema(String valueSchema) { 68 | this.valueSchema = valueSchema; 69 | } 70 | 71 | @JsonProperty("value_schema_id") 72 | public Integer getValueSchemaId() { 73 | return valueSchemaId; 74 | } 75 | 76 | public void setValueSchemaId(Integer valueSchemaId) { 77 | this.valueSchemaId = valueSchemaId; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/AvroTopicProduceRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import com.fasterxml.jackson.databind.JsonNode; 21 | 22 | import javax.validation.constraints.Min; 23 | 24 | public class AvroTopicProduceRecord extends AvroProduceRecord 25 | implements TopicProduceRecord { 26 | 27 | // When producing to a topic, a partition may be explicitly requested. 28 | @Min(0) 29 | protected Integer partition; 30 | 31 | public AvroTopicProduceRecord(@JsonProperty("key") JsonNode key, 32 | @JsonProperty("value") JsonNode value, 33 | @JsonProperty("partition") Integer partition) { 34 | super(key, value); 35 | this.partition = partition; 36 | } 37 | 38 | @JsonProperty 39 | public Integer getPartition() { 40 | return partition; 41 | } 42 | 43 | @JsonProperty 44 | public void setPartition(Integer partition) { 45 | this.partition = partition; 46 | } 47 | 48 | @Override 49 | public boolean equals(Object o) { 50 | if (this == o) { 51 | return true; 52 | } 53 | if (o == null || getClass() != o.getClass()) { 54 | return false; 55 | } 56 | if (!super.equals(o)) { 57 | return false; 58 | } 59 | 60 | AvroTopicProduceRecord that = (AvroTopicProduceRecord) o; 61 | 62 | if (partition != null ? !partition.equals(that.partition) : that.partition != null) { 63 | return false; 64 | } 65 | 66 | return true; 67 | } 68 | 69 | @Override 70 | public int hashCode() { 71 | int result = super.hashCode(); 72 | result = 31 * result + (partition != null ? partition.hashCode() : 0); 73 | return result; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/integration/ProducerTopicAutoCreationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.integration; 18 | 19 | import org.junit.Test; 20 | 21 | import java.util.Arrays; 22 | import java.util.List; 23 | import java.util.Properties; 24 | 25 | import io.confluent.kafkarest.entities.BinaryTopicProduceRecord; 26 | import io.confluent.kafkarest.entities.PartitionOffset; 27 | import kafka.serializer.Decoder; 28 | import kafka.serializer.DefaultDecoder; 29 | 30 | public class ProducerTopicAutoCreationTest extends AbstractProducerTest { 31 | 32 | private static final String topicName = "nonexistant"; 33 | 34 | private static final Decoder binaryDecoder = new DefaultDecoder(null); 35 | 36 | private final List topicRecords = Arrays.asList( 37 | new BinaryTopicProduceRecord("key".getBytes(), "value".getBytes()), 38 | new BinaryTopicProduceRecord("key".getBytes(), "value2".getBytes()), 39 | new BinaryTopicProduceRecord("key".getBytes(), "value3".getBytes()), 40 | new BinaryTopicProduceRecord("key".getBytes(), "value4".getBytes()) 41 | ); 42 | private final List partitionOffsets = Arrays.asList( 43 | new PartitionOffset(0, 0L, null, null), 44 | new PartitionOffset(0, 1L, null, null), 45 | new PartitionOffset(0, 2L, null, null), 46 | new PartitionOffset(0, 3L, null, null) 47 | ); 48 | 49 | public Properties overrideBrokerProperties(int i, Properties props) { 50 | Properties refined = (Properties) props.clone(); 51 | refined.setProperty("auto.create.topics.enable", "true"); 52 | return refined; 53 | } 54 | 55 | @Test 56 | public void testProduceToMissingTopic() { 57 | // Should create topic 58 | testProduceToTopic(topicName, topicRecords, binaryDecoder, binaryDecoder, partitionOffsets); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/resources/SpoolResource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.resources; 17 | 18 | import java.util.Collection; 19 | 20 | import javax.ws.rs.GET; 21 | import javax.ws.rs.POST; 22 | import javax.ws.rs.Path; 23 | import javax.ws.rs.PathParam; 24 | import javax.ws.rs.QueryParam; 25 | 26 | import io.confluent.kafkarest.Context; 27 | import io.confluent.kafkarest.SpoolProducer; 28 | import io.confluent.kafkarest.entities.SpoolMessage; 29 | import io.confluent.kafkarest.entities.SpoolShard; 30 | 31 | @Path("/spool") 32 | public class SpoolResource { 33 | 34 | private final Context ctx; 35 | 36 | public SpoolResource(Context ctx) { 37 | this.ctx = ctx; 38 | } 39 | 40 | @GET 41 | public Collection shards() { 42 | return SpoolProducer.getShards(); 43 | } 44 | 45 | @GET 46 | @Path("/{shard}") 47 | public SpoolShard shard(@PathParam("shard") int shard) { 48 | return SpoolProducer.getShard(shard); 49 | } 50 | 51 | @POST 52 | @Path("/{shard}/suspend") 53 | public void suspend(@PathParam("shard") int shard, 54 | @QueryParam("timestamp") long timestamp) { 55 | SpoolProducer.suspendShard(shard, timestamp); 56 | } 57 | 58 | @GET 59 | @Path("/{shard}/peek") 60 | public Collection peek(@PathParam("shard") int shard, 61 | @QueryParam("count") Integer count) throws Exception { 62 | return SpoolProducer.peekErroredRecords(shard, count == null ? 10 : count); 63 | } 64 | 65 | @POST 66 | @Path("/{shard}/revive") 67 | public void revive(@PathParam("shard") int shard, 68 | @QueryParam("count") Integer count) { 69 | SpoolProducer.reviveErroredRecords(shard, count == null ? 10 : count); 70 | } 71 | 72 | @POST 73 | @Path("/override") 74 | public void override(@QueryParam("always") boolean always) { 75 | SpoolProducer.overrideSpoolMode(always); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/Broker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import org.hibernate.validator.constraints.NotEmpty; 21 | 22 | //import javax.validation.constraints.NotNull; 23 | 24 | public class Broker { 25 | 26 | private int id; 27 | 28 | @NotEmpty 29 | private String host; 30 | 31 | private int port; 32 | 33 | public Broker(@JsonProperty("id") int id, 34 | @JsonProperty("host") String host, 35 | @JsonProperty("port") int port) { 36 | this.id = id; 37 | this.host = host; 38 | this.port = port; 39 | } 40 | 41 | @JsonProperty 42 | public int getId() { 43 | return id; 44 | } 45 | 46 | @JsonProperty 47 | public void setId(int id) { 48 | this.id = id; 49 | } 50 | 51 | @JsonProperty 52 | public String getHost() { 53 | return host; 54 | } 55 | 56 | @JsonProperty 57 | public void setHost(String host) { 58 | this.host = host; 59 | } 60 | 61 | @JsonProperty 62 | public int getPort() { 63 | return port; 64 | } 65 | 66 | @JsonProperty 67 | public void setPort(int port) { 68 | this.port = port; 69 | } 70 | 71 | @Override 72 | public boolean equals(Object o) { 73 | if (this == o) { 74 | return true; 75 | } 76 | if (o == null || getClass() != o.getClass()) { 77 | return false; 78 | } 79 | 80 | Broker broker = (Broker) o; 81 | 82 | if (id != broker.id) { 83 | return false; 84 | } 85 | 86 | if (host != null ? !host.equals(broker.host) : broker.host != null) { 87 | return false; 88 | } 89 | 90 | if (port != broker.port) { 91 | return false; 92 | } 93 | 94 | return true; 95 | } 96 | 97 | @Override 98 | public int hashCode() { 99 | int result = 31 * id + (host != null ? host.hashCode() : 0); 100 | result = 31 * result + port; 101 | return result; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /licenses/NOTICE.confluent-rest-utils.txt: -------------------------------------------------------------------------------- 1 | The following libraries are included in packaged versions of this project: 2 | 3 | * ClassMate 4 | * COPYRIGHT: Copyright 2010 The Apache Software Foundation 5 | * LICENSE: licenses/LICENSE.apache2.txt 6 | * HOMEPAGE: https://github.com/cowtowncoder/java-classmate 7 | 8 | * Confluent Common 9 | * COPYRIGHT: Confluent Inc. 10 | * LICENSE: licenses/LICENSE.apache2.txt 11 | * HOMEPAGE: https://github.com/confluentinc/common 12 | 13 | * Hamcrest 14 | * COPYRIGHT: Copyright (c) 2000-2006, www.hamcrest.org 15 | * LICENSE: licenses/LICENSE.bsd.txt 16 | * HOMEPAGE: http://hamcrest.org/ 17 | 18 | * Hibernate 19 | * COPYRIGHT: licenses/COPYRIGHT.hibernate.txt 20 | * LICENSE: licenses/LICENSE.apache2.txt 21 | * HOMEPAGE: http://hibernate.org/validator/ 22 | 23 | * HK2 24 | * COPYRIGHT: Copyright (c) 2010-2014 Oracle and/or its affiliates. All rights reserved. 25 | * LICENSE: licenses/LICENSE.cddl+gpl2.html 26 | * HOMEPAGE: https://hk2.java.net 27 | 28 | * Jackson annotations 29 | * LICENSE: licenses/LICENSE.jackson-annotations.txt (Apache 2) 30 | * HOMEPAGE: http://github.com/FasterXML/jackson 31 | 32 | * Jackson core 33 | * LICENSE: licenses/LICENSE.jackson-core.txt (Apache 2) 34 | * NOTICE: licenses/NOTICE.jackson-core.txt 35 | * HOMEPAGE: http://github.com/FasterXML/jackson 36 | 37 | * Jackson databind 38 | * LICENSE: licenses/LICENSE.jackson-databind.txt (Apache 2) 39 | * NOTICE: licenses/NOTICE.jackson-databind.txt 40 | * HOMEPAGE: http://github.com/FasterXML/jackson 41 | 42 | * Jackson jaxrs-json-provider 43 | * LICENSE: licenses/LICENSE.jackson-core.txt (Apache 2) 44 | * NOTICE: licenses/NOTICE.jackson-core.txt 45 | * HOMEPAGE: http://github.com/FasterXML/jackson 46 | 47 | * Javassist 48 | * COPYRIGHT: Copyright (C) 1999- by Shigeru Chiba, All rights reserved. 49 | * LICENSE: licenses/LICENSE.javassist.txt (MPL, LGPL, Apache 2) 50 | * HOMEPAGE: http://www.javassist.org 51 | 52 | * javax.annotation-api, javax.el, javax.el-api, javax.inject, javax.servlet, javax.ws.rs-api, javax.validation 53 | * COPYRIGHT: Coypright Oracle 54 | * LICENSE: licenses/LICENSE.cddl+gpl2.html 55 | 56 | * JBoss Logging 57 | * COPYRIGHT: Copyright 2014 Red Hat, Inc. 58 | * LICENSE: licenses/LICENSE.apache2.txt 59 | * HOMEPAGE: http://www.jboss.org 60 | 61 | * Jersey 62 | * LICENSE: licenses/LICENSE.cddl+gpl2.html 63 | * HOMEPAGE: http://jersey.java.net 64 | 65 | * Jetty 66 | * COPYRIGHT: Copyright Mort Bay Consulting Pty Ltd unless otherwise noted 67 | * LICENSE: licenses/LICENSE.apache2.txt, licenses/LICENSE.epl.html 68 | * NOTICE: licenses/NOTICE.jetty.txt 69 | * HOMEPAGE: http://eclipse.org/jetty/ 70 | 71 | * JUnit 72 | * LICENSE: licenses/LICENSE.epl.txt 73 | * NOTICE: licenses/NOTICE.junit.txt 74 | * HOMEPAGE: http://junit.org/ 75 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/ConsumerTopicState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | import java.util.concurrent.locks.Lock; 22 | import java.util.concurrent.locks.ReentrantLock; 23 | 24 | import kafka.consumer.ConsumerIterator; 25 | import kafka.consumer.KafkaStream; 26 | 27 | /** 28 | * Tracks a consumer's state for a single topic, including the underlying stream and consumed and 29 | * committed offsets. It provides manual synchronization primitives to support ConsumerWorkers 30 | * protecting access to the state while they process a read request in their processing loop. 31 | */ 32 | public class ConsumerTopicState { 33 | 34 | private final Lock lock = new ReentrantLock(); 35 | private final KafkaStream stream; 36 | private final Map consumedOffsets; 37 | private final Map committedOffsets; 38 | 39 | // The last read task on this topic that failed. Allows the next read to pick up where this one 40 | // left off, including accounting for response size limits 41 | private ConsumerReadTask failedTask; 42 | 43 | public ConsumerTopicState(KafkaStream stream) { 44 | this.stream = stream; 45 | this.consumedOffsets = new HashMap(); 46 | this.committedOffsets = new HashMap(); 47 | } 48 | 49 | public void lock() { 50 | lock.lock(); 51 | } 52 | 53 | public void unlock() { 54 | lock.unlock(); 55 | } 56 | 57 | public KafkaStream getStream() { 58 | return stream; 59 | } 60 | 61 | public ConsumerIterator getIterator() { 62 | return stream.iterator(); 63 | } 64 | 65 | public Map getConsumedOffsets() { 66 | return consumedOffsets; 67 | } 68 | 69 | public Map getCommittedOffsets() { 70 | return committedOffsets; 71 | } 72 | 73 | public ConsumerReadTask clearFailedTask() { 74 | ConsumerReadTask t = failedTask; 75 | failedTask = null; 76 | return t; 77 | } 78 | 79 | public void setFailedTask(ConsumerReadTask failedTask) { 80 | this.failedTask = failedTask; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/AvroConsumerState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | 21 | import java.util.Properties; 22 | 23 | import io.confluent.kafka.serializers.KafkaAvroDecoder; 24 | import io.confluent.kafkarest.converters.AvroConverter; 25 | import io.confluent.kafkarest.entities.AvroConsumerRecord; 26 | import kafka.javaapi.consumer.ConsumerConnector; 27 | import kafka.message.MessageAndMetadata; 28 | import kafka.serializer.Decoder; 29 | import kafka.utils.VerifiableProperties; 30 | 31 | /** 32 | * Avro implementation of ConsumerState, which decodes into GenericRecords or primitive types. 33 | */ 34 | public class AvroConsumerState extends ConsumerState { 35 | 36 | // Note that this could be a static variable and shared, but that causes tests to break in 37 | // subtle ways because it causes state to be shared across tests, but only for the consumer. 38 | private Decoder decoder = null; 39 | 40 | public AvroConsumerState(KafkaRestConfig config, 41 | ConsumerInstanceId instanceId, 42 | ConsumerConnector consumer) { 43 | super(config, instanceId, consumer); 44 | Properties props = new Properties(); 45 | props.setProperty("schema.registry.url", 46 | config.getString(KafkaRestConfig.SCHEMA_REGISTRY_URL_CONFIG)); 47 | decoder = new KafkaAvroDecoder(new VerifiableProperties(props)); 48 | } 49 | 50 | @Override 51 | protected Decoder getKeyDecoder() { 52 | return decoder; 53 | } 54 | 55 | @Override 56 | protected Decoder getValueDecoder() { 57 | return decoder; 58 | } 59 | 60 | @Override 61 | public ConsumerRecordAndSize createConsumerRecord( 62 | MessageAndMetadata msg) { 63 | AvroConverter.JsonNodeAndSize keyNode = AvroConverter.toJson(msg.key()); 64 | AvroConverter.JsonNodeAndSize valueNode = AvroConverter.toJson(msg.message()); 65 | return new ConsumerRecordAndSize( 66 | new AvroConsumerRecord(keyNode.json, valueNode.json, msg.partition(), msg.offset()), 67 | keyNode.size + valueNode.size 68 | ); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/PartitionReplica.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import javax.validation.constraints.Min; 21 | 22 | public class PartitionReplica { 23 | 24 | @Min(0) 25 | private int broker; 26 | private boolean leader; 27 | private boolean inSync; 28 | 29 | public PartitionReplica() { 30 | } 31 | 32 | public PartitionReplica(int broker, boolean leader, boolean inSync) { 33 | this.broker = broker; 34 | this.leader = leader; 35 | this.inSync = inSync; 36 | } 37 | 38 | @JsonProperty 39 | public int getBroker() { 40 | return broker; 41 | } 42 | 43 | @JsonProperty 44 | public void setBroker(int broker) { 45 | this.broker = broker; 46 | } 47 | 48 | @JsonProperty 49 | public boolean isLeader() { 50 | return leader; 51 | } 52 | 53 | @JsonProperty 54 | public void setLeader(boolean leader) { 55 | this.leader = leader; 56 | } 57 | 58 | @JsonProperty("in_sync") 59 | public boolean isInSync() { 60 | return inSync; 61 | } 62 | 63 | @JsonProperty("in_sync") 64 | public void setInSync(boolean inSync) { 65 | this.inSync = inSync; 66 | } 67 | 68 | @Override 69 | public boolean equals(Object o) { 70 | if (this == o) { 71 | return true; 72 | } 73 | if (!(o instanceof PartitionReplica)) { 74 | return false; 75 | } 76 | 77 | PartitionReplica that = (PartitionReplica) o; 78 | 79 | if (broker != that.broker) { 80 | return false; 81 | } 82 | if (inSync != that.inSync) { 83 | return false; 84 | } 85 | if (leader != that.leader) { 86 | return false; 87 | } 88 | 89 | return true; 90 | } 91 | 92 | @Override 93 | public int hashCode() { 94 | int result = broker; 95 | result = 31 * result + (leader ? 1 : 0); 96 | result = 31 * result + (inSync ? 1 : 0); 97 | return result; 98 | } 99 | 100 | @Override 101 | public String toString() { 102 | return "PartitionReplica{" + 103 | "broker=" + broker + 104 | ", leader=" + leader + 105 | ", inSync=" + inSync + 106 | '}'; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/BinaryTopicProduceRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import java.io.IOException; 22 | 23 | import javax.validation.constraints.Min; 24 | 25 | public class BinaryTopicProduceRecord extends BinaryProduceRecord 26 | implements TopicProduceRecord { 27 | 28 | // When producing to a topic, a partition may be explicitly requested. 29 | @Min(0) 30 | protected Integer partition; 31 | 32 | public BinaryTopicProduceRecord(@JsonProperty("key") String key, 33 | @JsonProperty("value") String value, 34 | @JsonProperty("partition") Integer partition) throws IOException { 35 | super(key, value); 36 | this.partition = partition; 37 | } 38 | 39 | public BinaryTopicProduceRecord(byte[] key, byte[] value, Integer partition) { 40 | super(key, value); 41 | this.partition = partition; 42 | } 43 | 44 | public BinaryTopicProduceRecord(byte[] key, byte[] value) { 45 | this(key, value, null); 46 | } 47 | 48 | public BinaryTopicProduceRecord(byte[] value, Integer partition) { 49 | this(null, value, partition); 50 | } 51 | 52 | public BinaryTopicProduceRecord(byte[] value) { 53 | this(null, value, null); 54 | } 55 | 56 | @JsonProperty 57 | public Integer getPartition() { 58 | return partition; 59 | } 60 | 61 | @JsonProperty 62 | public void setPartition(Integer partition) { 63 | this.partition = partition; 64 | } 65 | 66 | @Override 67 | public boolean equals(Object o) { 68 | if (this == o) { 69 | return true; 70 | } 71 | if (o == null || getClass() != o.getClass()) { 72 | return false; 73 | } 74 | if (!super.equals(o)) { 75 | return false; 76 | } 77 | 78 | BinaryTopicProduceRecord that = (BinaryTopicProduceRecord) o; 79 | 80 | if (partition != null ? !partition.equals(that.partition) : that.partition != null) { 81 | return false; 82 | } 83 | 84 | return true; 85 | } 86 | 87 | @Override 88 | public int hashCode() { 89 | int result = super.hashCode(); 90 | result = 31 * result + (partition != null ? partition.hashCode() : 0); 91 | return result; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/BinaryProduceRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonCreator; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | import java.io.IOException; 23 | import java.util.Arrays; 24 | 25 | import io.confluent.rest.validation.ConstraintViolations; 26 | 27 | public class BinaryProduceRecord extends ProduceRecordBase { 28 | 29 | @JsonCreator 30 | public BinaryProduceRecord(@JsonProperty("key") String key, @JsonProperty("value") String value) 31 | throws IOException { 32 | super(null, null); 33 | try { 34 | this.key = (key != null) ? EntityUtils.parseBase64Binary(key) : null; 35 | } catch (IllegalArgumentException e) { 36 | throw ConstraintViolations.simpleException("Record key contains invalid base64 encoding"); 37 | } 38 | try { 39 | this.value = (value != null) ? EntityUtils.parseBase64Binary(value) : null; 40 | } catch (IllegalArgumentException e) { 41 | throw ConstraintViolations.simpleException("Record value contains invalid base64 encoding"); 42 | } 43 | } 44 | 45 | public BinaryProduceRecord(byte[] key, byte[] value) { 46 | super(key, value); 47 | } 48 | 49 | public BinaryProduceRecord(byte[] unencoded_value) { 50 | this(null, unencoded_value); 51 | } 52 | 53 | @Override 54 | @JsonProperty("key") 55 | public String getJsonKey() { 56 | return (key == null ? null : EntityUtils.encodeBase64Binary(key)); 57 | } 58 | 59 | @Override 60 | @JsonProperty("value") 61 | public String getJsonValue() { 62 | return (value == null ? null : EntityUtils.encodeBase64Binary(value)); 63 | } 64 | 65 | @Override 66 | public boolean equals(Object o) { 67 | if (this == o) { 68 | return true; 69 | } 70 | if (o == null || getClass() != o.getClass()) { 71 | return false; 72 | } 73 | 74 | BinaryProduceRecord that = (BinaryProduceRecord) o; 75 | 76 | if (!Arrays.equals(key, that.key)) { 77 | return false; 78 | } 79 | if (!Arrays.equals(value, that.value)) { 80 | return false; 81 | } 82 | 83 | return true; 84 | } 85 | 86 | @Override 87 | public int hashCode() { 88 | int result = key != null ? Arrays.hashCode(key) : 0; 89 | result = 31 * result + (value != null ? Arrays.hashCode(value) : 0); 90 | return result; 91 | } 92 | } -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/unit/BrokersResourceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.unit; 17 | 18 | import org.easymock.EasyMock; 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | 22 | import java.util.Arrays; 23 | import java.util.List; 24 | 25 | import javax.ws.rs.core.GenericType; 26 | import javax.ws.rs.core.Response; 27 | 28 | import io.confluent.kafkarest.Context; 29 | import io.confluent.kafkarest.KafkaRestApplication; 30 | import io.confluent.kafkarest.KafkaRestConfig; 31 | import io.confluent.kafkarest.MetadataObserver; 32 | import io.confluent.kafkarest.ProducerPool; 33 | import io.confluent.kafkarest.TestUtils; 34 | import io.confluent.kafkarest.entities.BrokerList; 35 | import io.confluent.kafkarest.resources.BrokersResource; 36 | import io.confluent.rest.EmbeddedServerTestHarness; 37 | import io.confluent.rest.RestConfigException; 38 | 39 | import static io.confluent.kafkarest.TestUtils.assertOKResponse; 40 | import static org.junit.Assert.assertEquals; 41 | 42 | public class BrokersResourceTest 43 | extends EmbeddedServerTestHarness { 44 | 45 | private MetadataObserver mdObserver; 46 | private ProducerPool producerPool; 47 | private Context ctx; 48 | 49 | public BrokersResourceTest() throws RestConfigException { 50 | mdObserver = EasyMock.createMock(MetadataObserver.class); 51 | producerPool = EasyMock.createMock(ProducerPool.class); 52 | ctx = new Context(config, mdObserver, producerPool, null); 53 | addResource(new BrokersResource(ctx)); 54 | } 55 | 56 | @Before 57 | public void setUp() throws Exception { 58 | super.setUp(); 59 | EasyMock.reset(mdObserver, producerPool); 60 | } 61 | 62 | @Test 63 | public void testList() { 64 | for (TestUtils.RequestMediaType mediatype : TestUtils.V1_ACCEPT_MEDIATYPES) { 65 | final List brokerIds = Arrays.asList(1, 2, 3); 66 | EasyMock.expect(mdObserver.getBrokerIds()).andReturn(brokerIds); 67 | EasyMock.replay(mdObserver); 68 | 69 | Response response = request("/brokers", mediatype.header).get(); 70 | assertOKResponse(response, mediatype.expected); 71 | final BrokerList returnedBrokerIds = response.readEntity(new GenericType() { 72 | }); 73 | assertEquals(brokerIds, returnedBrokerIds.getBrokers()); 74 | EasyMock.verify(mdObserver); 75 | EasyMock.reset(mdObserver, producerPool); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/Partition.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import java.util.List; 21 | 22 | import javax.validation.constraints.Min; 23 | 24 | public class Partition { 25 | 26 | @Min(0) 27 | private int partition; 28 | @Min(0) 29 | private int leader; 30 | private List replicas; 31 | 32 | public Partition() { 33 | } 34 | 35 | public Partition(@JsonProperty int partition, @JsonProperty int leader, 36 | @JsonProperty List replicas) { 37 | this.partition = partition; 38 | this.leader = leader; 39 | this.replicas = replicas; 40 | } 41 | 42 | @JsonProperty 43 | public int getPartition() { 44 | return partition; 45 | } 46 | 47 | @JsonProperty 48 | public void setPartition(int partition) { 49 | this.partition = partition; 50 | } 51 | 52 | @JsonProperty 53 | public int getLeader() { 54 | return leader; 55 | } 56 | 57 | @JsonProperty 58 | public void setLeader(int leader) { 59 | this.leader = leader; 60 | } 61 | 62 | @JsonProperty 63 | public List getReplicas() { 64 | return replicas; 65 | } 66 | 67 | @JsonProperty 68 | public void setReplicas(List replicas) { 69 | this.replicas = replicas; 70 | } 71 | 72 | @Override 73 | public boolean equals(Object o) { 74 | if (this == o) { 75 | return true; 76 | } 77 | if (!(o instanceof Partition)) { 78 | return false; 79 | } 80 | 81 | Partition partition1 = (Partition) o; 82 | 83 | if (leader != partition1.leader) { 84 | return false; 85 | } 86 | if (partition != partition1.partition) { 87 | return false; 88 | } 89 | if (replicas != null ? !replicas.equals(partition1.replicas) : partition1.replicas != null) { 90 | return false; 91 | } 92 | 93 | return true; 94 | } 95 | 96 | @Override 97 | public int hashCode() { 98 | int result = partition; 99 | result = 31 * result + leader; 100 | result = 31 * result + (replicas != null ? replicas.hashCode() : 0); 101 | return result; 102 | } 103 | 104 | @Override 105 | public String toString() { 106 | return "Partition{" + 107 | "partition=" + partition + 108 | ", leader=" + leader + 109 | ", replicas=" + replicas + 110 | '}'; 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/Topic.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import org.hibernate.validator.constraints.NotEmpty; 21 | 22 | import java.util.List; 23 | import java.util.Properties; 24 | 25 | import javax.validation.constraints.NotNull; 26 | 27 | public class Topic { 28 | 29 | @NotEmpty 30 | private String name; 31 | 32 | @NotNull 33 | private Properties configs; 34 | 35 | @NotEmpty 36 | private List partitions; 37 | 38 | public Topic(@JsonProperty("name") String name, 39 | @JsonProperty("configs") Properties configs, 40 | @JsonProperty("partitions") List partitions) { 41 | this.name = name; 42 | this.configs = configs; 43 | this.partitions = partitions; 44 | } 45 | 46 | @JsonProperty 47 | public String getName() { 48 | return name; 49 | } 50 | 51 | @JsonProperty 52 | public void setName(String name) { 53 | this.name = name; 54 | } 55 | 56 | @JsonProperty 57 | public Properties getConfigs() { 58 | return configs; 59 | } 60 | 61 | @JsonProperty 62 | public void setConfigs(Properties configs) { 63 | this.configs = configs; 64 | } 65 | 66 | @JsonProperty 67 | public List getPartitions() { 68 | return partitions; 69 | } 70 | 71 | @JsonProperty 72 | public void setPartitions(List partitions) { 73 | this.partitions = partitions; 74 | } 75 | 76 | @Override 77 | public boolean equals(Object o) { 78 | if (this == o) { 79 | return true; 80 | } 81 | if (o == null || getClass() != o.getClass()) { 82 | return false; 83 | } 84 | 85 | Topic topic = (Topic) o; 86 | 87 | if (configs != null ? !configs.equals(topic.configs) : topic.configs != null) { 88 | return false; 89 | } 90 | if (name != null ? !name.equals(topic.name) : topic.name != null) { 91 | return false; 92 | } 93 | if (partitions != null ? !partitions.equals(topic.partitions) : topic.partitions != null) { 94 | return false; 95 | } 96 | 97 | return true; 98 | } 99 | 100 | @Override 101 | public int hashCode() { 102 | int result = name != null ? name.hashCode() : 0; 103 | result = 31 * result + (partitions != null ? partitions.hashCode() : 0); 104 | result = 31 * result + (configs != null ? configs.hashCode() : 0); 105 | return result; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/integration/AbstractProducerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.integration; 18 | 19 | import java.util.List; 20 | 21 | import javax.ws.rs.client.Entity; 22 | import javax.ws.rs.core.Response; 23 | 24 | import io.confluent.kafkarest.TestUtils; 25 | import io.confluent.kafkarest.Versions; 26 | import io.confluent.kafkarest.entities.PartitionOffset; 27 | import io.confluent.kafkarest.entities.ProduceResponse; 28 | import io.confluent.kafkarest.entities.TopicProduceRecord; 29 | import io.confluent.kafkarest.entities.TopicProduceRequest; 30 | import kafka.serializer.Decoder; 31 | 32 | import static io.confluent.kafkarest.TestUtils.assertOKResponse; 33 | import static org.junit.Assert.assertNotNull; 34 | 35 | public class AbstractProducerTest extends ClusterTestHarness { 36 | 37 | protected void testProduceToTopic(String topicName, 38 | List records, 39 | Decoder keyDecoder, Decoder valueDecoder, 40 | List offsetResponses) { 41 | TopicProduceRequest payload = new TopicProduceRequest(); 42 | payload.setRecords(records); 43 | Response response = request("/topics/" + topicName) 44 | .post(Entity.entity(payload, Versions.KAFKA_MOST_SPECIFIC_DEFAULT)); 45 | assertOKResponse(response, Versions.KAFKA_MOST_SPECIFIC_DEFAULT); 46 | final ProduceResponse produceResponse = response.readEntity(ProduceResponse.class); 47 | TestUtils.assertPartitionOffsetsEqual(offsetResponses, produceResponse.getOffsets()); 48 | TestUtils.assertTopicContains(zkConnect, topicName, 49 | payload.getRecords(), null, 50 | keyDecoder, valueDecoder, true); 51 | } 52 | 53 | protected void testProduceToTopicFails(String topicName, 54 | List records) { 55 | TopicProduceRequest payload = new TopicProduceRequest(); 56 | payload.setRecords(records); 57 | Response response = request("/topics/" + topicName) 58 | .post(Entity.entity(payload, Versions.KAFKA_MOST_SPECIFIC_DEFAULT)); 59 | assertOKResponse(response, Versions.KAFKA_MOST_SPECIFIC_DEFAULT); 60 | final ProduceResponse produceResponse = response.readEntity(ProduceResponse.class); 61 | for (PartitionOffset pOffset : produceResponse.getOffsets()) { 62 | assertNotNull(pOffset.getError()); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/unit/UriUtilsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.unit; 18 | 19 | import org.easymock.EasyMock; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | 23 | import java.net.URI; 24 | import java.util.Properties; 25 | 26 | import javax.ws.rs.core.UriBuilder; 27 | import javax.ws.rs.core.UriInfo; 28 | 29 | import io.confluent.kafkarest.KafkaRestConfig; 30 | import io.confluent.kafkarest.UriUtils; 31 | import io.confluent.rest.RestConfigException; 32 | 33 | import static org.junit.Assert.assertEquals; 34 | 35 | public class UriUtilsTest { 36 | 37 | private UriInfo uriInfo; 38 | 39 | @Before 40 | public void setUp() { 41 | uriInfo = EasyMock.createMock(UriInfo.class); 42 | } 43 | 44 | @Test 45 | public void testAbsoluteURIBuilderDefaultHost() throws RestConfigException { 46 | KafkaRestConfig config = new KafkaRestConfig(); 47 | EasyMock.expect(uriInfo.getAbsolutePathBuilder()) 48 | .andReturn(UriBuilder.fromUri("http://foo.com")); 49 | EasyMock.replay(uriInfo); 50 | assertEquals("http://foo.com", UriUtils.absoluteUriBuilder(config, uriInfo).build().toString()); 51 | EasyMock.verify(uriInfo); 52 | } 53 | 54 | @Test 55 | public void testAbsoluteURIBuilderOverrideHost() throws RestConfigException { 56 | Properties props = new Properties(); 57 | props.put(KafkaRestConfig.HOST_NAME_CONFIG, "bar.net"); 58 | KafkaRestConfig config = new KafkaRestConfig(props); 59 | EasyMock.expect(uriInfo.getAbsolutePathBuilder()) 60 | .andReturn(UriBuilder.fromUri("http://foo.com")); 61 | EasyMock.expect(uriInfo.getAbsolutePath()).andReturn(URI.create("http://foo.com")); 62 | EasyMock.replay(uriInfo); 63 | assertEquals("http://bar.net", UriUtils.absoluteUriBuilder(config, uriInfo).build().toString()); 64 | EasyMock.verify(uriInfo); 65 | } 66 | 67 | @Test 68 | public void testAbsoluteURIBuilderWithPort() throws RestConfigException { 69 | Properties props = new Properties(); 70 | props.put(KafkaRestConfig.HOST_NAME_CONFIG, "bar.net"); 71 | props.put(KafkaRestConfig.PORT_CONFIG, 5000); 72 | KafkaRestConfig config = new KafkaRestConfig(props); 73 | EasyMock.expect(uriInfo.getAbsolutePathBuilder()) 74 | .andReturn(UriBuilder.fromUri("http://foo.com:5000")); 75 | EasyMock.expect(uriInfo.getAbsolutePath()).andReturn(URI.create("http://foo.com:5000")); 76 | EasyMock.replay(uriInfo); 77 | assertEquals("http://bar.net:5000", 78 | UriUtils.absoluteUriBuilder(config, uriInfo).build().toString()); 79 | EasyMock.verify(uriInfo); 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/ProduceTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import org.apache.kafka.clients.producer.Callback; 20 | import org.apache.kafka.clients.producer.RecordMetadata; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | import io.confluent.kafkarest.entities.SchemaHolder; 28 | 29 | /** 30 | * Container for state associated with one REST-ful produce request, i.e. a batched send 31 | */ 32 | class ProduceTask { 33 | 34 | private static final Logger log = LoggerFactory.getLogger(ProduceTask.class); 35 | 36 | private final SchemaHolder schemaHolder; 37 | private final int numRecords; 38 | private final ProducerPool.ProduceRequestCallback callback; 39 | private int completed; 40 | private Integer keySchemaId; 41 | private Integer valueSchemaId; 42 | private List results; 43 | 44 | public ProduceTask(SchemaHolder schemaHolder, int numRecords, 45 | ProducerPool.ProduceRequestCallback callback) { 46 | this.schemaHolder = schemaHolder; 47 | this.numRecords = numRecords; 48 | this.callback = callback; 49 | this.completed = 0; 50 | this.results = new ArrayList(); 51 | } 52 | 53 | public synchronized Callback createCallback() { 54 | final int index = results.size(); 55 | // Dummy data, which just helps us keep track of the index & ensures there's a slot for 56 | // storage when we get the callback 57 | results.add(null); 58 | return new Callback() { 59 | @Override 60 | public void onCompletion(RecordMetadata metadata, Exception exception) { 61 | ProduceTask.this.onCompletion(index, metadata, exception); 62 | } 63 | }; 64 | } 65 | 66 | public synchronized void onCompletion(int messageNum, RecordMetadata metadata, 67 | Exception exception) { 68 | results.set(messageNum, new RecordMetadataOrException(metadata, exception)); 69 | 70 | if (exception != null) { 71 | log.error("Producer error for request " + this.toString(), exception); 72 | } 73 | 74 | completed += 1; 75 | 76 | if (completed == numRecords) { 77 | this.callback.onCompletion(keySchemaId, valueSchemaId, results); 78 | } 79 | } 80 | 81 | public SchemaHolder getSchemaHolder() { 82 | return schemaHolder; 83 | } 84 | 85 | public void setSchemaIds(Integer keySchemaId, Integer valueSchemaId) { 86 | this.keySchemaId = keySchemaId; 87 | this.valueSchemaId = valueSchemaId; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/integration/ConsumerTimeoutTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.integration; 17 | 18 | import org.junit.Before; 19 | import org.junit.Test; 20 | 21 | import java.util.List; 22 | import java.util.Properties; 23 | 24 | import javax.ws.rs.core.GenericType; 25 | 26 | import io.confluent.kafkarest.Versions; 27 | import io.confluent.kafkarest.entities.BinaryConsumerRecord; 28 | import io.confluent.kafkarest.entities.EmbeddedFormat; 29 | import kafka.utils.TestUtils; 30 | import scala.collection.JavaConversions; 31 | 32 | public class ConsumerTimeoutTest extends AbstractConsumerTest { 33 | 34 | private static final String topicName = "test"; 35 | private static final String groupName = "testconsumergroup"; 36 | 37 | private static final Integer requestTimeout = 500; 38 | // This is pretty large since there is sometimes significant overhead to doing a read (e.g. 39 | // checking topic existence in ZK) 40 | private static final Integer instanceTimeout = 1000; 41 | private static final Integer slackTime = 5; 42 | 43 | @Before 44 | @Override 45 | public void setUp() throws Exception { 46 | restProperties.setProperty("consumer.request.timeout.ms", requestTimeout.toString()); 47 | restProperties.setProperty("consumer.instance.timeout.ms", instanceTimeout.toString()); 48 | super.setUp(); 49 | final int numPartitions = 3; 50 | final int replicationFactor = 1; 51 | TestUtils.createTopic(zkClient, topicName, numPartitions, replicationFactor, 52 | JavaConversions.asScalaIterable(this.servers).toSeq(), new Properties()); 53 | } 54 | 55 | @Test 56 | public void testConsumerTimeout() throws InterruptedException { 57 | String instanceUri = startConsumeMessages(groupName, topicName, EmbeddedFormat.BINARY, 58 | Versions.KAFKA_V1_JSON_BINARY, 59 | Versions.KAFKA_V1_JSON_BINARY); 60 | // Even with identical timeouts, should be able to consume multiple times without the 61 | // instance timing out 62 | consumeForTimeout(instanceUri, topicName, 63 | Versions.KAFKA_V1_JSON_BINARY, Versions.KAFKA_V1_JSON_BINARY, 64 | new GenericType>() { 65 | }); 66 | consumeForTimeout(instanceUri, topicName, 67 | Versions.KAFKA_V1_JSON_BINARY, Versions.KAFKA_V1_JSON_BINARY, 68 | new GenericType>() { 69 | }); 70 | // Then sleep long enough for it to expire 71 | Thread.sleep(instanceTimeout + slackTime); 72 | 73 | consumeForNotFoundError(instanceUri, topicName); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/Versions.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | import java.util.Arrays; 19 | import java.util.List; 20 | 21 | public class Versions { 22 | 23 | public static final String KAFKA_V1_JSON = "application/vnd.kafka.v1+json"; 24 | // This is set < 1 because it is only the most-specific type if there isn't an embedded data type. 25 | public static final String KAFKA_V1_JSON_WEIGHTED = KAFKA_V1_JSON + "; qs=0.9"; 26 | public static final String KAFKA_V1_JSON_BINARY = "application/vnd.kafka.binary.v1+json"; 27 | public static final String KAFKA_V1_JSON_BINARY_WEIGHTED = KAFKA_V1_JSON_BINARY; 28 | // "LOW" weightings are used to permit using these for resources like consumer where it might 29 | // be convenient to always use the same type, but where their use should really be discouraged 30 | public static final String KAFKA_V1_JSON_BINARY_WEIGHTED_LOW = KAFKA_V1_JSON_BINARY + "; qs=0.1"; 31 | public static final String KAFKA_V1_JSON_AVRO = "application/vnd.kafka.avro.v1+json"; 32 | public static final String KAFKA_V1_JSON_AVRO_WEIGHTED = KAFKA_V1_JSON_AVRO; 33 | public static final String KAFKA_V1_JSON_AVRO_WEIGHTED_LOW = KAFKA_V1_JSON_AVRO + "; qs=0.1"; 34 | 35 | // These are defaults that track the most recent API version. These should always be specified 36 | // anywhere the latest version is produced/consumed. 37 | public static final String KAFKA_MOST_SPECIFIC_DEFAULT = KAFKA_V1_JSON; 38 | public static final String KAFKA_DEFAULT_JSON = "application/vnd.kafka+json"; 39 | public static final String KAFKA_DEFAULT_JSON_WEIGHTED = KAFKA_DEFAULT_JSON + "; qs=0.8"; 40 | public static final String JSON = "application/json"; 41 | public static final String JSON_WEIGHTED = JSON + "; qs=0.5"; 42 | 43 | // This is a fallback for when no type is provided. You usually should not need this. It is 44 | // mostly useful if you have two resource methods for the same endpoints but different Accept 45 | // or Content-Types. Adding this to one of them makes it the default even if no content type 46 | // information was specified in the request headers. 47 | public static final String ANYTHING = "*/*"; 48 | 49 | 50 | public static final List 51 | PREFERRED_RESPONSE_TYPES = 52 | Arrays.asList(Versions.KAFKA_V1_JSON, Versions.KAFKA_DEFAULT_JSON, Versions.JSON); 53 | 54 | // This type is completely generic and carries no actual information about the type of data, 55 | // but it is the default for request entities if no content type is specified. Well behaving 56 | // users of the API will always specify the content type, but ad hoc use may omit it. We treat 57 | // this as JSON since that's all we currently support. 58 | public static final String GENERIC_REQUEST = "application/octet-stream"; 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/BinaryConsumerRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest.entities; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import java.io.IOException; 22 | import java.util.Arrays; 23 | 24 | import io.confluent.rest.validation.ConstraintViolations; 25 | 26 | public class BinaryConsumerRecord extends ConsumerRecord { 27 | 28 | public BinaryConsumerRecord( 29 | @JsonProperty("key") String key, @JsonProperty("value") String value, 30 | @JsonProperty("partition") int partition, @JsonProperty("offset") long offset 31 | ) throws IOException { 32 | super(partition, offset); 33 | try { 34 | if (key != null) { 35 | this.key = EntityUtils.parseBase64Binary(key); 36 | } 37 | } catch (IllegalArgumentException e) { 38 | throw ConstraintViolations.simpleException("Record key contains invalid base64 encoding"); 39 | } 40 | try { 41 | this.value = EntityUtils.parseBase64Binary(value); 42 | } catch (IllegalArgumentException e) { 43 | throw ConstraintViolations.simpleException("Record value contains invalid base64 encoding"); 44 | } 45 | } 46 | 47 | public BinaryConsumerRecord(byte[] key, byte[] value, int partition, long offset) { 48 | super(key, value, partition, offset); 49 | } 50 | 51 | @Override 52 | @JsonProperty("key") 53 | public String getJsonKey() { 54 | if (key == null) { 55 | return null; 56 | } 57 | return EntityUtils.encodeBase64Binary(key); 58 | } 59 | 60 | @Override 61 | @JsonProperty("value") 62 | public String getJsonValue() { 63 | if (value == null) { 64 | return null; 65 | } 66 | return EntityUtils.encodeBase64Binary(value); 67 | } 68 | 69 | @Override 70 | public boolean equals(Object o) { 71 | if (this == o) { 72 | return true; 73 | } 74 | if (o == null || getClass() != o.getClass()) { 75 | return false; 76 | } 77 | 78 | BinaryConsumerRecord that = (BinaryConsumerRecord) o; 79 | 80 | if (offset != that.offset) { 81 | return false; 82 | } 83 | if (partition != that.partition) { 84 | return false; 85 | } 86 | if (!Arrays.equals(key, that.key)) { 87 | return false; 88 | } 89 | if (!Arrays.equals(value, that.value)) { 90 | return false; 91 | } 92 | 93 | return true; 94 | } 95 | 96 | @Override 97 | public int hashCode() { 98 | int result = key != null ? Arrays.hashCode(key) : 0; 99 | result = 31 * result + (value != null ? Arrays.hashCode(value) : 0); 100 | result = 31 * result + partition; 101 | result = 31 * result + (int) (offset ^ (offset >>> 32)); 102 | return result; 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/TopicPartitionOffset.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import org.hibernate.validator.constraints.NotEmpty; 21 | 22 | import javax.validation.constraints.Min; 23 | 24 | public class TopicPartitionOffset { 25 | 26 | @NotEmpty 27 | private String topic; 28 | @Min(0) 29 | private int partition; 30 | @Min(0) 31 | private long consumed; 32 | @Min(0) 33 | private long committed; 34 | 35 | public TopicPartitionOffset( 36 | @JsonProperty("topic") String topic, @JsonProperty("partition") int partition, 37 | @JsonProperty("consumed") long consumed, @JsonProperty("committed") long committed 38 | ) { 39 | this.topic = topic; 40 | this.partition = partition; 41 | this.consumed = consumed; 42 | this.committed = committed; 43 | } 44 | 45 | @JsonProperty 46 | public String getTopic() { 47 | return topic; 48 | } 49 | 50 | @JsonProperty 51 | public void setTopic(String topic) { 52 | this.topic = topic; 53 | } 54 | 55 | @JsonProperty 56 | public int getPartition() { 57 | return partition; 58 | } 59 | 60 | @JsonProperty 61 | public void setPartition(int partition) { 62 | this.partition = partition; 63 | } 64 | 65 | @JsonProperty 66 | public long getConsumed() { 67 | return consumed; 68 | } 69 | 70 | @JsonProperty 71 | public void setConsumed(long consumed) { 72 | this.consumed = consumed; 73 | } 74 | 75 | @JsonProperty 76 | public long getCommitted() { 77 | return committed; 78 | } 79 | 80 | @JsonProperty 81 | public void setCommitted(long committed) { 82 | this.committed = committed; 83 | } 84 | 85 | @Override 86 | public boolean equals(Object o) { 87 | if (this == o) { 88 | return true; 89 | } 90 | if (o == null || getClass() != o.getClass()) { 91 | return false; 92 | } 93 | 94 | TopicPartitionOffset that = (TopicPartitionOffset) o; 95 | 96 | if (committed != that.committed) { 97 | return false; 98 | } 99 | if (consumed != that.consumed) { 100 | return false; 101 | } 102 | if (partition != that.partition) { 103 | return false; 104 | } 105 | if (topic != null ? !topic.equals(that.topic) : that.topic != null) { 106 | return false; 107 | } 108 | 109 | return true; 110 | } 111 | 112 | @Override 113 | public int hashCode() { 114 | int result = topic != null ? topic.hashCode() : 0; 115 | result = 31 * result + partition; 116 | result = 31 * result + (int) (consumed ^ (consumed >>> 32)); 117 | result = 31 * result + (int) (committed ^ (committed >>> 32)); 118 | return result; 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/ConsumerRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import javax.validation.constraints.Min; 22 | import javax.validation.constraints.NotNull; 23 | 24 | public abstract class ConsumerRecord { 25 | 26 | protected K key; 27 | @NotNull 28 | protected V value; 29 | 30 | @Min(0) 31 | protected int partition; 32 | 33 | @Min(0) 34 | protected long offset; 35 | 36 | public ConsumerRecord(K key, V value, int partition, long offset) { 37 | this.key = key; 38 | this.value = value; 39 | this.partition = partition; 40 | this.offset = offset; 41 | } 42 | 43 | public ConsumerRecord(int partition, long offset) { 44 | this(null, null, partition, offset); 45 | } 46 | 47 | @JsonIgnore 48 | public K getKey() { 49 | return key; 50 | } 51 | 52 | @JsonProperty("key") 53 | public Object getJsonKey() { 54 | return key; 55 | } 56 | 57 | @JsonIgnore 58 | public void setKey(K key) { 59 | this.key = key; 60 | } 61 | 62 | @JsonIgnore 63 | public V getValue() { 64 | return value; 65 | } 66 | 67 | @JsonProperty("value") 68 | public Object getJsonValue() { 69 | return value; 70 | } 71 | 72 | @JsonIgnore 73 | public void setValue(V value) { 74 | this.value = value; 75 | } 76 | 77 | @JsonProperty 78 | public int getPartition() { 79 | return partition; 80 | } 81 | 82 | @JsonProperty 83 | public void setPartition(int partition) { 84 | this.partition = partition; 85 | } 86 | 87 | @JsonProperty 88 | public long getOffset() { 89 | return offset; 90 | } 91 | 92 | @JsonProperty 93 | public void setOffset(int offset) { 94 | this.offset = offset; 95 | } 96 | 97 | @Override 98 | public boolean equals(Object o) { 99 | if (this == o) { 100 | return true; 101 | } 102 | if (o == null || getClass() != o.getClass()) { 103 | return false; 104 | } 105 | 106 | ConsumerRecord that = (ConsumerRecord) o; 107 | 108 | if (offset != that.offset) { 109 | return false; 110 | } 111 | if (partition != that.partition) { 112 | return false; 113 | } 114 | if (key != null ? !key.equals(that.key) : that.key != null) { 115 | return false; 116 | } 117 | if (value != null ? !value.equals(that.value) : that.value != null) { 118 | return false; 119 | } 120 | 121 | return true; 122 | } 123 | 124 | @Override 125 | public int hashCode() { 126 | int result = key != null ? key.hashCode() : 0; 127 | result = 31 * result + (value != null ? value.hashCode() : 0); 128 | result = 31 * result + partition; 129 | result = 31 * result + (int) (offset ^ (offset >>> 32)); 130 | return result; 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /bin/kafka-rest-run-class: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Confluent Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | base_dir=$(dirname $0)/.. 18 | 19 | # Development jars. `mvn package` should collect all the required dependency jars here 20 | for dir in $base_dir/target/kafka-rest-*-development; do 21 | CLASSPATH=$CLASSPATH:$dir/share/java/kafka-rest/* 22 | done 23 | 24 | # Production jars, including kafka, rest-utils, and kafka-rest 25 | for library in "kafka" "confluent-common" "rest-utils" "kafka-rest"; do 26 | CLASSPATH=$CLASSPATH:$base_dir/share/java/$library/* 27 | done 28 | 29 | # logj4 settings 30 | if [ "x$KAFKAREST_LOG4J_OPTS" = "x" ]; then 31 | # Test for files from dev -> packages so this will work as expected in dev if you have packages 32 | # installed 33 | if [ -e "$base_dir/config/log4j.properties" ]; then # Dev environment 34 | KAFKAREST_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/config/log4j.properties" 35 | elif [ -e "$base_dir/../etc/kafka-rest/log4j.properties" ]; then # Simple zip file layout 36 | KAFKAREST_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../etc/kafka-rest/log4j.properties" 37 | elif [ -e "/etc/kafka-rest/log4j.properties" ]; then # Normal install layout 38 | KAFKAREST_LOG4J_OPTS="-Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties" 39 | fi 40 | fi 41 | 42 | # JMX settings 43 | if [ -z "$KAFKAREST_JMX_OPTS" ]; then 44 | KAFKAREST_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " 45 | fi 46 | 47 | # JMX port to use 48 | if [ $JMX_PORT ]; then 49 | KAFKAREST_JMX_OPTS="$KAFKAREST_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT " 50 | fi 51 | 52 | # Generic jvm settings you want to add 53 | if [ -z "$KAFKAREST_OPTS" ]; then 54 | KAFKAREST_OPTS="" 55 | fi 56 | 57 | # Which java to use 58 | if [ -z "$JAVA_HOME" ]; then 59 | JAVA="java" 60 | else 61 | JAVA="$JAVA_HOME/bin/java" 62 | fi 63 | 64 | # Memory options 65 | if [ -z "$KAFKAREST_HEAP_OPTS" ]; then 66 | KAFKAREST_HEAP_OPTS="-Xmx256M" 67 | fi 68 | 69 | # JVM performance options 70 | if [ -z "$KAFKAREST_JVM_PERFORMANCE_OPTS" ]; then 71 | KAFKAREST_JVM_PERFORMANCE_OPTS="-server -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -XX:+DisableExplicitGC -Djava.awt.headless=true" 72 | fi 73 | 74 | 75 | while [ $# -gt 0 ]; do 76 | COMMAND=$1 77 | case $COMMAND in 78 | -help) 79 | HELP="true" 80 | shift 81 | ;; 82 | -daemon) 83 | DAEMON_MODE="true" 84 | shift 85 | ;; 86 | *) 87 | break 88 | ;; 89 | esac 90 | done 91 | 92 | if [ "x$$HELP" = "xtrue" ]; then 93 | echo "USAGE: $0 [-daemon] [opts] [-help]" 94 | exit 0 95 | fi 96 | 97 | # Launch mode 98 | if [ "x$DAEMON_MODE" = "xtrue" ]; then 99 | nohup $JAVA $KAFKAREST_HEAP_OPTS $KAFKAREST_JVM_PERFORMANCE_OPTS $KAFKAREST_JMX_OPTS $KAFKAREST_LOG4J_OPTS -cp $CLASSPATH $KAFKAREST_OPTS "$@" 2>&1 < /dev/null & 100 | else 101 | exec $JAVA $KAFKAREST_HEAP_OPTS $KAFKAREST_JVM_PERFORMANCE_OPTS $KAFKAREST_JMX_OPTS $KAFKAREST_LOG4J_OPTS -cp $CLASSPATH $KAFKAREST_OPTS "$@" 102 | fi 103 | -------------------------------------------------------------------------------- /src/test/java/io/confluent/kafkarest/unit/RootResourceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.unit; 17 | 18 | import org.junit.Test; 19 | 20 | import java.util.Map; 21 | 22 | import javax.ws.rs.client.Entity; 23 | import javax.ws.rs.core.GenericType; 24 | import javax.ws.rs.core.Response; 25 | 26 | import io.confluent.kafkarest.Context; 27 | import io.confluent.kafkarest.KafkaRestApplication; 28 | import io.confluent.kafkarest.KafkaRestConfig; 29 | import io.confluent.kafkarest.TestUtils; 30 | import io.confluent.kafkarest.Versions; 31 | import io.confluent.kafkarest.resources.RootResource; 32 | import io.confluent.rest.EmbeddedServerTestHarness; 33 | import io.confluent.rest.RestConfigException; 34 | 35 | import static io.confluent.kafkarest.TestUtils.assertErrorResponse; 36 | import static io.confluent.kafkarest.TestUtils.assertOKResponse; 37 | import static org.junit.Assert.assertEquals; 38 | import static org.junit.Assert.assertNull; 39 | 40 | public class RootResourceTest 41 | extends EmbeddedServerTestHarness { 42 | 43 | private Context ctx; 44 | 45 | public RootResourceTest() throws RestConfigException { 46 | ctx = new Context(config, null, null, null); 47 | addResource(RootResource.class); 48 | } 49 | 50 | @Test 51 | public void testRootResource() { 52 | for (TestUtils.RequestMediaType mediatype : TestUtils.V1_ACCEPT_MEDIATYPES) { 53 | Response response = request("/", mediatype.header).get(); 54 | assertOKResponse(response, mediatype.expected); 55 | Map decoded = response.readEntity(new GenericType>() { 56 | }); 57 | } 58 | } 59 | 60 | @Test 61 | public void testInvalidAcceptMediatype() { 62 | for (String mediatype : TestUtils.V1_INVALID_MEDIATYPES) { 63 | Response response = request("/", mediatype).get(); 64 | // We would like to check for a normal error response, but Jersey/JAX-RS and possibly the 65 | // underlying servlet API spec specify that 406 Not Acceptable responses will not attach 66 | // entities. See https://java.net/jira/browse/JAX_RS_SPEC-363 for the corresponding JAX-RS 67 | // bug. We verify the little bit we can (status code) here. 68 | assertEquals(Response.Status.NOT_ACCEPTABLE.getStatusCode(), response.getStatus()); 69 | // These verify that we're seeing the *expected* but *incorrect* behavior. 70 | assertNull(response.getMediaType()); 71 | } 72 | } 73 | 74 | @Test 75 | public void testInvalidEntityContentType() { 76 | Response.Status UNSUPPORTED_MEDIA_TYPE = Response.Status.UNSUPPORTED_MEDIA_TYPE; 77 | for (String mediatype : TestUtils.V1_INVALID_REQUEST_MEDIATYPES) { 78 | Response 79 | response = 80 | request("/", Versions.KAFKA_MOST_SPECIFIC_DEFAULT + ", " + Versions.GENERIC_REQUEST) 81 | .post(Entity.entity("", mediatype)); 82 | assertErrorResponse( 83 | UNSUPPORTED_MEDIA_TYPE, response, 84 | UNSUPPORTED_MEDIA_TYPE.getStatusCode(), 85 | "HTTP " + UNSUPPORTED_MEDIA_TYPE.getStatusCode() + " " + UNSUPPORTED_MEDIA_TYPE 86 | .getReasonPhrase(), 87 | Versions.KAFKA_MOST_SPECIFIC_DEFAULT 88 | ); 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/PartitionOffset.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonCreator; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import javax.validation.constraints.Min; 22 | 23 | public class PartitionOffset { 24 | 25 | @Min(0) 26 | private Integer partition; 27 | @Min(0) 28 | private Long offset; 29 | 30 | private Integer errorCode; 31 | private String error; 32 | 33 | @JsonCreator 34 | public PartitionOffset(@JsonProperty("partition") Integer partition, 35 | @JsonProperty("offset") Long offset, 36 | @JsonProperty("error_code") Integer errorCode, 37 | @JsonProperty("error") String error) { 38 | this.partition = partition; 39 | this.offset = offset; 40 | this.errorCode = errorCode; 41 | this.error = error; 42 | } 43 | 44 | @JsonProperty 45 | public Integer getPartition() { 46 | return partition; 47 | } 48 | 49 | public void setPartition(Integer partition) { 50 | this.partition = partition; 51 | } 52 | 53 | @JsonProperty 54 | public Long getOffset() { 55 | return offset; 56 | } 57 | 58 | public void setOffset(Long offset) { 59 | this.offset = offset; 60 | } 61 | 62 | @JsonProperty("error_code") 63 | public Integer getErrorCode() { 64 | return errorCode; 65 | } 66 | 67 | public void setErrorCode(Integer errorCode) { 68 | this.errorCode = errorCode; 69 | } 70 | 71 | @JsonProperty("error") 72 | public String getError() { 73 | return error; 74 | } 75 | 76 | public void setError(String error) { 77 | this.error = error; 78 | } 79 | 80 | @Override 81 | public String toString() { 82 | return "PartitionOffset{" + 83 | "partition=" + partition + 84 | ", offset=" + offset + 85 | ", errorCode=" + errorCode + 86 | ", error='" + error + '\'' + 87 | '}'; 88 | } 89 | 90 | @Override 91 | public boolean equals(Object o) { 92 | if (this == o) { 93 | return true; 94 | } 95 | if (o == null || getClass() != o.getClass()) { 96 | return false; 97 | } 98 | 99 | PartitionOffset that = (PartitionOffset) o; 100 | 101 | if (error != null ? !error.equals(that.error) : that.error != null) { 102 | return false; 103 | } 104 | if (errorCode != null ? !errorCode.equals(that.errorCode) : that.errorCode != null) { 105 | return false; 106 | } 107 | if (offset != null ? !offset.equals(that.offset) : that.offset != null) { 108 | return false; 109 | } 110 | if (partition != null ? !partition.equals(that.partition) : that.partition != null) { 111 | return false; 112 | } 113 | 114 | return true; 115 | } 116 | 117 | @Override 118 | public int hashCode() { 119 | int result = partition != null ? partition.hashCode() : 0; 120 | result = 31 * result + (offset != null ? offset.hashCode() : 0); 121 | result = 31 * result + (errorCode != null ? errorCode.hashCode() : 0); 122 | result = 31 * result + (error != null ? error.hashCode() : 0); 123 | return result; 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/KafkaRestApplication.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | import org.I0Itec.zkclient.ZkClient; 19 | 20 | import java.util.Properties; 21 | 22 | import javax.ws.rs.core.Configurable; 23 | 24 | import io.confluent.kafkarest.exceptions.ZkExceptionMapper; 25 | import io.confluent.kafkarest.resources.BrokersResource; 26 | import io.confluent.kafkarest.resources.ConsumersResource; 27 | import io.confluent.kafkarest.resources.PartitionsResource; 28 | import io.confluent.kafkarest.resources.RootResource; 29 | import io.confluent.kafkarest.resources.SpoolResource; 30 | import io.confluent.kafkarest.resources.TopicsResource; 31 | import io.confluent.rest.Application; 32 | import io.confluent.rest.RestConfigException; 33 | import kafka.utils.ZKStringSerializer$; 34 | 35 | /** 36 | * Utilities for configuring and running an embedded Kafka server. 37 | */ 38 | public class KafkaRestApplication extends Application { 39 | 40 | ZkClient zkClient; 41 | Context context; 42 | 43 | public KafkaRestApplication() throws RestConfigException { 44 | this(new Properties()); 45 | } 46 | 47 | public KafkaRestApplication(Properties props) throws RestConfigException { 48 | super(new KafkaRestConfig(props)); 49 | } 50 | 51 | public KafkaRestApplication(KafkaRestConfig config) { 52 | super(config); 53 | } 54 | 55 | @Override 56 | public void setupResources(Configurable config, KafkaRestConfig appConfig) { 57 | setupInjectedResources(config, appConfig, null, null, null, null); 58 | } 59 | 60 | /** 61 | * Helper that does normal setup, but uses injected components so their configs or implementations 62 | * can be customized for testing. This only exists to support TestKafkaRestApplication 63 | */ 64 | protected void setupInjectedResources(Configurable config, KafkaRestConfig appConfig, 65 | ZkClient zkClient, MetadataObserver mdObserver, 66 | ProducerPool producerPool, 67 | ConsumerManager consumerManager) { 68 | config.register(new ZkExceptionMapper(appConfig)); 69 | 70 | if (zkClient == null) { 71 | zkClient = new ZkClient(appConfig.getString(KafkaRestConfig.ZOOKEEPER_CONNECT_CONFIG), 72 | 30000, 30000, ZKStringSerializer$.MODULE$); 73 | } 74 | if (mdObserver == null) { 75 | mdObserver = new MetadataObserver(appConfig, zkClient); 76 | } 77 | if (producerPool == null) { 78 | producerPool = new ProducerPool(appConfig, zkClient); 79 | } 80 | if (consumerManager == null) { 81 | consumerManager = new ConsumerManager(appConfig, mdObserver); 82 | } 83 | 84 | this.zkClient = zkClient; 85 | context = new Context(appConfig, mdObserver, producerPool, consumerManager); 86 | config.register(RootResource.class); 87 | config.register(new BrokersResource(context)); 88 | config.register(new TopicsResource(context)); 89 | config.register(new PartitionsResource(context)); 90 | config.register(new ConsumersResource(context)); 91 | config.register(new SpoolResource(context)); 92 | } 93 | 94 | @Override 95 | public void onShutdown() { 96 | context.getConsumerManager().shutdown(); 97 | context.getProducerPool().shutdown(); 98 | context.getMetadataObserver().shutdown(); 99 | zkClient.close(); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | The following libraries are included in packaged versions of this project: 2 | 3 | * Apache Avro 4 | * COPYRIGHT: Copyright 2010 The Apache Software Foundation 5 | * LICENSE: licenses/LICENSE.apache2.txt 6 | * NOTICE: licenses/NOTICE.avro.txt 7 | * HOMEPAGE: http://avro.apache.org 8 | 9 | * Apache Commons Compress 10 | * COPYRIGHT: Copyright 2002-2015 The Apache Software Foundation 11 | * LICENSE: licenses/LICENSE.apache2.txt 12 | * NOTICE: licenses/NOTICE.commons-compress.txt 13 | * HOMEPAGE: http://commons.apache.org/proper/commons-compress/ 14 | 15 | * Apache Kafka 16 | * COPYRIGHT: Copyright 2012 The Apache Software Foundation. 17 | * LICENSE: licenses/LICENSE.apache2.txt 18 | * NOTICE: licenses/NOTICE.kafka.txt 19 | * HOMEPAGE: http://kafka.apache.org 20 | 21 | * Apache log4j 22 | * COPYRIGHT: Copyright 2010 The Apache Software Foundation 23 | * LICENSE: licenses/LICENSE.apache2.txt 24 | * NOTICE: licenses/NOTICE.log4j.txt 25 | * HOMEPAGE: http://logging.apache.org/log4j/1.2/ 26 | 27 | * Apache ZooKeeper 28 | * COPYRIGHT: Copyright 2009-2014 The Apache Software Foundation 29 | * LICENSE: licenses/LICENSE.apache2.txt 30 | * NOTICE: licenses/NOTICE.zookeeper.txt 31 | * HOMEPAGE: http://zookeeper.apache.org/ 32 | 33 | * Confluent Common 34 | * COPYRIGHT: Confluent Inc. 35 | * LICENSE: licenses/LICENSE.apache2.txt 36 | * NOTICE: licenses/NOTICE.confluent-common.txt 37 | * HOMEPAGE: https://github.com/confluentinc/common 38 | 39 | * Confluent REST Utils 40 | * COPYRIGHT: Confluent Inc. 41 | * LICENSE: licenses/LICENSE.apache2.txt 42 | * NOTICE: licenses/NOTICE.confluent-rest-utils.txt 43 | * HOMEPAGE: https://github.com/confluentinc/rest-utils 44 | 45 | * Confluent Schema Registry 46 | * COPYRIGHT: Confluent Inc. 47 | * LICENSE: licenses/LICENSE.apache2.txt 48 | * NOTICE: licenses/NOTICE.confluent-schema-registry.txt 49 | * HOMEPAGE: https://github.com/confluentinc/schema-registry 50 | 51 | * Jackson core 52 | * LICENSE: licenses/LICENSE.jackson-core.txt (Apache 2) 53 | * NOTICE: licenses/NOTICE.jackson-core.txt 54 | * HOMEPAGE: http://github.com/FasterXML/jackson 55 | 56 | * Jackson databind 57 | * LICENSE: licenses/LICENSE.jackson-databind.txt (Apache 2) 58 | * NOTICE: licenses/NOTICE.jackson-databind.txt 59 | * HOMEPAGE: http://github.com/FasterXML/jackson 60 | 61 | * jline 62 | * COPYRIGHT: Copyright (c) 2002-2006, Marc Prud'hommeaux 63 | * LICENSE: licenses/LICENSE.bsd.txt 64 | * HOMEPAGE: http://jline.sourceforge.net/ 65 | 66 | * JOpt Simple 67 | * COPYRIGHT: Copyright (c) 2004-2014 Paul R. Holser, Jr. 68 | * LICENSE: licenses/LICENSE.mit.txt 69 | * HOMEPAGE: https://pholser.github.io/jopt-simple/ 70 | 71 | * LZ4 Java 72 | * LICENSE: licenses/LICENSE.apache2.txt 73 | * NOTICE: licenses/NOTICE.lz4-java.txt 74 | * HOMEPAGE: https://github.com/jpountz/lz4-java 75 | 76 | * Metrics 77 | * COPYRIGHT: Copyright (c) 2010-2012 Coda Hale, Yammer.com 78 | * LICENSE: licenses/LICENSE.apache2.txt 79 | * NOTICE: licenses/NOTICE.metrics.txt 80 | * HOMEPAGE: http://metrics.dropwizard.io/ 81 | 82 | * Netty 83 | * COPYRIGHT: Copyright 2014 The Netty Project 84 | * LICENSE: licenses/LICENSE.apache2.txt 85 | * NOTICE: licenses/NOTICE.netty.txt 86 | * HOMEPAGE: http://netty.io/ 87 | 88 | * paranamer 89 | * COPYRIGHT: Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc 90 | * LICENSE: licenses/LICENSE.bsd.txt 91 | * HOMEPAGE: http://xircles.codehaus.org/projects/paranamer 92 | 93 | * Scala 94 | * COPYRIGHT: Copyright (c) 2002-2015 EPFL, Copyright (c) 2011-2015 Typesafe, Inc. 95 | * LICENSE: licenses/LICENSE.bsd.txt 96 | * HOMEPAGE: http://www.scala-lang.org 97 | 98 | * SLF4J 99 | * COPYRIGHT: Copyright (c) 2004-2013 QOS.ch 100 | * LICENSE: licenses/LICENSE.mit.txt 101 | * HOMEPAGE: http://www.slf4j.org/ 102 | 103 | * snappy-java 104 | * LICENSE: licenses/LICENSE.apache2.txt 105 | * NOTICE: licenses/NOTICE.snappy-java.txt 106 | * HOMEPAGE: https://github.com/xerial/snappy-java 107 | 108 | * XZ for Java 109 | * COPYRIGHT: Original Author: Lasse Collin 110 | * LICENSE: Public Domain 111 | * HOMEPAGE: http://tukaani.org/xz/java.html 112 | 113 | * ZkClient 114 | * LICENSE: licenses/LICENSE.apache2.txt 115 | * HOMEPAGE: https://github.com/sgroschupf/zkclient 116 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/ConsumerWorker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.util.Comparator; 22 | import java.util.LinkedList; 23 | import java.util.PriorityQueue; 24 | import java.util.Queue; 25 | import java.util.concurrent.CountDownLatch; 26 | import java.util.concurrent.Future; 27 | import java.util.concurrent.atomic.AtomicBoolean; 28 | 29 | /** 30 | * Worker thread for consumers that multiplexes multiple consumer operations onto a single thread. 31 | */ 32 | public class ConsumerWorker extends Thread { 33 | 34 | private static final Logger log = LoggerFactory.getLogger(ConsumerWorker.class); 35 | 36 | KafkaRestConfig config; 37 | 38 | AtomicBoolean isRunning = new AtomicBoolean(true); 39 | CountDownLatch shutdownLatch = new CountDownLatch(1); 40 | 41 | Queue tasks = new LinkedList(); 42 | Queue waitingTasks = 43 | new PriorityQueue(1, new ReadTaskExpirationComparator()); 44 | 45 | public ConsumerWorker(KafkaRestConfig config) { 46 | this.config = config; 47 | } 48 | 49 | public synchronized 50 | Future readTopic(ConsumerState state, String topic, long maxBytes, 51 | ConsumerWorkerReadCallback callback) { 52 | log.trace("Consumer worker " + this.toString() + " reading topic " + topic 53 | + " for " + state.getId()); 54 | ConsumerReadTask task 55 | = new ConsumerReadTask(state, topic, maxBytes, callback); 56 | if (!task.isDone()) { 57 | tasks.add(task); 58 | this.notifyAll(); 59 | } 60 | return task; 61 | } 62 | 63 | @Override 64 | public void run() { 65 | while (isRunning.get()) { 66 | ConsumerReadTask task = null; 67 | synchronized (this) { 68 | if (tasks.isEmpty()) { 69 | try { 70 | long now = config.getTime().milliseconds(); 71 | long nextExpiration = nextWaitingExpiration(); 72 | if (nextExpiration > now) { 73 | long timeout = (nextExpiration == Long.MAX_VALUE ? 74 | 0 : nextExpiration - now); 75 | assert (timeout >= 0); 76 | config.getTime().waitOn(this, timeout); 77 | } 78 | } catch (InterruptedException e) { 79 | // Indication of shutdown 80 | } 81 | } 82 | 83 | long now = config.getTime().milliseconds(); 84 | while (nextWaitingExpiration() <= now) { 85 | tasks.add(waitingTasks.remove()); 86 | } 87 | 88 | task = tasks.poll(); 89 | if (task != null) { 90 | boolean backoff = task.doPartialRead(); 91 | if (!task.isDone()) { 92 | if (backoff) { 93 | waitingTasks.add(task); 94 | } else { 95 | tasks.add(task); 96 | } 97 | } 98 | } 99 | } 100 | } 101 | shutdownLatch.countDown(); 102 | } 103 | 104 | private long nextWaitingExpiration() { 105 | if (waitingTasks.isEmpty()) { 106 | return Long.MAX_VALUE; 107 | } else { 108 | return waitingTasks.peek().waitExpiration; 109 | } 110 | } 111 | 112 | public void shutdown() { 113 | try { 114 | isRunning.set(false); 115 | this.interrupt(); 116 | shutdownLatch.await(); 117 | } catch (InterruptedException e) { 118 | log.error("Interrupted while " 119 | + "consumer worker thread."); 120 | throw new Error("Interrupted when shutting down consumer worker thread."); 121 | } 122 | } 123 | 124 | private static class ReadTaskExpirationComparator implements Comparator { 125 | 126 | @Override 127 | public int compare(ConsumerReadTask t1, ConsumerReadTask t2) { 128 | if (t1.waitExpiration == t2.waitExpiration) { 129 | return 0; 130 | } else if (t1.waitExpiration < t2.waitExpiration) { 131 | return -1; 132 | } else { 133 | return 1; 134 | } 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/entities/ConsumerInstanceConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest.entities; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import javax.validation.constraints.NotNull; 22 | 23 | import io.confluent.rest.exceptions.RestConstraintViolationException; 24 | 25 | public class ConsumerInstanceConfig { 26 | 27 | private static final EmbeddedFormat DEFAULT_FORMAT = EmbeddedFormat.BINARY; 28 | 29 | private String id; 30 | @NotNull 31 | private EmbeddedFormat format; 32 | private String autoOffsetReset; 33 | private String autoCommitEnable; 34 | 35 | public ConsumerInstanceConfig() { 36 | this(DEFAULT_FORMAT); 37 | } 38 | 39 | public ConsumerInstanceConfig(EmbeddedFormat format) { 40 | // This constructor is only for tests so reparsing the format name is ok 41 | this(null, format.name(), null, null); 42 | } 43 | 44 | public ConsumerInstanceConfig(@JsonProperty("id") String id, 45 | @JsonProperty("format") String format, 46 | @JsonProperty("auto.offset.reset") String autoOffsetReset, 47 | @JsonProperty("auto.commit.enable") String autoCommitEnable) { 48 | this.id = id; 49 | if (format == null) { 50 | this.format = DEFAULT_FORMAT; 51 | } else { 52 | String formatCanonical = format.toUpperCase(); 53 | for (EmbeddedFormat f : EmbeddedFormat.values()) { 54 | if (f.name().equals(formatCanonical)) { 55 | this.format = f; 56 | break; 57 | } 58 | } 59 | if (this.format == null) { 60 | throw new RestConstraintViolationException( 61 | "Invalid format type.", 62 | RestConstraintViolationException.DEFAULT_ERROR_CODE); 63 | } 64 | } 65 | this.autoOffsetReset = autoOffsetReset; 66 | this.autoCommitEnable = autoCommitEnable; 67 | } 68 | 69 | @JsonProperty 70 | public String getId() { 71 | return id; 72 | } 73 | 74 | @JsonProperty 75 | public void setId(String id) { 76 | this.id = id; 77 | } 78 | 79 | @JsonIgnore 80 | public EmbeddedFormat getFormat() { 81 | return format; 82 | } 83 | 84 | @JsonProperty("format") 85 | public String getFormatJson() { 86 | return format.name().toLowerCase(); 87 | } 88 | 89 | @JsonProperty 90 | public void setFormat(EmbeddedFormat format) { 91 | this.format = format; 92 | } 93 | 94 | @JsonProperty("auto.offset.reset") 95 | public String getAutoOffsetReset() { 96 | return autoOffsetReset; 97 | } 98 | 99 | @JsonProperty("auto.offset.reset") 100 | public void setAutoOffsetReset(String autoOffsetReset) { 101 | this.autoOffsetReset = autoOffsetReset; 102 | } 103 | 104 | @JsonProperty("auto.commit.enable") 105 | public String getAutoCommitEnable() { 106 | return autoCommitEnable; 107 | } 108 | 109 | @JsonProperty("auto.commit.enable") 110 | public void setAutoCommitEnable(String autoCommitEnable) { 111 | this.autoCommitEnable = autoCommitEnable; 112 | } 113 | 114 | @Override 115 | public boolean equals(Object o) { 116 | if (this == o) { 117 | return true; 118 | } 119 | if (o == null || getClass() != o.getClass()) { 120 | return false; 121 | } 122 | 123 | ConsumerInstanceConfig that = (ConsumerInstanceConfig) o; 124 | 125 | if (autoCommitEnable != null ? !autoCommitEnable.equals(that.autoCommitEnable) 126 | : that.autoCommitEnable != null) { 127 | return false; 128 | } 129 | if (autoOffsetReset != null ? !autoOffsetReset.equals(that.autoOffsetReset) 130 | : that.autoOffsetReset != null) { 131 | return false; 132 | } 133 | if (format != that.format) { 134 | return false; 135 | } 136 | if (id != null ? !id.equals(that.id) : that.id != null) { 137 | return false; 138 | } 139 | 140 | return true; 141 | } 142 | 143 | @Override 144 | public int hashCode() { 145 | int result = id != null ? id.hashCode() : 0; 146 | result = 31 * result + (format != null ? format.hashCode() : 0); 147 | result = 31 * result + (autoOffsetReset != null ? autoOffsetReset.hashCode() : 0); 148 | result = 31 * result + (autoCommitEnable != null ? autoCommitEnable.hashCode() : 0); 149 | return result; 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /docs/monitoring.rst: -------------------------------------------------------------------------------- 1 | .. _kafkarest_monitoring: 2 | 3 | Monitoring 4 | ---------- 5 | 6 | The REST proxy reports a variety of metrics through JMX. It can also be configured to report 7 | stats using additional pluggable stats reporters using the ``metrics.reporters`` configuration 8 | option. 9 | 10 | The easiest way to view the available metrics is to use jconsole to browse JMX MBeans. In 11 | addition to the metrics specific to the REST proxy listed below, you can also view and monitor 12 | the metrics for the underlying producers and consumers. 13 | 14 | The REST proxy has two types of metrics. Global metrics help you monitor the overall health of 15 | the service. Per-endpoint metrics monitor each API endpoint request method and are 16 | prefixed by a name of the endpoint (e.g. ``brokers.list``). These help you 17 | understand how the proxy is being used and track down specific performance problems. 18 | 19 | Global Metrics 20 | ~~~~~~~~~~~~~~ 21 | 22 | **MBean: kafka.rest:type=jetty-metrics** 23 | 24 | ``connections-active`` 25 | Total number of active TCP connections. 26 | 27 | ``connections-accepted-rate`` 28 | The average rate per second of accepted TCP connections. 29 | 30 | ``connections-opened-rate`` 31 | The average rate per second of opened TCP connections. 32 | 33 | ``connections-closed-rate`` 34 | The average rate per second of closed TCP connections. 35 | 36 | 37 | Per-Endpoint Metrics 38 | ~~~~~~~~~~~~~~~~~~~~ 39 | 40 | The following are the metrics available for each endpoint request method. Metrics for all 41 | requests are also aggregated into a global instance for each one. These aggregate instances have 42 | no prefix in their name. 43 | 44 | **MBean: kafka.rest:type=jersey-metrics** 45 | 46 | ``.request-byte-rate`` 47 | Bytes/second of incoming requests 48 | 49 | ``.request-error-rate`` 50 | The average number of requests per second that resulted in HTTP error responses 51 | 52 | ``.request-latency-avg`` 53 | The average request latency in ms 54 | 55 | ``.request-latency-max`` 56 | The maximum request latency in ms 57 | 58 | ``.request-rate`` 59 | The average number of HTTP requests per second. 60 | 61 | ``.request-size-avg`` 62 | The average request size in bytes 63 | 64 | ``.request-size-max`` 65 | The maximum request size in bytes 66 | 67 | ``.response-byte-rate`` 68 | Bytes/second of outgoing responses 69 | 70 | ``.response-rate`` 71 | The average number of HTTP responses per second. 72 | 73 | ``.response-size-avg`` 74 | The average response size in bytes 75 | 76 | ``.response-size-max`` 77 | The maximum response size in bytes 78 | 79 | 80 | Endpoints 81 | ~~~~~~~~~ 82 | 83 | The following is a list of all the API endpoint methods. The naming should map intuitively to 84 | each of the API operations. To create a full metric name, prefix a per-endpoint metric name with 85 | one of these values. For example, to find the rate of ``GET /brokers`` API calls, combine the 86 | endpoint name ``brokers.list`` with the metric name ``request-rate`` to get 87 | ``brokers.list.request-rate``. 88 | 89 | ============================== =================================================================== 90 | ``brokers.list`` ``GET /brokers`` 91 | ``consumer.commit`` ``POST /consumers/{group}/instances/{instance}/offsets`` 92 | ``consumer.create`` ``POST /consumers/{group}`` 93 | ``consumer.delete`` ``DELETE /consumers/{group}/instances/{instance}`` 94 | ``consumer.topic.read-avro`` ``GET /consumers/{group}/instances/{instance}/topics/{topic}`` 95 | with ``Accept: application/vnd.kafka.avro.v1+json`` header 96 | ``consumer.topic.read-binary`` ``GET /consumers/{group}/instances/{instance}/topics/{topic}`` 97 | with ``Accept: application/vnd.kafka.binary.v1+json`` header 98 | ``partition.get`` ``GET /topics/{topic}/partitions/{partition}`` 99 | ``partition.produce-avro`` ``POST /topics/{topic}/partitions/{partition}`` with 100 | ``Content-Type: application/vnd.kafka.avro.v1+json`` header 101 | ``partition.produce-binary`` ``POST /topics/{topic}/partitions/{partition}`` with 102 | ``Content-Type: application/vnd.kafka.binary.v1+json`` header 103 | ``partitions.list`` ``GET /topics/{topic}/partitions`` 104 | ``topic.get`` ``GET /topics/{topic}`` 105 | ``topic.produce-avro`` ``POST /topics/{topic}`` with 106 | ``Content-Type: application/vnd.kafka.avro.v1+json`` header 107 | ``topic.produce-binary`` ``POST /topics/{topic}`` with 108 | ``Content-Type: application/vnd.kafka.binary.v1+json`` header 109 | ``topics.list`` ``GET /topics`` 110 | ============================== =================================================================== 111 | 112 | 113 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/AvroRestProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | 17 | package io.confluent.kafkarest; 18 | 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | 21 | import org.apache.avro.Schema; 22 | import org.apache.kafka.clients.producer.ProducerRecord; 23 | 24 | import java.io.IOException; 25 | import java.util.ArrayList; 26 | import java.util.Collection; 27 | 28 | import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; 29 | import io.confluent.kafka.serializers.KafkaAvroSerializer; 30 | import io.confluent.kafkarest.SpoolProducer; 31 | import io.confluent.kafkarest.converters.AvroConverter; 32 | import io.confluent.kafkarest.converters.ConversionException; 33 | import io.confluent.kafkarest.entities.ProduceRecord; 34 | import io.confluent.kafkarest.entities.SchemaHolder; 35 | import io.confluent.kafkarest.entities.SpoolMode; 36 | import io.confluent.rest.exceptions.RestException; 37 | 38 | public class AvroRestProducer implements RestProducer { 39 | 40 | protected final SpoolProducer producer; 41 | protected final KafkaAvroSerializer keySerializer; 42 | protected final KafkaAvroSerializer valueSerializer; 43 | 44 | public AvroRestProducer(SpoolProducer producer, 45 | KafkaAvroSerializer keySerializer, 46 | KafkaAvroSerializer valueSerializer) { 47 | this.producer = producer; 48 | this.keySerializer = keySerializer; 49 | this.valueSerializer = valueSerializer; 50 | } 51 | 52 | public void produce(ProduceTask task, String topic, Integer partition, SpoolMode spoolMode, 53 | Collection> records) { 54 | SchemaHolder schemaHolder = task.getSchemaHolder(); 55 | Schema keySchema = null, valueSchema = null; 56 | Integer keySchemaId = schemaHolder.getKeySchemaId(); 57 | Integer valueSchemaId = schemaHolder.getValueSchemaId(); 58 | try { 59 | // If both ID and schema are null, that may be ok. Validation of the ProduceTask by the 60 | // caller should have checked this already. 61 | if (keySchemaId != null) { 62 | keySchema = keySerializer.getByID(keySchemaId); 63 | } else if (schemaHolder.getKeySchema() != null) { 64 | keySchema = new Schema.Parser().parse(schemaHolder.getKeySchema()); 65 | keySchemaId = keySerializer.register(topic + "-key", keySchema); 66 | } 67 | 68 | if (valueSchemaId != null) { 69 | valueSchema = valueSerializer.getByID(valueSchemaId); 70 | } else if (schemaHolder.getValueSchema() != null) { 71 | valueSchema = new Schema.Parser().parse(schemaHolder.getValueSchema()); 72 | valueSchemaId = valueSerializer.register(topic + "-value", valueSchema); 73 | } 74 | } catch (RestClientException e) { 75 | // FIXME We should return more specific error codes (unavailable vs registration failed in 76 | // a way that isn't retriable?). 77 | throw new RestException("Schema registration or lookup failed", 408, 40801, e); 78 | } catch (IOException e) { 79 | throw new RestException("Schema registration or lookup failed", 408, 40801, e); 80 | } 81 | 82 | // Store the schema IDs in the task. These will be used to include the IDs in the response 83 | task.setSchemaIds(keySchemaId, valueSchemaId); 84 | 85 | // Convert everything to Avro before doing any sends so if any conversion fails we can kill 86 | // the entire request so we don't get partially sent requests 87 | ArrayList> kafkaRecords 88 | = new ArrayList>(); 89 | try { 90 | for (ProduceRecord record : records) { 91 | // Beware of null schemas and NullNodes here: we need to avoid attempting the conversion 92 | // if there isn't a schema. Validation will have already checked that all the keys/values 93 | // were NullNodes. 94 | Object key = (keySchema != null ? AvroConverter.toAvro(record.getKey(), keySchema) : null); 95 | Object value = (valueSchema != null 96 | ? AvroConverter.toAvro(record.getValue(), valueSchema) : null); 97 | kafkaRecords.add(new ProducerRecord(topic, partition, key, value)); 98 | } 99 | } catch (ConversionException e) { 100 | throw Errors.jsonAvroConversionException(); 101 | } 102 | for (ProducerRecord rec : kafkaRecords) { 103 | producer.send(spoolMode, rec, task.createCallback()); 104 | } 105 | } 106 | 107 | public void close() { 108 | producer.close(); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/io/confluent/kafkarest/SpoolRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Confluent Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | **/ 16 | package io.confluent.kafkarest; 17 | 18 | import org.apache.flume.ChannelException; 19 | import org.apache.flume.Event; 20 | import org.apache.flume.channel.file.FileChannel; 21 | import org.apache.flume.event.EventBuilder; 22 | import org.apache.kafka.clients.producer.ProducerRecord; 23 | 24 | import java.nio.ByteBuffer; 25 | 26 | // Helper class for dealing with the take and put operations in the specified 27 | // channel. 28 | class SpoolRecord { 29 | 30 | public final Integer attempt; 31 | public final Long timestamp; 32 | public final ProducerRecord payload; 33 | 34 | private final Event attemptEvent; 35 | private final Event timestampEvent; 36 | private final Event topicEvent; 37 | private final Event partitionEvent; 38 | private final Event keyEvent; 39 | private final Event valueEvent; 40 | 41 | // Helper method to take a record out of the specified channel. 42 | public static SpoolRecord take(FileChannel channel) { 43 | try { 44 | Event attemptEvent = channel.take(); 45 | Event timestampEvent = channel.take(); 46 | Event topicEvent = channel.take(); 47 | Event partitionEvent = channel.take(); 48 | Event keyEvent = channel.take(); 49 | Event valueEvent = channel.take(); 50 | if (attemptEvent != null && 51 | timestampEvent != null && 52 | topicEvent != null && 53 | partitionEvent != null && 54 | keyEvent != null && 55 | valueEvent != null) { 56 | return new SpoolRecord(attemptEvent, timestampEvent, topicEvent, 57 | partitionEvent, keyEvent, valueEvent); 58 | } 59 | } catch (ChannelException e) { 60 | // NOTE: What can be done at this point to mitigate the corrupted event? 61 | // In the case FileChannelConfiguration.FSYNC_PER_TXN is true, then 62 | // ChannelException will fire upon any corrupted event. What course of 63 | // action can be taken at this point? We force this to be false, hence 64 | // FileBackedTransaction.doTake will automatically skip over and find the 65 | // next available event without throwing this exception. Keeping this 66 | // catch-statement so compiler will not complain. 67 | } 68 | return null; 69 | } 70 | 71 | // Helper to construct a new spool record while bumping the attempt count. 72 | private SpoolRecord(Event attemptEvent, Event timestampEvent, Event topicEvent, 73 | Event partitionEvent, Event keyEvent, Event valueEvent) { 74 | this.attempt = ByteBuffer.wrap(attemptEvent.getBody()).getInt() + 1; 75 | this.timestamp = ByteBuffer.wrap(timestampEvent.getBody()).getLong(); 76 | byte[] partition = partitionEvent.getBody(); 77 | this.payload = new ProducerRecord(new String(topicEvent.getBody()), 78 | (partition.length) == 0 ? null : ByteBuffer.wrap(partition).getInt(), 79 | keyEvent.getBody(), valueEvent.getBody()); 80 | this.attemptEvent = EventBuilder.withBody(ByteBuffer.allocate(4).putInt(attempt).array()); 81 | this.timestampEvent = timestampEvent; 82 | this.topicEvent = topicEvent; 83 | this.partitionEvent = partitionEvent; 84 | this.keyEvent = keyEvent; 85 | this.valueEvent = valueEvent; 86 | } 87 | 88 | // Construct a new spool record. 89 | public SpoolRecord(long timestamp, String topic, Integer partition, byte[] key, byte[] value) { 90 | this.attempt = 1; 91 | this.timestamp = timestamp; 92 | this.payload = new ProducerRecord(topic, partition, key, value); 93 | byte[] serializedAttempt = ByteBuffer.allocate(4).putInt(attempt).array(); 94 | byte[] serializedTimestamp = ByteBuffer.allocate(8).putLong(timestamp).array(); 95 | byte[] serializedTopic = topic.getBytes(); 96 | byte[] serializedPartition = (partition == null) ? new byte[0] : 97 | ByteBuffer.allocate(4).putInt(partition.intValue()).array(); 98 | this.attemptEvent = EventBuilder.withBody(serializedAttempt); 99 | this.timestampEvent = EventBuilder.withBody(serializedTimestamp); 100 | this.topicEvent = EventBuilder.withBody(serializedTopic); 101 | this.partitionEvent = EventBuilder.withBody(serializedPartition); 102 | this.keyEvent = EventBuilder.withBody(key); 103 | this.valueEvent = EventBuilder.withBody(value); 104 | } 105 | 106 | // Helper method to put a record into the specified channel. 107 | public void put(FileChannel channel) { 108 | channel.put(attemptEvent); 109 | channel.put(timestampEvent); 110 | channel.put(topicEvent); 111 | channel.put(partitionEvent); 112 | channel.put(keyEvent); 113 | channel.put(valueEvent); 114 | } 115 | } 116 | --------------------------------------------------------------------------------