├── .gitignore ├── .travis.yml ├── LICENSE ├── NOTICE ├── README.md ├── conditionalPublish.sh ├── docker ├── docker-compose.yml └── elasticsearch │ ├── Dockerfile │ └── data │ └── config │ └── elasticsearch.yml ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── img ├── IndexerV2Design.jpg ├── IndexerV2Design.pptx └── build-dir.png ├── settings.gradle └── src ├── main ├── java │ └── org │ │ └── elasticsearch │ │ └── kafka │ │ └── indexer │ │ ├── CommonKafkaUtils.java │ │ ├── FailedEventsLogger.java │ │ ├── KafkaESIndexerProcess.java │ │ ├── exception │ │ ├── ConsumerNonRecoverableException.java │ │ ├── ConsumerRecoverableException.java │ │ ├── IndexerESNotRecoverableException.java │ │ └── IndexerESRecoverableException.java │ │ ├── jobs │ │ ├── ConsumerManager.java │ │ ├── ConsumerThreadPool.java │ │ ├── ConsumerWorker.java │ │ ├── IConsumerWorker.java │ │ ├── OffsetLoggingCallbackImpl.java │ │ ├── StartOption.java │ │ └── StartOptionParser.java │ │ └── service │ │ ├── ElasticSearchBatchService.java │ │ ├── ElasticSearchClientService.java │ │ ├── IBatchMessageProcessor.java │ │ └── impl │ │ └── examples │ │ └── ESBatchMessageProcessorImpl.java └── resources │ ├── config │ ├── kafka-es-indexer-custom-start-options.properties │ ├── kafka-es-indexer.properties │ └── logback.xml │ └── spring │ ├── common.xml │ └── kafka-es-context-public.xml └── test ├── java └── org │ └── elasticsearch │ └── kafka │ └── indexer │ ├── CommonKafkaUtilsTest.java │ ├── jobs │ ├── ConsumerManagerTest.java │ ├── ConsumerWorkerTest.java │ └── StartOptionParserTest.java │ └── service │ └── ElasticSearchBatchServiceTest.java ├── org └── elasticsearch │ └── kafka │ └── indexer │ └── jmx │ └── KafkaEsIndexerStatusTest.java └── resources ├── kafka-es-indexer-docker.properties ├── test-start-options-custom-5-partitions.properties ├── test-start-options-custom-empty.properties ├── test-start-options-custom-malformed.properties └── test-start-options-custom.properties /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/LICENSE -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/NOTICE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/README.md -------------------------------------------------------------------------------- /conditionalPublish.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/conditionalPublish.sh -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/docker/docker-compose.yml -------------------------------------------------------------------------------- /docker/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/docker/elasticsearch/Dockerfile -------------------------------------------------------------------------------- /docker/elasticsearch/data/config/elasticsearch.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/docker/elasticsearch/data/config/elasticsearch.yml -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/gradle.properties -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/gradle/wrapper/gradle-wrapper.properties -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/gradlew -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/gradlew.bat -------------------------------------------------------------------------------- /img/IndexerV2Design.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/img/IndexerV2Design.jpg -------------------------------------------------------------------------------- /img/IndexerV2Design.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/img/IndexerV2Design.pptx -------------------------------------------------------------------------------- /img/build-dir.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/img/build-dir.png -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-elasticsearch-consumer' 2 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/CommonKafkaUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/CommonKafkaUtils.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/FailedEventsLogger.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/FailedEventsLogger.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/KafkaESIndexerProcess.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/KafkaESIndexerProcess.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/exception/ConsumerNonRecoverableException.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/exception/ConsumerNonRecoverableException.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/exception/ConsumerRecoverableException.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/exception/ConsumerRecoverableException.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/exception/IndexerESNotRecoverableException.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/exception/IndexerESNotRecoverableException.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/exception/IndexerESRecoverableException.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/exception/IndexerESRecoverableException.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/jobs/ConsumerManager.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/jobs/ConsumerManager.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/jobs/ConsumerThreadPool.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/jobs/ConsumerThreadPool.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/jobs/ConsumerWorker.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/jobs/ConsumerWorker.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/jobs/IConsumerWorker.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/jobs/IConsumerWorker.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/jobs/OffsetLoggingCallbackImpl.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/jobs/OffsetLoggingCallbackImpl.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/jobs/StartOption.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/jobs/StartOption.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/jobs/StartOptionParser.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/jobs/StartOptionParser.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/service/ElasticSearchBatchService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/service/ElasticSearchBatchService.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/service/ElasticSearchClientService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/service/ElasticSearchClientService.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/service/IBatchMessageProcessor.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/service/IBatchMessageProcessor.java -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/kafka/indexer/service/impl/examples/ESBatchMessageProcessorImpl.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/java/org/elasticsearch/kafka/indexer/service/impl/examples/ESBatchMessageProcessorImpl.java -------------------------------------------------------------------------------- /src/main/resources/config/kafka-es-indexer-custom-start-options.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/resources/config/kafka-es-indexer-custom-start-options.properties -------------------------------------------------------------------------------- /src/main/resources/config/kafka-es-indexer.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/resources/config/kafka-es-indexer.properties -------------------------------------------------------------------------------- /src/main/resources/config/logback.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/resources/config/logback.xml -------------------------------------------------------------------------------- /src/main/resources/spring/common.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/resources/spring/common.xml -------------------------------------------------------------------------------- /src/main/resources/spring/kafka-es-context-public.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/main/resources/spring/kafka-es-context-public.xml -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/kafka/indexer/CommonKafkaUtilsTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/java/org/elasticsearch/kafka/indexer/CommonKafkaUtilsTest.java -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/kafka/indexer/jobs/ConsumerManagerTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/java/org/elasticsearch/kafka/indexer/jobs/ConsumerManagerTest.java -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/kafka/indexer/jobs/ConsumerWorkerTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/java/org/elasticsearch/kafka/indexer/jobs/ConsumerWorkerTest.java -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/kafka/indexer/jobs/StartOptionParserTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/java/org/elasticsearch/kafka/indexer/jobs/StartOptionParserTest.java -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/kafka/indexer/service/ElasticSearchBatchServiceTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/java/org/elasticsearch/kafka/indexer/service/ElasticSearchBatchServiceTest.java -------------------------------------------------------------------------------- /src/test/org/elasticsearch/kafka/indexer/jmx/KafkaEsIndexerStatusTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/org/elasticsearch/kafka/indexer/jmx/KafkaEsIndexerStatusTest.java -------------------------------------------------------------------------------- /src/test/resources/kafka-es-indexer-docker.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/resources/kafka-es-indexer-docker.properties -------------------------------------------------------------------------------- /src/test/resources/test-start-options-custom-5-partitions.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/resources/test-start-options-custom-5-partitions.properties -------------------------------------------------------------------------------- /src/test/resources/test-start-options-custom-empty.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/resources/test-start-options-custom-empty.properties -------------------------------------------------------------------------------- /src/test/resources/test-start-options-custom-malformed.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/resources/test-start-options-custom-malformed.properties -------------------------------------------------------------------------------- /src/test/resources/test-start-options-custom.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BigDataDevs/kafka-elasticsearch-consumer/HEAD/src/test/resources/test-start-options-custom.properties --------------------------------------------------------------------------------