├── .gitignore ├── LICENSE ├── README.md ├── docker ├── pom.xml └── src │ └── main │ └── resources │ └── Dockerfile ├── pom.xml ├── rest ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── enthusiast94 │ │ └── kafkavisualizer │ │ ├── App.java │ │ ├── AppBootstrapper.java │ │ ├── JerseyConfig.java │ │ ├── api │ │ └── RestResource.java │ │ ├── domain │ │ ├── AppEnvironment.java │ │ ├── CommandLineArgs.java │ │ ├── IntegerMicroType.java │ │ ├── MaxTopicMessageCount.java │ │ └── kafka │ │ │ ├── KafkaBroker.java │ │ │ ├── KafkaConsumerInfo.java │ │ │ ├── KafkaStatics.java │ │ │ └── KafkaTopic.java │ │ ├── service │ │ ├── KafkaAllTopicsConsumer.java │ │ ├── KafkaBrokersTracker.java │ │ ├── KafkaProducerWrapper.java │ │ ├── KafkaTopicsDataTracker.java │ │ ├── KafkaTopicsTracker.java │ │ ├── KafkaUtils.java │ │ └── VersionedResponse.java │ │ └── util │ │ ├── HttpResponseFactory.java │ │ └── exception │ │ └── DefectException.java │ └── resources │ └── application.properties ├── screenshot_1.png └── web-react ├── pom.xml └── src └── main └── react-app ├── README.md ├── package-lock.json ├── package.json ├── public ├── favicon.ico ├── index.html └── manifest.json ├── src ├── actions │ ├── actionTypes.ts │ ├── brokerActions.ts │ └── topicActions.ts ├── api │ └── Api.ts ├── components │ ├── app │ │ ├── App.css │ │ └── App.tsx │ ├── brokers │ │ ├── BrokerList.test.tsx │ │ ├── BrokerList.tsx │ │ └── BrokerListItem.tsx │ ├── tabs │ │ └── Tabs.tsx │ └── topics │ │ ├── MessageListItem.tsx │ │ ├── MessagePublisher.tsx │ │ ├── SelectedTopicPartitionInfo.tsx │ │ ├── TopicConsumers.tsx │ │ ├── TopicList.test.tsx │ │ ├── TopicList.tsx │ │ ├── TopicListItem.tsx │ │ └── TopicMessage.tsx ├── constants │ └── envVariables.ts ├── domain │ ├── Broker.ts │ ├── Consumer.ts │ ├── RecordMetadata.ts │ ├── Topic.ts │ └── TopicMessage.ts ├── index.css ├── index.tsx ├── logo.svg ├── reducers │ ├── brokersReducer.ts │ ├── index.ts │ ├── initialState.ts │ ├── topicConsumersReducer.ts │ ├── topicMessagesReducer.ts │ └── topicReducer.ts ├── registerServiceWorker.ts ├── setupTests.ts ├── store │ └── configureStore.ts └── util │ └── StringUtils.ts ├── tsconfig.json ├── tsconfig.test.json └── tslint.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea/ 3 | *.iml 4 | **/target/* 5 | **/.settings/ 6 | .project 7 | .classpath 8 | .sw? 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Manas Bajaj 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka Visualizer 2 | 3 | A web client for visualizing your Kafka cluster. Developed using **Spring Boot**, **ReactJS** and **Bootstrap 4**. 4 | 5 | ![Screenshot](https://github.com/enthusiast94/kafka-visualizer/blob/master/screenshot_1.png) 6 | 7 | ## How to build? 8 | 9 | Run the following command on the parent maven module `kafka-visualizer`: 10 | 11 | `$ mvn package` 12 | 13 | The executable jar will be generated under the `target` directory of the `kafkavisualizer/rest` module. 14 | 15 | **Requirements:** 16 | 17 | 1. Maven 18 | 2. OpenJDK 12 19 | 3. Node Package Manager (npm) 20 | 21 | ## How to run (locally)? 22 | 23 | Run the executable jar using the following command and then navigate to `localhost:8080` on your browser: 24 | 25 | `$ java -jar /path/to/kafka-visualizer-rest-1.0-SNAPSHOT.jar --zookeeper=hostname:port --kafka=hostname:port --env=` 26 | 27 | ## How to run (Dockerfile)? 28 | 29 | ```sh 30 | $ docker pull kbhargava/kafka-visuals 31 | $ docker run -p 8080:8080 --rm kbhargava/kafka-visuals 32 | ``` 33 | 34 | Verify the deployment by navigating to your server address in your preferred browser. 35 | 36 | ```sh 37 | localhost:8080 38 | ``` 39 | 40 | ## Rest API endpoints 41 | 42 | - **`GET /api/brokers`**: Returns a list of all brokers in the cluster. 43 | - **`GET /api/topics`**: Returns a list of all topics in the cluster 44 | - **`GET /api/consumers`**: Returns a list of all active consumers. 45 | - **`GET /api/consumers/{topicName}/{partition}`**: Returns a list of active consumers for a certain topic-partition pair. 46 | - **`GET /api/topics/{topicName}/{partition}`**: Returns a list of messages on a certain topic-partition pair. 47 | - **`POST /api/topics/{topicName}`**: Accepts `text/plain` message with a `key` and a `value` (eg: key=1&value=2), and publishes it to a certain topic. 48 | -------------------------------------------------------------------------------- /docker/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | com.enthusiast94.kafkavisualizer 9 | kafka-visualizer 10 | 1.0-SNAPSHOT 11 | 12 | 13 | docker 14 | 15 | 16 | ${project.build.directory}/image 17 | ${project.parent.artifactId} 18 | 19 | 20 | 21 | 22 | 23 | ${project.groupId} 24 | rest 25 | ${project.version} 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | org.apache.maven.plugins 36 | maven-resources-plugin 37 | 38 | 39 | Copy the Dockerfile 40 | 41 | copy-resources 42 | 43 | process-resources 44 | 45 | 46 | 47 | src/main/resources 48 | true 49 | 50 | 51 | ${image.build.dir} 52 | 53 | 54 | 55 | 56 | default-resources 57 | 58 | resources 59 | 60 | none 61 | 62 | 63 | default-testResources 64 | 65 | testResources 66 | 67 | none 68 | 69 | 70 | 71 | 72 | 73 | org.apache.maven.plugins 74 | maven-dependency-plugin 75 | 76 | 77 | Copy REST App 78 | 79 | copy-dependencies 80 | 81 | process-resources 82 | 83 | rest 84 | ${image.build.dir} 85 | 86 | 87 | 88 | 89 | 90 | 91 | org.codehaus.mojo 92 | exec-maven-plugin 93 | 94 | 95 | Build Docker image 96 | 97 | exec 98 | 99 | package 100 | 101 | ${image.build.dir} 102 | docker 103 | 104 | build 105 | -t 106 | ${image.name}:${project.version} 107 | . 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | org.apache.maven.plugins 116 | maven-jar-plugin 117 | 118 | 119 | 120 | default-jar 121 | 122 | jar 123 | 124 | none 125 | 126 | 127 | 128 | 129 | 130 | org.apache.maven.plugins 131 | maven-compiler-plugin 132 | 133 | 134 | 135 | default-compile 136 | 137 | compile 138 | 139 | none 140 | 141 | 142 | default-testCompile 143 | 144 | testCompile 145 | 146 | none 147 | 148 | 149 | 150 | 151 | 152 | org.apache.maven.plugins 153 | maven-surefire-plugin 154 | 155 | 156 | 157 | default-test 158 | 159 | test 160 | 161 | none 162 | 163 | 164 | 165 | 166 | 167 | org.apache.maven.plugins 168 | maven-install-plugin 169 | 2.5.2 170 | 171 | 172 | 173 | default-install 174 | 175 | install 176 | 177 | none 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | -------------------------------------------------------------------------------- /docker/src/main/resources/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:12-jdk-alpine 2 | 3 | RUN apk --update add nodejs 4 | 5 | ARG app_jar=rest-@{project.version}.jar 6 | 7 | COPY rest-@{project.version}.jar / 8 | 9 | EXPOSE 8080 10 | 11 | ENTRYPOINT ["java", "-jar", "/rest-@{project.version}.jar"] 12 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | org.springframework.boot 9 | spring-boot-starter-parent 10 | 2.0.0.RELEASE 11 | 12 | 13 | com.enthusiast94.kafkavisualizer 14 | kafka-visualizer 15 | pom 16 | 1.0-SNAPSHOT 17 | 18 | 19 | web-react 20 | rest 21 | docker 22 | 23 | 24 | 25 | 12 26 | @{*} 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | org.apache.maven.plugins 36 | maven-compiler-plugin 37 | 3.8.1 38 | 39 | ${compiler.java.version} 40 | ${compiler.java.version} 41 | 42 | 43 | 44 | 45 | org.apache.maven.plugins 46 | maven-dependency-plugin 47 | 3.1.1 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-resources-plugin 53 | 3.1.0 54 | 55 | 56 | 57 | ${resources.filter.delimiter} 58 | 59 | false 60 | 61 | 62 | 63 | 64 | org.codehaus.mojo 65 | exec-maven-plugin 66 | 1.6.0 67 | 68 | 69 | 70 | org.apache.maven.plugins 71 | maven-jar-plugin 72 | 3.1.2 73 | 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-surefire-plugin 79 | 2.22.2 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /rest/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | com.enthusiast94.kafkavisualizer 9 | kafka-visualizer 10 | 1.0-SNAPSHOT 11 | 12 | 13 | rest 14 | 15 | 16 | 17 | 18 | org.springframework.boot 19 | spring-boot-starter-web 20 | 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter-jersey 25 | 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-starter-test 30 | test 31 | 32 | 33 | 34 | com.google.guava 35 | guava 36 | 23.0 37 | 38 | 39 | 40 | com.google.code.gson 41 | gson 42 | 2.8.1 43 | 44 | 45 | 46 | javax.xml.bind 47 | jaxb-api 48 | 2.3.0 49 | 50 | 51 | 52 | 53 | org.apache.kafka 54 | kafka_2.12 55 | 2.3.0 56 | 57 | 58 | 59 | com.sun.activation 60 | jakarta.activation 61 | 1.2.1 62 | 63 | 64 | 65 | ${project.groupId} 66 | web-react 67 | ${project.version} 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-dependency-plugin 79 | 80 | 81 | Unpack React App 82 | 83 | unpack-dependencies 84 | 85 | prepare-package 86 | 87 | web-react 88 | META-INF/** 89 | ${project.build.directory}/classes/static 90 | 91 | 92 | 93 | 94 | 95 | 96 | org.springframework.boot 97 | spring-boot-maven-plugin 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/App.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class App { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(App.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/AppBootstrapper.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer; 2 | 3 | import com.enthusiast94.kafkavisualizer.api.RestResource; 4 | import com.enthusiast94.kafkavisualizer.domain.CommandLineArgs; 5 | import com.enthusiast94.kafkavisualizer.service.*; 6 | import com.enthusiast94.kafkavisualizer.util.HttpResponseFactory; 7 | import kafka.admin.AdminClient; 8 | import org.I0Itec.zkclient.ZkClient; 9 | import org.I0Itec.zkclient.exception.ZkMarshallingError; 10 | import org.I0Itec.zkclient.serialize.ZkSerializer; 11 | import org.springframework.boot.ApplicationArguments; 12 | import org.springframework.context.annotation.Bean; 13 | import org.springframework.context.annotation.Configuration; 14 | 15 | import java.util.concurrent.TimeUnit; 16 | 17 | @Configuration 18 | public class AppBootstrapper { 19 | 20 | @Bean 21 | public CommandLineArgs commandLineArgs(ApplicationArguments applicationArguments) { 22 | return new CommandLineArgs(applicationArguments.getSourceArgs()); 23 | } 24 | 25 | @Bean 26 | public HttpResponseFactory httpResponseFactory() { 27 | return new HttpResponseFactory(); 28 | } 29 | 30 | @Bean(destroyMethod = "close") 31 | public ZkClient zkClient(CommandLineArgs commandLineArgs) { 32 | var zkClient = new ZkClient(commandLineArgs.zookeeperServers, 10000, 10000, 33 | new ZkSerializer() { 34 | public byte[] serialize(Object data) throws ZkMarshallingError { 35 | return String.valueOf(data).getBytes(); 36 | } 37 | 38 | public Object deserialize(byte[] bytes) throws ZkMarshallingError { 39 | return new String(bytes); 40 | } 41 | }); 42 | zkClient.waitUntilConnected(10, TimeUnit.SECONDS); 43 | return zkClient; 44 | } 45 | 46 | @Bean(destroyMethod = "close") 47 | public AdminClient adminClient(CommandLineArgs commandLineArgs) { 48 | return AdminClient.createSimplePlaintext(commandLineArgs.kafkaServers); 49 | } 50 | 51 | @Bean 52 | public KafkaUtils kafkaAdmin(ZkClient zkClient, AdminClient adminClient) { 53 | return new KafkaUtils(zkClient, adminClient); 54 | } 55 | 56 | @Bean(destroyMethod = "close") 57 | public KafkaAllTopicsConsumer kafkaConsumerWrapper(CommandLineArgs commandLineArgs) { 58 | return new KafkaAllTopicsConsumer(commandLineArgs.kafkaServers); 59 | } 60 | 61 | @Bean(destroyMethod = "close") 62 | public KafkaProducerWrapper kafkaProducerWrapper(CommandLineArgs commandLineArgs) { 63 | return new KafkaProducerWrapper(commandLineArgs.kafkaServers); 64 | } 65 | 66 | @Bean 67 | public KafkaTopicsDataTracker kafkaTopicsDataTracker(KafkaAllTopicsConsumer kafkaAllTopicsConsumer, 68 | CommandLineArgs commandLineArgs) { 69 | var kafkaTopicsDataTracker = new KafkaTopicsDataTracker(kafkaAllTopicsConsumer, commandLineArgs.maxTopicMessagesCount); 70 | kafkaTopicsDataTracker.start(); 71 | return kafkaTopicsDataTracker; 72 | } 73 | 74 | @Bean(destroyMethod = "close") 75 | public KafkaBrokersTracker kafkaBrokersTracker(ZkClient zkClient) { 76 | var kafkaBrokersTracker = new KafkaBrokersTracker(zkClient); 77 | kafkaBrokersTracker.start(); 78 | return kafkaBrokersTracker; 79 | } 80 | 81 | @Bean(destroyMethod = "close") 82 | public KafkaTopicsTracker kafkaTopicsTracker(ZkClient zkClient) { 83 | var kafkaTopicsTracker = new KafkaTopicsTracker(zkClient); 84 | kafkaTopicsTracker.start(); 85 | return kafkaTopicsTracker; 86 | } 87 | 88 | @Bean 89 | public RestResource mainResource(CommandLineArgs commandLineArgs, 90 | HttpResponseFactory httpResponseFactory, 91 | KafkaProducerWrapper kafkaProducerWrapper, 92 | KafkaBrokersTracker kafkaBrokersTracker, 93 | KafkaTopicsTracker kafkaTopicsTracker, 94 | KafkaTopicsDataTracker kafkaTopicsDataTracker, 95 | KafkaUtils kafkaUtils) { 96 | return new RestResource(commandLineArgs.environment, httpResponseFactory, kafkaProducerWrapper, 97 | kafkaBrokersTracker, kafkaTopicsTracker, kafkaTopicsDataTracker, kafkaUtils); 98 | } 99 | } -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/JerseyConfig.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer; 2 | 3 | import com.enthusiast94.kafkavisualizer.api.RestResource; 4 | import org.glassfish.jersey.server.ResourceConfig; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | import javax.ws.rs.ApplicationPath; 8 | 9 | @Configuration 10 | @ApplicationPath("/api") 11 | public class JerseyConfig extends ResourceConfig { 12 | 13 | public JerseyConfig(RestResource restResource) { 14 | register(restResource); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/api/RestResource.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.api; 2 | 3 | import com.enthusiast94.kafkavisualizer.domain.AppEnvironment; 4 | import com.enthusiast94.kafkavisualizer.service.*; 5 | import com.enthusiast94.kafkavisualizer.util.HttpResponseFactory; 6 | import org.apache.kafka.common.TopicPartition; 7 | import org.springframework.http.MediaType; 8 | 9 | import javax.ws.rs.*; 10 | import javax.ws.rs.core.Response; 11 | import java.net.URLDecoder; 12 | import java.nio.charset.StandardCharsets; 13 | import java.util.regex.Matcher; 14 | import java.util.regex.Pattern; 15 | 16 | @Path("/") 17 | @Produces(MediaType.APPLICATION_JSON_VALUE) 18 | public class RestResource { 19 | 20 | private final AppEnvironment environment; 21 | private final HttpResponseFactory responseFactory; 22 | private final KafkaProducerWrapper kafkaProducerWrapper; 23 | private final KafkaBrokersTracker kafkaBrokersTracker; 24 | private final KafkaTopicsTracker kafkaTopicsTracker; 25 | private final KafkaTopicsDataTracker kafkaTopicsDataTracker; 26 | private final KafkaUtils kafkaUtils; 27 | 28 | public RestResource(AppEnvironment environment, 29 | HttpResponseFactory responseFactory, 30 | KafkaProducerWrapper kafkaProducerWrapper, 31 | KafkaBrokersTracker kafkaBrokersTracker, 32 | KafkaTopicsTracker kafkaTopicsTracker, 33 | KafkaTopicsDataTracker kafkaTopicsDataTracker, KafkaUtils kafkaUtils) { 34 | this.environment = environment; 35 | this.responseFactory = responseFactory; 36 | this.kafkaProducerWrapper = kafkaProducerWrapper; 37 | this.kafkaBrokersTracker = kafkaBrokersTracker; 38 | this.kafkaTopicsTracker = kafkaTopicsTracker; 39 | this.kafkaTopicsDataTracker = kafkaTopicsDataTracker; 40 | this.kafkaUtils = kafkaUtils; 41 | } 42 | 43 | @GET 44 | @Path("/environment") 45 | public Response environment() { 46 | return responseFactory.createOkResponse(environment.toString()); 47 | } 48 | 49 | @GET 50 | @Path("/brokers") 51 | public Response brokers(@QueryParam("version") long version) { 52 | var brokers = kafkaBrokersTracker.getBrokers(version); 53 | if (brokers.isPresent()) { 54 | return responseFactory.createOkResponse(brokers.get()); 55 | } else { 56 | return responseFactory.createNotModifiedResponse(); 57 | } 58 | } 59 | 60 | @GET 61 | @Path("/topics") 62 | public Response topics(@QueryParam("version") long version) { 63 | var topics = kafkaTopicsTracker.getTopics(version); 64 | if (topics.isPresent()) { 65 | return responseFactory.createOkResponse(topics.get()); 66 | } else { 67 | return responseFactory.createNotModifiedResponse(); 68 | } 69 | } 70 | 71 | @GET 72 | @Path("/consumers") 73 | public Response consumers() { 74 | try { 75 | return responseFactory.createOkResponse(kafkaUtils.getAllConsumers()); 76 | } catch (Exception e) { 77 | return responseFactory.createServerErrorResponse(e); 78 | } 79 | } 80 | 81 | @GET 82 | @Path("/consumers/{topicName}/{partition}") 83 | public Response consumersForTopic(@PathParam("topicName") String topicName, @PathParam("partition") int partition) { 84 | try { 85 | if (!doesTopicAndPartitionExist(topicName, partition)) { 86 | return responseFactory.createNotFoundResponse(String.format("No (topic, partition) pair found for " + 87 | "[(%s, %s)]", topicName, partition)); 88 | } 89 | 90 | return responseFactory.createOkResponse(kafkaUtils.getConsumersForTopic(topicName, partition)); 91 | } catch (Exception e) { 92 | return responseFactory.createServerErrorResponse(e); 93 | } 94 | } 95 | 96 | @GET 97 | @Path("/topics/{topicName}/{partition}") 98 | public Response topicData(@PathParam("topicName") String topicName, 99 | @PathParam("partition") int partition, 100 | @QueryParam("version") long version, 101 | @QueryParam("query") String query) { 102 | try { 103 | var records = 104 | kafkaTopicsDataTracker.getRecords(new TopicPartition(topicName, partition), version, query); 105 | if (!records.isPresent()) { 106 | return responseFactory.createNotModifiedResponse(); 107 | } 108 | 109 | return responseFactory.createOkResponse(records.get()); 110 | } catch (Exception e) { 111 | return responseFactory.createServerErrorResponse(e); 112 | } 113 | } 114 | 115 | @POST 116 | @Path("/topics/{topicName}") 117 | public Response postTopicData(@PathParam("topicName") String topicName, 118 | String data) { 119 | try { 120 | var keyValuePair = parseKeyValuePair(data); 121 | 122 | if (!doesTopicExist(topicName)) { 123 | return responseFactory.createNotFoundResponse(String.format("No topic exists with the name [%s]", topicName)); 124 | } 125 | 126 | var metadata = kafkaProducerWrapper.publish(keyValuePair[0], keyValuePair[1], topicName); 127 | return responseFactory.createOkResponse(metadata); 128 | } catch (Exception e) { 129 | return responseFactory.createServerErrorResponse(e); 130 | } 131 | } 132 | 133 | private boolean doesTopicAndPartitionExist(String topicName, int partition) { 134 | return kafkaUtils.getAllTopics().stream() 135 | .anyMatch(topic -> topic.name.equals(topicName) && partition < topic.numPartitions); 136 | } 137 | 138 | private boolean doesTopicExist(String topicName) { 139 | return kafkaUtils.getAllTopics().stream() 140 | .anyMatch(topic -> topic.name.equals(topicName)); 141 | } 142 | 143 | private Pattern keyValuePairPattern = Pattern.compile("key=(.*)&value=(.*)", Pattern.DOTALL); 144 | 145 | private String[] parseKeyValuePair(String input) { 146 | input = URLDecoder.decode(input, StandardCharsets.UTF_8); 147 | if (input.startsWith("?")) { 148 | input = input.substring(1); 149 | } 150 | 151 | Matcher matcher = keyValuePairPattern.matcher(input); 152 | if (!matcher.find()){ 153 | throw new IllegalArgumentException("no key/value params found in the posted body"); 154 | } 155 | 156 | return new String[]{matcher.group(1), matcher.group(2)}; 157 | } 158 | } -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/AppEnvironment.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain; 2 | 3 | import com.enthusiast94.kafkavisualizer.util.exception.DefectException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum AppEnvironment { 8 | DEV, QA, UAT, PROD; 9 | 10 | public static AppEnvironment fromString(String environmentString) { 11 | return Arrays.stream(values()) 12 | .filter(env -> env.name().toLowerCase().equals(environmentString.toLowerCase())) 13 | .findFirst() 14 | .orElseThrow(() -> new DefectException(String.format("Failed to find app environment with name [%s]." + 15 | " Allowed values are [%s].", environmentString, Arrays.toString(values())))); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/CommandLineArgs.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain; 2 | 3 | import com.enthusiast94.kafkavisualizer.util.exception.DefectException; 4 | 5 | import java.util.Arrays; 6 | import java.util.regex.Pattern; 7 | 8 | public class CommandLineArgs { 9 | 10 | public final AppEnvironment environment; 11 | public final String kafkaServers; 12 | public final String zookeeperServers; 13 | public final MaxTopicMessageCount maxTopicMessagesCount; 14 | 15 | public CommandLineArgs(String[] args) { 16 | //TODO Use some standard library to parse command line args 17 | //TODO The 'env' arg seems meaningless 18 | if (args.length < 3) { 19 | throw new DefectException(String.format("The following command line arguments are required: --env=%s, " + 20 | "--zookeeper=host:port and --kafka=host:port", Arrays.toString(AppEnvironment.values()))); 21 | } 22 | 23 | var argsString = String.join(" ", args); 24 | 25 | var envMatcher = Pattern.compile("--env=(\\w*)").matcher(argsString); 26 | if (!envMatcher.find()) { 27 | throw new DefectException(String.format("Incorrect app environment format! The format must be [%s]", 28 | envMatcher.pattern().toString())); 29 | } else { 30 | environment = AppEnvironment.fromString(envMatcher.group(1)); 31 | } 32 | 33 | var kafkaMatcher = Pattern.compile("--kafka=([^\\s,]+:[0-9]+(,[^\\s,]+:[0-9]+)*)").matcher(argsString); 34 | if (!kafkaMatcher.find()) { 35 | throw new DefectException(String.format("Incorrect kafka servers format! The format must be [%s]", 36 | kafkaMatcher.pattern().toString())); 37 | } 38 | 39 | kafkaServers = kafkaMatcher.group(1); 40 | 41 | var zookeeperMatcher = Pattern.compile("--zookeeper=([^\\s,]+:[0-9]+(,[^\\s,]+:[0-9]+)*)").matcher(argsString); 42 | if (!zookeeperMatcher.find()) { 43 | throw new DefectException(String.format("Incorrect zookeeper servers format! The format must be [%s]", 44 | kafkaMatcher.pattern().toString())); 45 | } 46 | 47 | zookeeperServers = zookeeperMatcher.group(1); 48 | 49 | var maxTopicMessagesCountMatcher = Pattern.compile("--maxTopicMessagesCount=([0-9]+)").matcher(argsString); 50 | if (!maxTopicMessagesCountMatcher.find()) { 51 | maxTopicMessagesCount = MaxTopicMessageCount.EMPTY; 52 | } else { 53 | maxTopicMessagesCount = new MaxTopicMessageCount(Integer.parseInt(maxTopicMessagesCountMatcher.group(1))); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/IntegerMicroType.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain; 2 | 3 | import java.util.Objects; 4 | 5 | public class IntegerMicroType { 6 | 7 | public static final int EMPTY_VALUE = Integer.MIN_VALUE; 8 | public final int value; 9 | 10 | public IntegerMicroType(int value) { 11 | this.value = value; 12 | } 13 | 14 | @Override 15 | public boolean equals(Object o) { 16 | if (this == o) return true; 17 | if (o == null || getClass() != o.getClass()) return false; 18 | IntegerMicroType that = (IntegerMicroType) o; 19 | return value == that.value; 20 | } 21 | 22 | @Override 23 | public int hashCode() { 24 | return Objects.hash(value); 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return "IntegerMicroType{" + 30 | "value=" + value + 31 | '}'; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/MaxTopicMessageCount.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain; 2 | 3 | public class MaxTopicMessageCount extends IntegerMicroType { 4 | 5 | public static final MaxTopicMessageCount EMPTY = new MaxTopicMessageCount(EMPTY_VALUE); 6 | 7 | public MaxTopicMessageCount(int value) { 8 | super(value); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/kafka/KafkaBroker.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain.kafka; 2 | 3 | public class KafkaBroker { 4 | public final String id; 5 | public final String hostname; 6 | public final int port; 7 | 8 | public KafkaBroker(String id, String hostname, int port) { 9 | this.id = id; 10 | this.hostname = hostname; 11 | this.port = port; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/kafka/KafkaConsumerInfo.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain.kafka; 2 | 3 | import java.util.List; 4 | import java.util.Objects; 5 | 6 | public class KafkaConsumerInfo { 7 | 8 | public final String consumerId; 9 | public final String clientId; 10 | public final List assignments; 11 | public final String groupId; 12 | 13 | public KafkaConsumerInfo(String consumerId, String clientId, List assignments, String groupId) { 14 | this.consumerId = consumerId; 15 | this.clientId = clientId; 16 | this.assignments = assignments; 17 | this.groupId = groupId; 18 | } 19 | 20 | @Override 21 | public boolean equals(Object o) { 22 | if (this == o) return true; 23 | if (o == null || getClass() != o.getClass()) return false; 24 | KafkaConsumerInfo that = (KafkaConsumerInfo) o; 25 | return Objects.equals(consumerId, that.consumerId) && 26 | Objects.equals(clientId, that.clientId) && 27 | Objects.equals(assignments, that.assignments) && 28 | Objects.equals(groupId, that.groupId); 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return "KafkaConsumerInfo{" + 34 | "consumerId='" + consumerId + '\'' + 35 | ", clientId='" + clientId + '\'' + 36 | ", assignments=" + assignments + 37 | ", groupId='" + groupId + '\'' + 38 | '}'; 39 | } 40 | 41 | @Override 42 | public int hashCode() { 43 | return Objects.hash(consumerId, clientId, assignments, groupId); 44 | } 45 | 46 | public static class Assignment { 47 | public String topic; 48 | public int partition; 49 | 50 | public Assignment(String topic, int partition) { 51 | this.topic = topic; 52 | this.partition = partition; 53 | } 54 | 55 | @Override 56 | public boolean equals(Object o) { 57 | if (this == o) return true; 58 | if (o == null || getClass() != o.getClass()) return false; 59 | Assignment that = (Assignment) o; 60 | return partition == that.partition && 61 | Objects.equals(topic, that.topic); 62 | } 63 | 64 | @Override 65 | public int hashCode() { 66 | return Objects.hash(topic, partition); 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | return "Assignment{" + 72 | "topic='" + topic + '\'' + 73 | ", partition=" + partition + 74 | '}'; 75 | } 76 | } 77 | } 78 | 79 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/kafka/KafkaStatics.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain.kafka; 2 | 3 | import java.util.UUID; 4 | 5 | public class KafkaStatics { 6 | public static String GROUP_ID = "CORTEX_KAFKA_VISUALIZER_" + UUID.randomUUID().toString(); 7 | } 8 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/domain/kafka/KafkaTopic.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.domain.kafka; 2 | 3 | import java.util.Objects; 4 | 5 | public class KafkaTopic { 6 | public String name; 7 | public int numPartitions; 8 | 9 | public KafkaTopic(String name, int numPartitions) { 10 | this.name = name; 11 | this.numPartitions = numPartitions; 12 | } 13 | 14 | @Override 15 | public boolean equals(Object o) { 16 | if (this == o) return true; 17 | if (o == null || getClass() != o.getClass()) return false; 18 | KafkaTopic that = (KafkaTopic) o; 19 | return numPartitions == that.numPartitions && 20 | Objects.equals(name, that.name); 21 | } 22 | 23 | @Override 24 | public int hashCode() { 25 | return Objects.hash(name, numPartitions); 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return "KafkaTopic{" + 31 | "name='" + name + '\'' + 32 | ", numPartitions=" + numPartitions + 33 | '}'; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/service/KafkaAllTopicsConsumer.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.service; 2 | 3 | import com.enthusiast94.kafkavisualizer.domain.kafka.KafkaStatics; 4 | import com.enthusiast94.kafkavisualizer.util.exception.DefectException; 5 | import com.google.common.collect.ImmutableList; 6 | import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; 7 | import org.apache.kafka.clients.consumer.ConsumerRecord; 8 | import org.apache.kafka.clients.consumer.KafkaConsumer; 9 | import org.apache.kafka.common.TopicPartition; 10 | import org.apache.kafka.common.serialization.StringDeserializer; 11 | 12 | import java.util.Collection; 13 | import java.util.Properties; 14 | import java.util.concurrent.Executors; 15 | import java.util.concurrent.ScheduledExecutorService; 16 | import java.util.concurrent.TimeUnit; 17 | import java.util.concurrent.atomic.AtomicBoolean; 18 | import java.util.function.Consumer; 19 | import java.util.regex.Pattern; 20 | 21 | public class KafkaAllTopicsConsumer implements AutoCloseable { 22 | 23 | private final KafkaConsumer kafkaConsumer; 24 | private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); 25 | private final AtomicBoolean isSubscribed = new AtomicBoolean(); 26 | 27 | public KafkaAllTopicsConsumer(String kafkaServersString) { 28 | var properties = new Properties(); 29 | properties.put("bootstrap.servers", kafkaServersString); 30 | properties.put("group.id", KafkaStatics.GROUP_ID); 31 | properties.put("enable.auto.commit", "true"); 32 | properties.put("metadata.max.age.ms", 5000); 33 | 34 | kafkaConsumer = new KafkaConsumer<>(properties, new StringDeserializer(), new StringDeserializer()); 35 | } 36 | 37 | public void subscribe(Consumer> recordConsumer) { 38 | if (!isSubscribed.compareAndSet(false, true)) { 39 | throw new DefectException("Can only subscribe once!"); 40 | } 41 | 42 | subscribeFromBeginning(); 43 | 44 | executorService.scheduleAtFixedRate(() -> { 45 | var records = kafkaConsumer.poll(100); 46 | records.forEach(recordConsumer); 47 | }, 0, 1000, TimeUnit.MILLISECONDS); 48 | } 49 | 50 | private void subscribeFromBeginning() { 51 | kafkaConsumer.subscribe(Pattern.compile("^((?!__consumer_offsets).)*$"), new ConsumerRebalanceListener() { 52 | @Override 53 | public void onPartitionsRevoked(Collection partitions) { 54 | } 55 | 56 | @Override 57 | public void onPartitionsAssigned(Collection partitions) { 58 | } 59 | }); 60 | kafkaConsumer.poll(0); // subscribe is lazy, and therefore we need to poll before seeking 61 | kafkaConsumer.seekToBeginning(ImmutableList.of()); 62 | } 63 | 64 | @Override 65 | public void close() { 66 | kafkaConsumer.close(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/service/KafkaBrokersTracker.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.service; 2 | 3 | import com.enthusiast94.kafkavisualizer.domain.kafka.KafkaBroker; 4 | import com.enthusiast94.kafkavisualizer.util.exception.DefectException; 5 | import com.google.common.collect.ImmutableList; 6 | import com.google.common.collect.ImmutableSet; 7 | import com.google.common.collect.Sets; 8 | import com.google.gson.JsonParser; 9 | import org.I0Itec.zkclient.ZkClient; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.Optional; 14 | import java.util.Set; 15 | import java.util.concurrent.ConcurrentHashMap; 16 | import java.util.concurrent.ExecutorService; 17 | import java.util.concurrent.Executors; 18 | import java.util.concurrent.TimeUnit; 19 | import java.util.concurrent.atomic.AtomicBoolean; 20 | import java.util.concurrent.atomic.AtomicLong; 21 | 22 | public class KafkaBrokersTracker implements AutoCloseable { 23 | 24 | private static final Logger log = LoggerFactory.getLogger(KafkaBrokersTracker.class); 25 | 26 | private final ZkClient zkClient; 27 | private final JsonParser jsonParser = new JsonParser(); 28 | private final ExecutorService executor = Executors.newSingleThreadExecutor(); 29 | private final ConcurrentHashMap brokersById = new ConcurrentHashMap<>(); 30 | private final AtomicLong version = new AtomicLong(); 31 | private final AtomicBoolean started = new AtomicBoolean(); 32 | 33 | public KafkaBrokersTracker(ZkClient zkClient) { 34 | this.zkClient = zkClient; 35 | } 36 | 37 | public void start() { 38 | if (!started.compareAndSet(false, true)) { 39 | throw new DefectException("Can only be started once!"); 40 | } 41 | 42 | zkClient.waitUntilExists("/brokers", TimeUnit.SECONDS, 10); 43 | 44 | executor.submit(() -> { 45 | zkClient.subscribeChildChanges("/brokers/ids", (parentPath, currentChilds) -> 46 | updateBrokers(ImmutableSet.copyOf(currentChilds))); 47 | updateBrokers(ImmutableSet.copyOf(zkClient.getChildren("/brokers/ids"))); 48 | }); 49 | } 50 | 51 | public Optional>> getBrokers(long clientVersion) { 52 | if (clientVersion != 0 && clientVersion >= version.get()) { 53 | return Optional.empty(); 54 | } 55 | 56 | return Optional.of(new VersionedResponse<>(version.get(), ImmutableList.copyOf(brokersById.values()))); 57 | } 58 | 59 | private void updateBrokers(Set newBrokerIds) { 60 | var currentBrokers = brokersById.keySet(); 61 | var brokersToAdd = Sets.difference(newBrokerIds, currentBrokers); 62 | var brokersToRemove = Sets.difference(currentBrokers, newBrokerIds); 63 | 64 | brokersToAdd.forEach(brokerId -> { 65 | try { 66 | var jsonString = zkClient.readData("/brokers/ids/" + brokerId); 67 | var json = jsonParser.parse(jsonString).getAsJsonObject(); 68 | brokersById.put(brokerId, new KafkaBroker(brokerId, json.get("host").getAsString(), json.get("port").getAsInt())); 69 | version.incrementAndGet(); 70 | log.info("Broker added: [{}]", brokerId); 71 | } catch (Exception e) { 72 | log.error("Exception fetching broker info for broker [{}]: ", brokerId, e); 73 | } 74 | }); 75 | 76 | brokersToRemove.forEach(brokerId -> { 77 | brokersById.remove(brokerId); 78 | version.incrementAndGet(); 79 | log.info("Broker deleted: [{}]", brokerId); 80 | }); 81 | } 82 | 83 | @Override 84 | public void close() { 85 | executor.shutdownNow(); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/service/KafkaProducerWrapper.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.service; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerRecord; 5 | import org.apache.kafka.clients.producer.RecordMetadata; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | 8 | import java.util.Properties; 9 | import java.util.concurrent.TimeUnit; 10 | 11 | public class KafkaProducerWrapper implements AutoCloseable { 12 | 13 | private final KafkaProducer kafkaProducer; 14 | 15 | public KafkaProducerWrapper(String kafkaServersString) { 16 | var properties = new Properties(); 17 | properties.put("bootstrap.servers", kafkaServersString); 18 | 19 | kafkaProducer = new KafkaProducer<>(properties, new StringSerializer(), new StringSerializer()); 20 | } 21 | 22 | public RecordMetadata publish(String key, String value, String topicName) throws Exception { 23 | var record = new ProducerRecord<>(topicName, key, value); 24 | return kafkaProducer.send(record).get(5000, TimeUnit.MILLISECONDS); 25 | } 26 | 27 | @Override 28 | public void close() { 29 | kafkaProducer.close(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/service/KafkaTopicsDataTracker.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.service; 2 | 3 | import com.enthusiast94.kafkavisualizer.domain.MaxTopicMessageCount; 4 | import com.enthusiast94.kafkavisualizer.util.exception.DefectException; 5 | import com.google.common.base.Strings; 6 | import com.google.common.collect.ImmutableList; 7 | import com.google.common.collect.Lists; 8 | import org.apache.kafka.clients.consumer.ConsumerRecord; 9 | import org.apache.kafka.common.TopicPartition; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.HashMap; 14 | import java.util.LinkedList; 15 | import java.util.Map; 16 | import java.util.Optional; 17 | import java.util.concurrent.atomic.AtomicBoolean; 18 | import java.util.concurrent.atomic.AtomicLong; 19 | 20 | public class KafkaTopicsDataTracker { 21 | 22 | private static final Logger log = LoggerFactory.getLogger(KafkaTopicsDataTracker.class); 23 | 24 | private final KafkaAllTopicsConsumer kafkaAllTopicsConsumer; 25 | private final MaxTopicMessageCount maxTopicMessagesCount; 26 | private final Map messagesByTopicPartition = new HashMap<>(); 27 | private final AtomicBoolean started = new AtomicBoolean(); 28 | 29 | public KafkaTopicsDataTracker(KafkaAllTopicsConsumer kafkaAllTopicsConsumer, MaxTopicMessageCount maxTopicMessagesCount) { 30 | this.kafkaAllTopicsConsumer = kafkaAllTopicsConsumer; 31 | this.maxTopicMessagesCount = maxTopicMessagesCount; 32 | } 33 | 34 | public void start() { 35 | if (!started.compareAndSet(false, true)) { 36 | throw new DefectException("Can only be started once!"); 37 | } 38 | 39 | kafkaAllTopicsConsumer.subscribe(this::onMessageConsumed); 40 | } 41 | 42 | public synchronized Optional>>> getRecords(TopicPartition topicPartition, 43 | long clientVersion, 44 | String query) { 45 | if (!messagesByTopicPartition.containsKey(topicPartition)) { 46 | return Optional.of(new VersionedResponse<>(0, ImmutableList.of())); 47 | } 48 | 49 | var versionedMessages = messagesByTopicPartition.get(topicPartition); 50 | 51 | if (clientVersion != 0 && clientVersion >= versionedMessages.version.get()) { 52 | return Optional.empty(); 53 | } 54 | 55 | ImmutableList> filteredMessages; 56 | 57 | if (Strings.isNullOrEmpty(query)) { 58 | filteredMessages = ImmutableList.copyOf(versionedMessages.messages); 59 | } else { 60 | filteredMessages = versionedMessages.messages.stream() 61 | .filter(message -> message.value().toLowerCase().contains(query) || message.key().toLowerCase().contains(query)) 62 | .collect(ImmutableList.toImmutableList()); 63 | } 64 | 65 | return Optional.of(new VersionedResponse<>(versionedMessages.version.get(), filteredMessages)); 66 | } 67 | 68 | private synchronized void onMessageConsumed(ConsumerRecord record) { 69 | log.info(String.format("Consumed message: [%s]", record)); 70 | 71 | var topicPartition = new TopicPartition(record.topic(), record.partition()); 72 | 73 | if (!messagesByTopicPartition.containsKey(topicPartition)) { 74 | messagesByTopicPartition.put(topicPartition, new VersionedMessages(Lists.newLinkedList())); 75 | } 76 | 77 | var versionedMessages = messagesByTopicPartition.get(topicPartition); 78 | var messages = versionedMessages.messages; 79 | messages.addFirst(record); 80 | 81 | if (!maxTopicMessagesCount.equals(MaxTopicMessageCount.EMPTY) && messages.size() > maxTopicMessagesCount.value) { 82 | messages.removeLast(); 83 | } 84 | 85 | versionedMessages.version.incrementAndGet(); 86 | } 87 | 88 | private static class VersionedMessages { 89 | final AtomicLong version = new AtomicLong(); 90 | final LinkedList> messages; 91 | 92 | private VersionedMessages(LinkedList> messages) { 93 | this.messages = messages; 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/service/KafkaTopicsTracker.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.service; 2 | 3 | import com.enthusiast94.kafkavisualizer.domain.kafka.KafkaTopic; 4 | import com.enthusiast94.kafkavisualizer.util.exception.DefectException; 5 | import com.google.common.collect.ImmutableList; 6 | import com.google.common.collect.ImmutableSet; 7 | import com.google.common.collect.Sets; 8 | import org.I0Itec.zkclient.ZkClient; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.Optional; 13 | import java.util.Set; 14 | import java.util.concurrent.ConcurrentHashMap; 15 | import java.util.concurrent.ExecutorService; 16 | import java.util.concurrent.Executors; 17 | import java.util.concurrent.TimeUnit; 18 | import java.util.concurrent.atomic.AtomicBoolean; 19 | import java.util.concurrent.atomic.AtomicLong; 20 | 21 | public class KafkaTopicsTracker implements AutoCloseable { 22 | 23 | private static final Logger log = LoggerFactory.getLogger(KafkaBrokersTracker.class); 24 | 25 | private final ZkClient zkClient; 26 | private final ExecutorService executor = Executors.newSingleThreadExecutor(); 27 | private final ConcurrentHashMap topicsByName = new ConcurrentHashMap<>(); 28 | private final AtomicLong version = new AtomicLong(); 29 | private final AtomicBoolean started = new AtomicBoolean(); 30 | 31 | public KafkaTopicsTracker(ZkClient zkClient) { 32 | this.zkClient = zkClient; 33 | } 34 | 35 | public void start() { 36 | if (!started.compareAndSet(false, true)) { 37 | throw new DefectException("Can only be started once!"); 38 | } 39 | 40 | zkClient.waitUntilExists("/brokers", TimeUnit.SECONDS, 10); 41 | 42 | executor.submit(() -> { 43 | zkClient.subscribeChildChanges("/brokers/topics", (parentPath, currentChilds) -> 44 | updateTopics(ImmutableSet.copyOf(currentChilds))); 45 | updateTopics(ImmutableSet.copyOf(zkClient.getChildren("/brokers/topics"))); 46 | }); 47 | } 48 | 49 | public Optional>> getTopics(long clientVersion) { 50 | if (clientVersion != 0 && clientVersion >= version.get()) { 51 | return Optional.empty(); 52 | } 53 | 54 | return Optional.of(new VersionedResponse<>(version.get(), ImmutableList.copyOf(topicsByName.values()))); 55 | } 56 | 57 | private void updateTopics(Set topicNames) { 58 | var currentTopics = topicsByName.keySet(); 59 | var topicsToAdd = Sets.difference(topicNames, currentTopics); 60 | var topicsToRemove = Sets.difference(currentTopics, topicNames); 61 | 62 | topicsToAdd.stream() 63 | .filter(topicName -> !topicName.equals("__consumer_offsets")) 64 | .forEach(topicName -> { 65 | try { 66 | var partitionsPath = "/brokers/topics/" + topicName + "/partitions"; 67 | zkClient.waitUntilExists(partitionsPath, TimeUnit.SECONDS, 5); 68 | topicsByName.put(topicName, new KafkaTopic(topicName, zkClient.getChildren(partitionsPath).size())); 69 | version.incrementAndGet(); 70 | log.info("Topic added: [{}]", topicName); 71 | } catch (Exception e) { 72 | log.error("Exception fetching info for topic [{}]: {}", topicName, e.getMessage(), e); 73 | } 74 | }); 75 | 76 | topicsToRemove.forEach(topicName -> { 77 | topicsByName.remove(topicName); 78 | version.incrementAndGet(); 79 | log.info("Topic deleted: [{}]", topicName); 80 | }); 81 | } 82 | 83 | @Override 84 | public void close() { 85 | executor.shutdownNow(); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/service/KafkaUtils.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.service; 2 | 3 | import com.enthusiast94.kafkavisualizer.domain.kafka.KafkaBroker; 4 | import com.enthusiast94.kafkavisualizer.domain.kafka.KafkaConsumerInfo; 5 | import com.enthusiast94.kafkavisualizer.domain.kafka.KafkaStatics; 6 | import com.enthusiast94.kafkavisualizer.domain.kafka.KafkaTopic; 7 | import com.google.common.collect.ImmutableList; 8 | import com.google.gson.JsonParser; 9 | import kafka.admin.AdminClient; 10 | import org.I0Itec.zkclient.ZkClient; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import scala.collection.JavaConversions; 14 | 15 | import java.time.Duration; 16 | 17 | public class KafkaUtils { 18 | 19 | private static final Logger log = LoggerFactory.getLogger(KafkaUtils.class); 20 | 21 | private final ZkClient zkClient; 22 | private final AdminClient adminClient; 23 | private final JsonParser jsonParser = new JsonParser(); 24 | 25 | public KafkaUtils(ZkClient zkClient, AdminClient adminClient) { 26 | this.zkClient = zkClient; 27 | this.adminClient = adminClient; 28 | } 29 | 30 | public ImmutableList getAllBrokers() throws Exception { 31 | var brokersBuilder = ImmutableList.builder(); 32 | var brokerIds = ImmutableList.copyOf(zkClient.getChildren("/brokers/ids")); 33 | 34 | for (String brokerId : brokerIds) { 35 | String jsonString; 36 | try { 37 | jsonString = zkClient.readData("/brokers/ids/" + brokerId); 38 | } catch (Exception e) { 39 | throw new Exception(String.format("Failed to fetch broker data for broker [%s]", brokerId)); 40 | } 41 | var json = jsonParser.parse(jsonString).getAsJsonObject(); 42 | var broker = new KafkaBroker(brokerId, json.get("host").getAsString(), json.get("port").getAsInt()); 43 | brokersBuilder.add(broker); 44 | } 45 | 46 | return brokersBuilder.build(); 47 | } 48 | 49 | public ImmutableList getAllTopics() { 50 | var topicNames = ImmutableList.copyOf(zkClient.getChildren("/brokers/topics")); 51 | return topicNames.stream() 52 | .filter(topicName -> !topicName.equals("__consumer_offsets")) 53 | .map(topicName -> new KafkaTopic(topicName, zkClient.getChildren("/brokers/topics/" + 54 | topicName + "/partitions").size())) 55 | .collect(ImmutableList.toImmutableList()); 56 | } 57 | 58 | public ImmutableList getAllConsumers() { 59 | var consumerGroups = 60 | ImmutableList.copyOf(JavaConversions.asJavaCollection(adminClient.listAllConsumerGroupsFlattened())); 61 | 62 | var consumers = ImmutableList.builder(); 63 | 64 | consumerGroups.stream() 65 | .filter(groupOverview -> !groupOverview.groupId().equals(KafkaStatics.GROUP_ID)) 66 | .forEach(consumerGroup -> { 67 | var consumerGroupSummary = 68 | adminClient.describeConsumerGroup(consumerGroup.groupId(), Duration.ofSeconds(10).toMillis()); 69 | 70 | if (!consumerGroupSummary.consumers().isDefined()) { 71 | return; 72 | } 73 | 74 | var consumerSummaries = ImmutableList.copyOf( 75 | JavaConversions.asJavaCollection(consumerGroupSummary.consumers().get())); 76 | 77 | if (consumerSummaries.isEmpty()) { 78 | return; 79 | } 80 | 81 | consumers.addAll(convertToKafkaConsumerInfos(consumerSummaries, consumerGroup.groupId())); 82 | }); 83 | 84 | return consumers.build(); 85 | } 86 | 87 | public ImmutableList getConsumersForTopic(String topic, int partition) { 88 | return getAllConsumers().stream() 89 | .filter(consumer -> consumer.assignments.stream() 90 | .anyMatch(assignment -> assignment.partition == partition && assignment.topic.equals(topic))) 91 | .collect(ImmutableList.toImmutableList()); 92 | } 93 | 94 | private ImmutableList convertToKafkaConsumerInfos(ImmutableList consumerSummaries, String groupId) { 95 | return consumerSummaries.stream() 96 | .map(consumerSummary -> { 97 | ImmutableList assignments = JavaConversions.asJavaCollection(consumerSummary.assignment()) 98 | .stream() 99 | .map(assignment -> new KafkaConsumerInfo.Assignment(assignment.topic(), assignment.partition())) 100 | .collect(ImmutableList.toImmutableList()); 101 | 102 | return new KafkaConsumerInfo(consumerSummary.consumerId(), consumerSummary.clientId(), assignments, groupId); 103 | }) 104 | .collect(ImmutableList.toImmutableList()); 105 | } 106 | } 107 | 108 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/service/VersionedResponse.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.service; 2 | 3 | import java.util.Objects; 4 | 5 | public class VersionedResponse { 6 | public final long version; 7 | public final T data; 8 | 9 | public VersionedResponse(long version, T data) { 10 | this.version = version; 11 | this.data = data; 12 | } 13 | 14 | @Override 15 | public boolean equals(Object o) { 16 | if (this == o) return true; 17 | if (o == null || getClass() != o.getClass()) return false; 18 | VersionedResponse that = (VersionedResponse) o; 19 | return version == that.version && 20 | Objects.equals(data, that.data); 21 | } 22 | 23 | @Override 24 | public int hashCode() { 25 | return Objects.hash(version, data); 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return "VersionedResponse{" + 31 | "version=" + version + 32 | ", data=" + data + 33 | '}'; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/util/HttpResponseFactory.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.util; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.JsonObject; 5 | 6 | import javax.ws.rs.core.Response; 7 | 8 | public class HttpResponseFactory { 9 | 10 | private final Gson gson = new Gson(); 11 | 12 | public Response createOkResponse(T body) { 13 | return Response.ok().entity(gson.toJson(body)).build(); 14 | } 15 | 16 | public Response createNotModifiedResponse() { 17 | return Response.notModified().build(); 18 | } 19 | 20 | public Response createServerErrorResponse(Exception e) { 21 | return Response.serverError().entity(createErrorJsonString(e.getMessage())).build(); 22 | } 23 | 24 | public Response createBadRequestResponse(String errorMessage) { 25 | return Response.status(Response.Status.BAD_REQUEST).entity(createErrorJsonString(errorMessage)).build(); 26 | } 27 | 28 | public Response createNotFoundResponse(String errorMessage) { 29 | return Response.status(Response.Status.NOT_FOUND).entity(createErrorJsonString(errorMessage)).build(); 30 | } 31 | 32 | private String createErrorJsonString(String errorMessage) { 33 | var jsonObject = new JsonObject(); 34 | jsonObject.addProperty("error", errorMessage); 35 | return gson.toJson(jsonObject); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rest/src/main/java/com/enthusiast94/kafkavisualizer/util/exception/DefectException.java: -------------------------------------------------------------------------------- 1 | package com.enthusiast94.kafkavisualizer.util.exception; 2 | 3 | public class DefectException extends RuntimeException { 4 | 5 | public DefectException(Throwable throwable) { 6 | super(throwable); 7 | } 8 | 9 | public DefectException(String message) { 10 | super(message); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /rest/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | logging.level.root = INFO -------------------------------------------------------------------------------- /screenshot_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manasb-uoe/kafka-visualizer/d2b44f2cc133e82b4cead236fdd8daebcb6d15a8/screenshot_1.png -------------------------------------------------------------------------------- /web-react/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | kafka-visualizer 9 | com.enthusiast94.kafkavisualizer 10 | 1.0-SNAPSHOT 11 | 12 | 13 | web-react 14 | 15 | 16 | ${project.build.directory}/react-app 17 | 18 | 19 | 20 | 21 | 22 | 23 | org.apache.maven.plugins 24 | maven-resources-plugin 25 | 26 | 27 | Copy the source code of the React App 28 | 29 | copy-resources 30 | 31 | process-resources 32 | 33 | 34 | 35 | src/main/react-app 36 | false 37 | 38 | 39 | ${react-app.build.dir} 40 | 41 | 42 | 43 | 44 | default-resources 45 | 46 | resources 47 | 48 | none 49 | 50 | 51 | default-testResources 52 | 53 | testResources 54 | 55 | none 56 | 57 | 58 | 59 | 60 | 61 | org.codehaus.mojo 62 | exec-maven-plugin 63 | 64 | 65 | Install NodeJS modules 66 | 67 | exec 68 | 69 | compile 70 | 71 | ${react-app.build.dir} 72 | npm 73 | 74 | install 75 | 76 | 77 | 78 | 79 | Build React App 80 | 81 | exec 82 | 83 | compile 84 | 85 | ${react-app.build.dir} 86 | npm 87 | 88 | run 89 | build 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | org.apache.maven.plugins 98 | maven-jar-plugin 99 | 100 | 101 | Package React App 102 | 103 | jar 104 | 105 | package 106 | 107 | ${react-app.build.dir}/build 108 | 109 | 110 | 111 | 112 | default-jar 113 | 114 | jar 115 | 116 | none 117 | 118 | 119 | 120 | 121 | 122 | org.apache.maven.plugins 123 | maven-compiler-plugin 124 | 125 | 126 | 127 | default-compile 128 | 129 | compile 130 | 131 | none 132 | 133 | 134 | default-testCompile 135 | 136 | testCompile 137 | 138 | none 139 | 140 | 141 | 142 | 143 | 144 | org.apache.maven.plugins 145 | maven-surefire-plugin 146 | 147 | 148 | 149 | default-test 150 | 151 | test 152 | 153 | none 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /web-react/src/main/react-app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "app", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@types/react-redux": "^5.0.15", 7 | "bootstrap": "^4.0.0", 8 | "font-awesome": "^4.7.0", 9 | "jquery": "^3.5.0", 10 | "lodash": "^4.17.5", 11 | "react": "^16.2.0", 12 | "react-dom": "^16.2.0", 13 | "react-redux": "^5.0.7", 14 | "react-router-dom": "^4.2.2", 15 | "react-scripts-ts": "2.13.0", 16 | "redux": "^3.7.2", 17 | "redux-thunk": "^2.2.0", 18 | "rxjs": "^5.5.6", 19 | "vkbeautify": "^0.99.3" 20 | }, 21 | "scripts": { 22 | "start": "react-scripts-ts start", 23 | "build": "react-scripts-ts build", 24 | "test": "react-scripts-ts test --env=jsdom", 25 | "eject": "react-scripts-ts eject" 26 | }, 27 | "devDependencies": { 28 | "@types/bootstrap": "^4.0.1", 29 | "@types/enzyme": "^3.1.9", 30 | "@types/enzyme-adapter-react-16": "^1.0.2", 31 | "@types/jest": "^22.2.0", 32 | "@types/jquery": "^3.3.1", 33 | "@types/lodash": "^4.14.104", 34 | "@types/node": "^9.4.6", 35 | "@types/react": "^16.0.38", 36 | "@types/react-dom": "^16.0.4", 37 | "@types/react-router-dom": "^4.2.4", 38 | "@types/react-test-renderer": "^16.0.1", 39 | "@types/vkbeautify": "^0.99.2", 40 | "enzyme": "^3.3.0", 41 | "enzyme-adapter-react-16": "^1.1.1", 42 | "react-test-renderer": "^16.2.0", 43 | "typescript": "^2.7.2" 44 | }, 45 | "proxy": { 46 | "/api/*": { 47 | "target": "http://localhost:8080" 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /web-react/src/main/react-app/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manasb-uoe/kafka-visualizer/d2b44f2cc133e82b4cead236fdd8daebcb6d15a8/web-react/src/main/react-app/public/favicon.ico -------------------------------------------------------------------------------- /web-react/src/main/react-app/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 11 | 12 | 13 | 22 | Kafka Visualizer 23 | 24 | 25 | 28 |
29 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /web-react/src/main/react-app/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": "./index.html", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/actions/actionTypes.ts: -------------------------------------------------------------------------------- 1 | export const LOAD_ALL_TOPICS_SUCCESS = 'LOAD_ALL_TOPICS_SUCCESS'; 2 | export const LOAD_ALL_TOPICS_FAILURE = 'LOAD_ALL_TOPICS_FAILURE'; 3 | export const LOAD_ALL_TOPICS_STARTED = 'LOAD_ALL_TOPICS_STARTED'; 4 | 5 | export const SELECT_TOPIC = 'SELECT_TOPIC'; 6 | export const SELECT_PARITITON = 'SELECT_PARTITION'; 7 | 8 | export const LOAD_ALL_BROKERS_SUCCESS = 'LOAD_ALL_BROKERS_SUCCESS'; 9 | export const LOAD_ALL_BROKERS_FAILURE = 'LOAD_ALL_BROKERS_FAILURE'; 10 | export const LOAD_ALL_BROKERS_STARTED = 'LOAD_ALL_BROKERS_STARTED'; 11 | 12 | export const LOAD_TOPIC_MESSAGES_SUCCESS = 'LOAD_TOPIC_MESSAGES_SUCCESS'; 13 | export const LOAD_TOPIC_MESSAGES_FAILURE = 'LOAD_TOPIC_MESSAGES_FAILURE'; 14 | export const LOAD_TOPIC_MESSAGES_STARTED = 'LOAD_TOPIC_MESSAGES_STARTED'; 15 | 16 | export const LOAD_TOPIC_CONSUMERS_SUCCESS = 'LOAD_TOPIC_CONSUMERS_SUCCESS'; 17 | export const LOAD_TOPIC_CONSUMERS_FAILURE = 'LOAD_TOPIC_CONSUMERS_FAILURE'; 18 | export const LOAD_TOPIC_CONSUMERS_STARTED = 'LOAD_TOPIC_CONSUMERS_STARTED'; -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/actions/brokerActions.ts: -------------------------------------------------------------------------------- 1 | import { Action } from 'redux'; 2 | import { BrokerAction } from './brokerActions'; 3 | import api from '../api/Api'; 4 | import * as types from '../actions/actionTypes'; 5 | import Broker from '../domain/Broker'; 6 | import { Dispatch } from 'react-redux'; 7 | 8 | export interface BrokerAction extends Action { 9 | brokers: Broker[]; 10 | error: string; 11 | } 12 | 13 | export function loadAllBrokersStarted(): BrokerAction { 14 | return { type: types.LOAD_ALL_BROKERS_STARTED, brokers: [], error: '' }; 15 | } 16 | 17 | export function loadAllBrokersSuccess(brokers: Broker[]): BrokerAction { 18 | return { type: types.LOAD_ALL_BROKERS_SUCCESS, brokers: brokers, error: '' }; 19 | } 20 | 21 | export function loadAllBrokerFailure(error: string): BrokerAction { 22 | return { type: types.LOAD_ALL_BROKERS_FAILURE, brokers: [], error: error }; 23 | } 24 | 25 | export function loadAllBrokers() { 26 | // tslint:disable-next-line:no-any 27 | return (dispatch: Dispatch) => { 28 | dispatch(loadAllBrokersStarted()); 29 | api.getBrokers().subscribe( 30 | brokers => dispatch(loadAllBrokersSuccess(brokers)), 31 | error => dispatch(loadAllBrokerFailure(error)) 32 | ); 33 | }; 34 | } -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/actions/topicActions.ts: -------------------------------------------------------------------------------- 1 | import { LOAD_TOPIC_CONSUMERS_STARTED, LOAD_TOPIC_CONSUMERS_SUCCESS, LOAD_TOPIC_CONSUMERS_FAILURE } from './actionTypes'; 2 | import { Action } from 'redux'; 3 | import Topic from '../domain/Topic'; 4 | import * as types from '../actions/actionTypes'; 5 | import api from '../api/Api'; 6 | import { Dispatch } from 'react-redux'; 7 | import TopicMessage from '../domain/TopicMessage'; 8 | import { ISubscription } from 'rxjs/Subscription'; 9 | import { Consumer } from '../domain/Consumer'; 10 | 11 | export interface LoadTopicsAction extends Action { 12 | topics: Topic[]; 13 | error: string; 14 | } 15 | 16 | export interface SelectTopicAction extends Action { 17 | topic: Topic; 18 | } 19 | 20 | export interface SelectPartitionAction extends Action { 21 | paritition: number; 22 | } 23 | 24 | export interface LoadTopicMessagesAction extends Action { 25 | messages: TopicMessage[]; 26 | error: string; 27 | } 28 | 29 | export interface LoadTopicConsumersAction extends Action { 30 | consumers: Consumer[]; 31 | error: string; 32 | } 33 | 34 | export function loadAllTopicsStarted(): LoadTopicsAction { 35 | return { type: types.LOAD_ALL_TOPICS_STARTED, topics: [], error: '', }; 36 | } 37 | 38 | export function loadAllTopicsSuccess(topics: Array): LoadTopicsAction { 39 | return { type: types.LOAD_ALL_TOPICS_SUCCESS, topics: topics, error: '' }; 40 | } 41 | 42 | export function loadAllTopicsFailure(error: string): LoadTopicsAction { 43 | return { type: types.LOAD_ALL_TOPICS_FAILURE, topics: [], error: error }; 44 | } 45 | 46 | export function loadAllTopics() { 47 | // tslint:disable-next-line:no-any 48 | return (dispatch: Dispatch) => { 49 | dispatch(loadAllTopicsStarted()); 50 | api.getTopics() 51 | .subscribe( 52 | topics => dispatch(loadAllTopicsSuccess(topics)), 53 | error => dispatch(loadAllTopicsFailure(error))); 54 | }; 55 | } 56 | 57 | export function selectTopic(topic: Topic): SelectTopicAction { 58 | return { type: types.SELECT_TOPIC, topic: topic }; 59 | } 60 | 61 | export function selectPartition(partition: number): SelectPartitionAction { 62 | return { type: types.SELECT_PARITITON, paritition: partition }; 63 | } 64 | 65 | export function loadTopicMessagesSuccess(messages: TopicMessage[]): LoadTopicMessagesAction { 66 | return { type: types.LOAD_TOPIC_MESSAGES_SUCCESS, messages: messages, error: '' }; 67 | } 68 | 69 | export function loadTopicMessagesFailure(error: string): LoadTopicMessagesAction { 70 | return { type: types.LOAD_TOPIC_MESSAGES_FAILURE, error: error, messages: [] }; 71 | } 72 | 73 | export function loadTopicMessagesStarted(): LoadTopicMessagesAction { 74 | return { type: types.LOAD_TOPIC_MESSAGES_STARTED, messages: [], error: '' }; 75 | } 76 | 77 | let subscription: ISubscription; 78 | 79 | export function loadTopicMessages(topic: Topic, partition: number, query: string) { 80 | // tslint:disable-next-line:no-any 81 | return (dispatch: Dispatch) => { 82 | dispatch(loadTopicMessagesStarted()); 83 | if (subscription) { 84 | subscription.unsubscribe(); 85 | } 86 | 87 | subscription = api.getTopicMessages(topic, partition, query) 88 | .subscribe( 89 | messages => dispatch(loadTopicMessagesSuccess(messages)), 90 | error => dispatch(loadTopicMessagesFailure(error)) 91 | ); 92 | }; 93 | } 94 | 95 | export function loadTopicConsumersStarted(): LoadTopicConsumersAction { 96 | return { type: LOAD_TOPIC_CONSUMERS_STARTED, consumers: [], error: '' }; 97 | } 98 | 99 | export function loadTopicConsumersSuccess(consumers: Consumer[]) { 100 | return { type: LOAD_TOPIC_CONSUMERS_SUCCESS, consumers: consumers, error: '' }; 101 | } 102 | 103 | export function loadTopicConsumersFailure(error: string) { 104 | return { type: LOAD_TOPIC_CONSUMERS_FAILURE, consumers: [], error: error }; 105 | } 106 | 107 | export function loadTopicConsumers(topic: Topic, partition: number) { 108 | // tslint:disable-next-line:no-any 109 | return (dispatch: Dispatch) => { 110 | dispatch(loadTopicConsumersStarted()); 111 | api.getTopicConsumers(topic, partition) 112 | .subscribe( 113 | consumers => dispatch(loadTopicConsumersSuccess(consumers)), 114 | error => dispatch(loadTopicConsumersFailure(error)) 115 | ); 116 | }; 117 | } -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/api/Api.ts: -------------------------------------------------------------------------------- 1 | import { Consumer } from './../domain/Consumer'; 2 | import { Observable, Subject } from 'rxjs'; 3 | import Topic from '../domain/Topic'; 4 | import Broker from '../domain/Broker'; 5 | import TopicMessage from '../domain/TopicMessage'; 6 | import RecordMetadata from '../domain/RecordMetadata'; 7 | 8 | export interface IApi { 9 | getTopics(): Observable>; 10 | getBrokers(): Observable>; 11 | getTopicMessages(topic: Topic, partition: number, query: string): Observable>; 12 | publishTopicMessage(topic: Topic, key: string, value: string): Observable; 13 | getTopicConsumers(topic: Topic, partition: number): Observable>; 14 | } 15 | 16 | class Api implements IApi { 17 | 18 | public getTopics(): Observable> { 19 | return this.getVersionedResponseStream('/api/topics', 2000); 20 | } 21 | 22 | public getBrokers(): Observable { 23 | return this.getVersionedResponseStream('/api/brokers', 2000); 24 | } 25 | 26 | public publishTopicMessage(topic: Topic, key: string, value: string): Observable { 27 | const body = new URLSearchParams(); 28 | body.append('key', key); 29 | body.append('value', value); 30 | 31 | const subject = new Subject(); 32 | 33 | fetch(`/api/topics/${topic.name}`, { 34 | method: 'post', 35 | body: body.toString() 36 | }) 37 | .then((response) => response.json()) 38 | .then(jsonResponse => { 39 | subject.next({ 40 | offset: jsonResponse.offset, 41 | partition: jsonResponse.topicPartition.partition 42 | }); 43 | }) 44 | .catch(error => subject.error(error)); 45 | 46 | return subject.asObservable(); 47 | } 48 | 49 | public getTopicMessages(topic: Topic, partition: number, query: string): Observable { 50 | if (!query) { 51 | query = ''; 52 | } 53 | 54 | return this.getVersionedResponseStream( 55 | `/api/topics/${topic.name}/${partition}?query=${query}&`, 56 | 2000 57 | ); 58 | } 59 | 60 | getTopicConsumers(topic: Topic, partition: number): Observable { 61 | const subject = new Subject(); 62 | fetch(`/api/consumers/${topic.name}/${partition}`) 63 | .then((response) => response.json()) 64 | .then(jsonResponse => { 65 | if (jsonResponse.error) { 66 | return subject.error(jsonResponse.error); 67 | } 68 | return subject.next(jsonResponse); 69 | }) 70 | .catch(error => subject.error(error)); 71 | 72 | return subject.asObservable(); 73 | } 74 | 75 | private getVersionedResponseStream(endpoint: string, period: number): Observable { 76 | return this.getResponseStream(endpoint, period, true); 77 | } 78 | 79 | private getResponseStream(endpoint: string, period: number, isVersioned: boolean): Observable { 80 | const subject = new Subject(); 81 | let version = 0; 82 | 83 | const handler = () => { 84 | fetch(isVersioned ? this.createVersionedEndpoint(endpoint, version) : endpoint) 85 | .then(response => { 86 | if (response.status === 304) { 87 | throw new Error(response.status.toString()); 88 | } else { 89 | return response.json(); 90 | } 91 | }) 92 | .then( 93 | response => { 94 | if (response.error) { 95 | subject.error(response.error); 96 | } else if (isVersioned) { 97 | version = response.version; 98 | subject.next(response.data); 99 | } else { 100 | subject.next(response); 101 | } 102 | 103 | setTimeout(handler, period); 104 | }) 105 | .catch(error => { 106 | if (error.message !== '304') { 107 | console.error(error); 108 | } 109 | 110 | setTimeout(handler, period); 111 | }); 112 | }; 113 | 114 | handler(); 115 | return subject.asObservable(); 116 | } 117 | 118 | private createVersionedEndpoint(endpoint: string, version: number): string { 119 | if (endpoint.endsWith('&')) { 120 | return `${endpoint}version=${version}`; 121 | } else { 122 | return `${endpoint}?version=${version}`; 123 | } 124 | } 125 | } 126 | 127 | export default new Api(); 128 | -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/components/app/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | animation: App-logo-spin infinite 20s linear; 7 | height: 80px; 8 | } 9 | 10 | .App-header { 11 | background-color: #222; 12 | height: 150px; 13 | padding: 20px; 14 | color: white; 15 | } 16 | 17 | .App-title { 18 | font-size: 1.5em; 19 | } 20 | 21 | .App-intro { 22 | font-size: large; 23 | } 24 | 25 | @keyframes App-logo-spin { 26 | from { transform: rotate(0deg); } 27 | to { transform: rotate(360deg); } 28 | } 29 | -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/components/app/App.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import './App.css'; 3 | import TopicList from '../topics/TopicList'; 4 | import BrokerList from '../brokers/BrokerList'; 5 | import TopicMessages from '../topics/TopicMessage'; 6 | import { Tabs, Tab } from '../tabs/Tabs'; 7 | import TopicConsumers from '../topics/TopicConsumers'; 8 | 9 | class App extends React.Component { 10 | render() { 11 | return ( 12 |
13 |
14 |
15 | 16 | 17 |
18 |
19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 |
28 |
29 |
30 | ); 31 | } 32 | } 33 | 34 | export default App; 35 | -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/components/brokers/BrokerList.test.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { mount } from 'enzyme'; 3 | import { BrokerList } from './BrokerList'; 4 | import { BrokersState } from '../../reducers/initialState'; 5 | import Broker from '../../domain/Broker'; 6 | import BrokerListItem from './BrokerListItem'; 7 | 8 | describe('', () => { 9 | 10 | // tslint:disable-next-line:no-any 11 | let loadAllBrokers: any; 12 | 13 | beforeEach(() => { 14 | loadAllBrokers = jest.fn(); 15 | }); 16 | 17 | it('should load brokers on mount', () => { 18 | const brokersState: BrokersState = { isLoading: false, items: [], error: '' }; 19 | 20 | mount(); 21 | 22 | expect(loadAllBrokers.mock.calls.length).toBe(1); 23 | }); 24 | 25 | it('should show loading text when loading topics', () => { 26 | const brokersState: BrokersState = { isLoading: true, items: [], error: '' }; 27 | 28 | const wrapper = mount(); 29 | 30 | const listItems = wrapper.find('.sidebarListItem'); 31 | expect(listItems.length).toBe(1); 32 | expect(listItems.get(0).props.children).toEqual('Loading...'); 33 | }); 34 | 35 | it('should not show any brokers if none exist', () => { 36 | const brokersState: BrokersState = { isLoading: false, items: [], error: '' }; 37 | 38 | const wrapper = mount(); 39 | 40 | const listItems = wrapper.find('.sidebarListItem'); 41 | expect(listItems.length).toBe(1); 42 | expect(listItems.get(0).props.children).toEqual('No brokers found'); 43 | }); 44 | 45 | it('should show list of brokers', () => { 46 | const brokerItems: Broker[] = [ 47 | { hostname: 'hostname1', id: 1, port: 1234 }, 48 | { hostname: 'hostname2', id: 2, port: 4321 } 49 | ]; 50 | const brokersState: BrokersState = { isLoading: false, items: brokerItems, error: '' }; 51 | 52 | const wrapper = mount(); 53 | 54 | const listItems = wrapper.find(BrokerListItem); 55 | expect(listItems.length).toBe(2); 56 | expect(listItems.at(0).find('span').at(0).text()).toEqual(brokerItems[0].hostname + ':' + brokerItems[0].port); 57 | expect(listItems.at(1).find('span').at(0).text()).toEqual(brokerItems[1].hostname + ':' + brokerItems[1].port); 58 | }); 59 | }); -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/components/brokers/BrokerList.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { connect, Dispatch } from 'react-redux'; 3 | import { IAppState, BrokersState } from '../../reducers/initialState'; 4 | import * as brokerActions from '../../actions/brokerActions'; 5 | import BrokerListItem from './BrokerListItem'; 6 | 7 | interface BrokerListProps { 8 | brokers: BrokersState; 9 | // tslint:disable-next-line:no-any 10 | loadAllBrokers: any; 11 | } 12 | 13 | export class BrokerList extends React.Component { 14 | 15 | componentDidMount() { 16 | this.props.loadAllBrokers(); 17 | } 18 | 19 | render() { 20 | const brokerList = this.props.brokers.items.map((broker, index) => { 21 | return ( 22 | 23 | ); 24 | }); 25 | 26 | return ( 27 |
28 |
Brokers
29 | {this.props.brokers.isLoading &&
Loading...
} 30 | {!this.props.brokers.isLoading && this.props.brokers.items.length === 0 31 | &&
No brokers found
} 32 |
{brokerList}
33 |
34 | ); 35 | } 36 | } 37 | 38 | function mapStateToProps(state: IAppState) { 39 | return { 40 | brokers: state.brokers 41 | }; 42 | } 43 | 44 | // tslint:disable-next-line:no-any 45 | function mapDispatchToProps(dispatch: Dispatch) { 46 | return { 47 | loadAllBrokers: () => dispatch(brokerActions.loadAllBrokers()) 48 | }; 49 | } 50 | 51 | export default connect(mapStateToProps, mapDispatchToProps)(BrokerList); -------------------------------------------------------------------------------- /web-react/src/main/react-app/src/components/brokers/BrokerListItem.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import Broker from '../../domain/Broker'; 3 | 4 | interface BrokerListItemProps { 5 | broker: Broker; 6 | } 7 | 8 | export default function BrokerListItem({ broker }: BrokerListItemProps) { 9 | return ( 10 |
11 |
12 |