├── .DS_Store ├── .gitignore ├── .vscode └── settings.json ├── DashBoards └── MashDashboard.json ├── Dockerfile ├── README.md ├── Utils ├── .gitignore ├── .vscode │ └── settings.json ├── Dockerfile ├── package-lock.json ├── package.json ├── scripts │ └── prometheus.yml ├── test-front │ ├── main.js │ ├── main.js.map │ ├── main.ts │ ├── tsconfig.json │ └── webpack.config.js └── testbed │ ├── consumer │ ├── consumer.js │ ├── consumer.js.map │ └── consumer.ts │ ├── index.html │ ├── main.css │ ├── producer │ ├── producer.js │ ├── producer.js.map │ └── producer.ts │ ├── server.js │ ├── server.js.map │ ├── server.ts │ ├── topics │ ├── topics.js │ ├── topics.js.map │ └── topics.ts │ ├── tsconfig.json │ └── types │ ├── socket.js │ ├── socket.js.map │ └── socket.ts ├── assets ├── consumerConsumptionRate.gif ├── consumerLag.gif ├── screenRecording.mov ├── testbedConnection.gif ├── testbedConsumption.gif └── testbedProduction.gif ├── docker-compose.yml ├── index.html ├── install.sh ├── package-lock.json ├── package.json ├── plugin ├── consumer-lag │ ├── .circleci │ │ └── config.yml │ ├── .codeclimate.yml │ ├── .editorconfig │ ├── .gitignore │ ├── .prettierrc.js │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── appveyor.yml │ ├── docs │ │ └── README.md │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── Components │ │ │ ├── Consumer.tsx │ │ │ └── Topic.tsx │ │ ├── SimplePanel.tsx │ │ ├── img │ │ │ └── logo.svg │ │ ├── module.test.ts │ │ ├── module.ts │ │ ├── plugin.json │ │ └── types.ts │ ├── tsconfig.json │ └── yarn.lock ├── consumer-message-rate │ ├── .circleci │ │ └── config.yml │ ├── .codeclimate.yml │ ├── .editorconfig │ ├── .gitignore │ ├── .prettierrc.js │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── appveyor.yml │ ├── docs │ │ └── README.md │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── SimplePanel.tsx │ │ ├── img │ │ │ └── logo.svg │ │ ├── module.test.ts │ │ ├── module.ts │ │ ├── plugin.json │ │ └── types.ts │ ├── tsconfig.json │ └── yarn.lock └── control-panel │ ├── .circleci │ └── config.yml │ ├── .codeclimate.yml │ ├── .editorconfig │ ├── .gitignore │ ├── .prettierrc.js │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── appveyor.yml │ ├── docs │ └── README.md │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── Components │ │ ├── Alerts │ │ │ └── kafkaConnection.tsx │ │ ├── Buttons │ │ │ └── Produce.tsx │ │ └── Containers │ │ │ ├── Consumer.tsx │ │ │ ├── Producer.tsx │ │ │ └── TopicBox.tsx │ ├── ControlPanel.tsx │ ├── img │ │ └── logo.svg │ ├── module.test.ts │ ├── module.ts │ ├── plugin.json │ └── types.ts │ ├── tsconfig.json │ └── yarn.lock ├── testbed ├── Dockerfile ├── __tests__ │ ├── frontSocket.test.ts │ ├── puppeteer.js │ └── socketIo_test.js ├── babel.config.js ├── package-lock.json ├── package.json ├── scripts │ └── prometheus.yml ├── test-front │ ├── main.js │ ├── main.js.map │ ├── main.ts │ ├── socketEvents.ts │ ├── tsconfig.json │ └── webpack.config.js ├── testbed │ ├── consumer │ │ ├── consumer.js │ │ ├── consumer.js.map │ │ └── consumer.ts │ ├── index.html │ ├── main.css │ ├── producer │ │ ├── producer.js │ │ ├── producer.js.map │ │ └── producer.ts │ ├── server.js │ ├── server.js.map │ ├── server.ts │ ├── topics │ │ ├── topics.js │ │ ├── topics.js.map │ │ └── topics.ts │ ├── tsconfig.json │ └── types │ │ ├── socket.js │ │ ├── socket.js.map │ │ └── socket.ts └── webpack.config.js ├── tsconfig.json └── webpack.config.js /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "deno.enable": false 3 | } -------------------------------------------------------------------------------- /DashBoards/MashDashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "Prometheus", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "limit": 100, 11 | "name": "Annotations & Alerts", 12 | "showIn": 0, 13 | "type": "dashboard", 14 | "useValueForTime": true 15 | } 16 | ] 17 | }, 18 | "editable": true, 19 | "gnetId": null, 20 | "graphTooltip": 1, 21 | "id": 8, 22 | "links": [ 23 | { 24 | "asDropdown": true, 25 | "icon": "external link", 26 | "includeVars": true, 27 | "keepTime": true, 28 | "tags": [], 29 | "targetBlank": true, 30 | "type": "dashboards" 31 | } 32 | ], 33 | "panels": [ 34 | { 35 | "datasource": null, 36 | "fieldConfig": { 37 | "defaults": { 38 | "custom": {} 39 | }, 40 | "overrides": [] 41 | }, 42 | "gridPos": { 43 | "h": 9, 44 | "w": 6, 45 | "x": 0, 46 | "y": 0 47 | }, 48 | "id": 6, 49 | "options": { 50 | "text": "kafka:9092" 51 | }, 52 | "pluginVersion": "7.1.0", 53 | "timeFrom": null, 54 | "timeShift": null, 55 | "title": "Control Panel", 56 | "type": "envoya-control-panel" 57 | }, 58 | { 59 | "datasource": null, 60 | "fieldConfig": { 61 | "defaults": { 62 | "custom": {} 63 | }, 64 | "overrides": [] 65 | }, 66 | "gridPos": { 67 | "h": 9, 68 | "w": 10, 69 | "x": 6, 70 | "y": 0 71 | }, 72 | "id": 2, 73 | "interval": "", 74 | "options": { 75 | "seriesCountSize": "sm", 76 | "showSeriesCount": false, 77 | "text": "Default value of text input option" 78 | }, 79 | "pluginVersion": "7.1.0", 80 | "targets": [ 81 | { 82 | "expr": "kafka_topic_partition_current_offset", 83 | "interval": "", 84 | "legendFormat": "", 85 | "refId": "A" 86 | }, 87 | { 88 | "expr": "kafka_consumergroup_current_offset", 89 | "interval": "", 90 | "legendFormat": "", 91 | "refId": "B" 92 | } 93 | ], 94 | "timeFrom": null, 95 | "timeShift": null, 96 | "title": "Consumer Lag", 97 | "transformations": [], 98 | "type": "envoya-consumer-lag" 99 | }, 100 | { 101 | "datasource": null, 102 | "fieldConfig": { 103 | "defaults": { 104 | "custom": {} 105 | }, 106 | "overrides": [] 107 | }, 108 | "gridPos": { 109 | "h": 9, 110 | "w": 8, 111 | "x": 16, 112 | "y": 0 113 | }, 114 | "id": 4, 115 | "options": { 116 | "seriesCountSize": "sm", 117 | "showSeriesCount": false, 118 | "text": "Default value of text input option" 119 | }, 120 | "pluginVersion": "7.1.0", 121 | "targets": [ 122 | { 123 | "expr": "kafka_consumergroup_current_offset", 124 | "interval": "", 125 | "legendFormat": "", 126 | "refId": "A" 127 | } 128 | ], 129 | "timeFrom": null, 130 | "timeShift": null, 131 | "title": "Consumer Message Rate", 132 | "type": "envoya-consumer-message-rate" 133 | } 134 | ], 135 | "refresh": "10s", 136 | "schemaVersion": 26, 137 | "style": "dark", 138 | "tags": [], 139 | "templating": { 140 | "list": [] 141 | }, 142 | "time": { 143 | "from": "now-6h", 144 | "to": "now" 145 | }, 146 | "timepicker": { 147 | "refresh_intervals": [ 148 | "10s", 149 | "30s", 150 | "1m", 151 | "5m", 152 | "15m", 153 | "30m", 154 | "1h", 155 | "2h", 156 | "1d" 157 | ] 158 | }, 159 | "timezone": "", 160 | "title": "MASH", 161 | "uid": "HSDwRZSGz", 162 | "version": 3 163 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:latest 2 | 3 | COPY . /build 4 | 5 | WORKDIR /build 6 | 7 | RUN npm i 8 | 9 | EXPOSE 2022 10 | 11 | CMD ["npm", "run", "testbed"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MASH 2 | 3 | Welcome to the newest product from Envoya, MASH! 4 | 5 | As the scale of your product grows, so does the complexity, and nowhere more so than in your message passing and stream processing infrastructure. 6 | 7 | MASH makes it easy to see your most important metrics at a glance. Made specifically to visualize your Kafka cluster, MASH brings you a novel way to simply understand the health of your consumers, and show metrics such as consumer lag in a way that can be understood instantly. 8 | 9 | ![Consumer Lag Panel](assets/consumerLag.gif) 10 | 11 | Organization of data such as the average and outlier message consumption rates is also presented clearly and concisely. 12 | 13 | ![Consumption Rate Panel](assets/consumerConsumptionRate.gif) 14 | 15 | In addition to the visulization of these critical metrics, MASH also provides an integrated testbed for message production and consumption to an existing or test Kafka cluster. 16 | 17 | Using the control panel, you can create topics and produce messages to them in a clean and efficient way. You can also create consumer groups and consume messages with instant feedback. 18 | 19 | ![Testbed Create a Consumer Topic and Produce Data](assets/testbedProduction.gif) 20 | 21 | TO USE: 22 | 23 | MASH is integrated tightly into the tools you already use. MASH uses the Grafana platform to present its panels and dashboards. In addition, MASH uses Prometheus with the Kafka Exporter for data collection and aggregation. 24 | 25 | MASH will need a running instance of Kafka, along with Kafka Exporter and Prometheus. If you are not currently integrated with Prometheus and Kafka Exporter, or are creating a new Grafana instance, a docker-compose file has been provided for your convenience. 26 | 27 | Go into the base directory and run the following commands: 28 | 29 | ``` 30 | chmod +x install.sh 31 | ./install.sh 32 | ``` 33 | 34 | The docker-compose file will spin up docker containers with the following: 35 | 36 | - Kafka 37 | 38 | - Zookeeper 39 | 40 | - Prometheus (available at port 9090) 41 | 42 | - Kafka Exporter 43 | 44 | - Grafana (available at port 3000) 45 | 46 | - The MASH Testbed (available at port 2022) 47 | 48 | USING THE TESTBED: 49 | 50 | As part of developing MASH, our team attempted to procure a testing platform which would allow us to produce mock data to the Kafka platform, but unfortunately none were available in our preferred language and tech stack. Since we wanted control over the process, we developed an internal testing platform which we feel can provide useful functionality to others. 51 | 52 | Therefore, we have incorporated the testbed into the MASH package. This can be controlled through Grafana by using the Control Panel. This will provide the current topics of your Kafka instance, along with the ability to send and consume messages from the cluster through a proxy server, and to create new topics and produce messages to them. 53 | 54 | A near-term goal for MASH is to incorporate customizable data for more control of the test data produced through the platform. 55 | 56 | CONTRIBUTING: 57 | 58 | The team at Envoya invites all testing, feedback, and suggestions that the community can provide. We pride ourselves on our commitment to the Open Source Community and believe in the collaborative nature of software design and growth. 59 | -------------------------------------------------------------------------------- /Utils/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /Utils/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "deno.enable": false 3 | } -------------------------------------------------------------------------------- /Utils/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:latest 2 | 3 | COPY . /image 4 | 5 | WORKDIR /image 6 | 7 | RUN npm i 8 | 9 | EXPOSE 2022 10 | 11 | CMD [""] -------------------------------------------------------------------------------- /Utils/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MASH_testbed", 3 | "version": "1.0.0", 4 | "description": "A kafka testbed for MASH", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "testbed": "node ./testbed/server.js", 9 | "build-testbed": "webpack test-front/main.ts -o test-front/main.js" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "dependencies": { 15 | "@types/express": "^4.17.7", 16 | "@types/kafkajs": "^1.9.0", 17 | "@types/socket.io": "^2.1.9", 18 | "@types/socket.io-client": "^1.4.33", 19 | "express": "^4.17.1", 20 | "kafkajs": "^1.12.0", 21 | "socket.io": "^2.3.0", 22 | "socket.io-client": "^2.3.0", 23 | "ts-loader": "^8.0.1", 24 | "typescript": "^3.9.7", 25 | "webpack": "^4.43.0", 26 | "webpack-cli": "^3.3.12", 27 | "webpack-dev-server": "^3.11.0" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Utils/scripts/prometheus.yml: -------------------------------------------------------------------------------- 1 | scrape_configs: 2 | - job_name: 'kafka_exporter' 3 | scrape_interval: 5s 4 | static_configs: 5 | - targets: ['kafka_exporter:9308'] 6 | labels: 7 | group: 'kafka_exporter' 8 | -------------------------------------------------------------------------------- /Utils/test-front/main.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"main.js","sourceRoot":"","sources":["main.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAIvC,MAAM,MAAM,GAAG,EAAE,EAAE,CAAC;AAEpB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;IACxB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,EAAE;IACxC,MAAM,IAAI,GAAmB;QAC3B,aAAa,EAAE,GAAG;KACnB,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,GAAG,EAAE;IAC7B,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACzB,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,IAAqB,EAAE,EAAE;IACpD,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC1C,OAAO,CAAC,SAAS,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC9D,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAClD,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9C,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACzB,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,IAAqB,EAAE,EAAE;IACpD,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC1C,OAAO,CAAC,SAAS,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC9E,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;IACjD,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACtD,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACzB,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,QAAQ,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;AAChE,MAAM,aAAa,GAAG,QAAQ,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;AAEhE,aAAa,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;IAC3C,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC;IACzC,iBAAiB,CAAC,CAAC,WAAW,CAAC,CAAC;IAChC,UAAU,CAAC,SAAS,GAAG,EAAE,CAAC;AAC5B,CAAC,CAAC,CAAC;AAEH,aAAa,CAAC,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;AAE3D,MAAM,CAAC,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC;AAClD,MAAM,CAAC,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC"} -------------------------------------------------------------------------------- /Utils/test-front/main.ts: -------------------------------------------------------------------------------- 1 | import * as io from 'socket.io-client'; 2 | import { SocketMessages, SocketNames } from '../testbed/types/socket'; 3 | 4 | const socket = io(); 5 | let consumeCount = 0; 6 | let produceCount = 0; 7 | socket.on('connect', () => { 8 | console.log('socket connected'); 9 | }); 10 | 11 | const sendProduceSignal = (num: number) => { 12 | const produceNum: SocketMessages.produceNum = { 13 | messagesCount: num, 14 | }; 15 | socket.emit(SocketNames.produceNum, produceNum); 16 | }; 17 | 18 | const sendConsumeSignal = (topic: string) => { 19 | const consumeAll: SocketMessages.consumeAll = { 20 | topic, 21 | }; 22 | socket.emit(SocketNames.consumeAll, consumeAll); 23 | }; 24 | 25 | const sendOverloadPartitionSignal = ( 26 | messages: number = 100, 27 | numPartitions: number = 2, 28 | topic: string = 'test-topic' 29 | ) => { 30 | const overload: SocketMessages.overloadPartition = { 31 | messages, 32 | numPartitions, 33 | topic, 34 | }; 35 | socket.emit(SocketNames.overloadPartition, overload); 36 | }; 37 | 38 | const sendProduceAtRateSignal = (rate: number) => { 39 | const produceAtRate: SocketMessages.produceRate = { 40 | rate, 41 | }; 42 | socket.emit(SocketNames.produceRate, produceAtRate); 43 | }; 44 | 45 | const sendConsumeAtRateSignal = (topic: string, rate: number) => { 46 | const consumeAtRate: SocketMessages.consumeRate = { 47 | rate, 48 | topic, 49 | }; 50 | socket.emit(SocketNames.consumeRate, consumeAtRate); 51 | }; 52 | 53 | const sendUnderReplicateSignal = (topic: string, numReplicants: number = 2) => { 54 | const sendUnderReplicate: SocketMessages.underReplicate = { 55 | topic, 56 | numReplicants, 57 | }; 58 | socket.emit(SocketNames.underReplicate, sendUnderReplicate); 59 | }; 60 | 61 | const listProduceResponse = (data: any) => { 62 | produceCount++; 63 | const container = document.getElementById('produce-container'); 64 | const img = document.getElementById('produce-img'); 65 | container.innerText = `PRODUCED: ${produceCount} - ${data.value.name}: ${data.value.quote}`; 66 | container.style.border = 67 | '2px solid ' + '#' + produceCount.toString(16).padStart(6, '0'); 68 | img.setAttribute('src', data.value.image_url); 69 | }; 70 | 71 | const listConsumeResponse = async (data: any) => { 72 | data = await JSON.parse(data); 73 | consumeCount++; 74 | const container = document.getElementById('consume-container'); 75 | const img = document.getElementById('consume-img'); 76 | container.innerText = ` CONSUMED: ${consumeCount} - ${data.name}: ${data.quote}`; 77 | container.style.border = 78 | '2px solid ' + '#' + produceCount.toString(16).padStart(6, '0'); 79 | img.setAttribute('src', data.image_url); 80 | }; 81 | 82 | const produceButton = document.getElementById('produce-button'); 83 | const consumeButton = document.getElementById('consume-button'); 84 | const sendOverloadPartitionButton = document.getElementById( 85 | 'OverloadPartition-button' 86 | ); 87 | const sendProduceAtRateButton = document.getElementById('produceRate-button'); 88 | const sendConsumeAtRateButton = document.getElementById('consumeRate-button'); 89 | const sendUnderReplicateButton = document.getElementById( 90 | 'underReplicate-button' 91 | ); 92 | 93 | produceButton.addEventListener('click', () => { 94 | const inputField = document.getElementById('message-num') as HTMLInputElement; 95 | const numMessages = inputField.value; 96 | sendProduceSignal(+numMessages); 97 | inputField.value = ''; 98 | }); 99 | 100 | consumeButton.addEventListener('click', () => { 101 | const inputField = document.getElementById( 102 | 'consume-input' 103 | ) as HTMLInputElement; 104 | const topic = inputField.value; 105 | inputField.value = ''; 106 | sendConsumeSignal(topic); 107 | }); 108 | 109 | sendOverloadPartitionButton.addEventListener('click', () => { 110 | const inputField = document.getElementById( 111 | 'OverloadPartition' 112 | ) as HTMLInputElement; 113 | const numMessages = inputField.value; 114 | sendOverloadPartitionSignal(+numMessages); 115 | inputField.value = ''; 116 | }); 117 | 118 | sendProduceAtRateButton.addEventListener('click', () => { 119 | const inputField = document.getElementById('produceRate') as HTMLInputElement; 120 | const numMessages = inputField.value; 121 | sendProduceAtRateSignal(+numMessages); 122 | inputField.value = ''; 123 | }); 124 | 125 | sendConsumeAtRateButton.addEventListener('click', () => { 126 | const inputString = document.getElementById( 127 | 'consumeRateSt' 128 | ) as HTMLInputElement; 129 | const inputNumber = document.getElementById( 130 | 'consumeRateNum' 131 | ) as HTMLInputElement; 132 | const stringMessages = inputString.value; 133 | const numberMessages = inputString.value; 134 | sendConsumeAtRateSignal(stringMessages, +numberMessages); 135 | inputNumber.value = ''; 136 | }); 137 | 138 | sendUnderReplicateButton.addEventListener('click', () => { 139 | const inputString = document.getElementById( 140 | 'underReplicateSt' 141 | ) as HTMLInputElement; 142 | const inputNumber = document.getElementById( 143 | 'underReplicateNum' 144 | ) as HTMLInputElement; 145 | const stringMessages = inputString.value; 146 | const numberMessages = inputString.value; 147 | sendUnderReplicateSignal(stringMessages, +numberMessages); 148 | inputNumber.value = ''; 149 | }); 150 | 151 | socket.on('consumeResponse', listConsumeResponse); 152 | socket.on('produceResponse', listProduceResponse); 153 | -------------------------------------------------------------------------------- /Utils/test-front/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": true, 4 | "sourceMap": true, 5 | "module": "ES2015", 6 | "target": "ES2015" 7 | }, 8 | "files": ["main.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /Utils/test-front/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | mode: 'development', 5 | entry: './main.ts', 6 | output: { 7 | filename: 'main.js', 8 | path: path.resolve(__dirname), 9 | }, 10 | module: { 11 | rules: [ 12 | { 13 | test: /\.tsx?$/, 14 | use: 'ts-loader', 15 | exclude: /node_modules/, 16 | }, 17 | ], 18 | }, 19 | resolve: { 20 | extensions: ['.tsx', '.ts', '.js'], 21 | }, 22 | }; 23 | -------------------------------------------------------------------------------- /Utils/testbed/consumer/consumer.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | exports.consumeAtRate = exports.consume = void 0; 40 | var kafkajs_1 = require("kafkajs"); 41 | var kafka = new kafkajs_1.Kafka({ 42 | clientId: 'test-consumer', 43 | brokers: ['kafka:9092'] 44 | }); 45 | var count = 0; 46 | var consumer = kafka.consumer({ groupId: 'test-group' }); 47 | consumer.subscribe({ topic: 'test-topic' }); 48 | exports.consume = function (socket, topic) { 49 | if (topic === void 0) { topic = 'test-topic'; } 50 | return __awaiter(void 0, void 0, void 0, function () { 51 | var consumer; 52 | return __generator(this, function (_a) { 53 | switch (_a.label) { 54 | case 0: 55 | consumer = kafka.consumer({ groupId: 'test-group' }); 56 | consumer.subscribe({ topic: topic }); 57 | return [4 /*yield*/, consumer.run({ 58 | eachMessage: function (payload) { return __awaiter(void 0, void 0, void 0, function () { 59 | var message; 60 | return __generator(this, function (_a) { 61 | message = payload.message; 62 | socket.emit('consumeResponse', message.value.toString()); 63 | return [2 /*return*/]; 64 | }); 65 | }); } 66 | })]; 67 | case 1: 68 | _a.sent(); 69 | return [2 /*return*/]; 70 | } 71 | }); 72 | }); 73 | }; 74 | exports.consumeAtRate = function (socket, rate, topic) { return __awaiter(void 0, void 0, void 0, function () { 75 | var ratedConsumer_1, err_1; 76 | return __generator(this, function (_a) { 77 | switch (_a.label) { 78 | case 0: 79 | _a.trys.push([0, 4, , 5]); 80 | ratedConsumer_1 = kafka.consumer(); 81 | return [4 /*yield*/, ratedConsumer_1.connect()]; 82 | case 1: 83 | _a.sent(); 84 | return [4 /*yield*/, ratedConsumer_1.subscribe({ topic: topic })]; 85 | case 2: 86 | _a.sent(); 87 | return [4 /*yield*/, ratedConsumer_1.run({ 88 | eachMessage: function (payload) { return __awaiter(void 0, void 0, void 0, function () { 89 | var value; 90 | return __generator(this, function (_a) { 91 | value = payload.message.value; 92 | socket.emit('ratedConsumeResponse', value.toString()); 93 | ratedConsumer_1.pause([{ topic: topic }]); 94 | setTimeout(function () { return ratedConsumer_1.resume([{ topic: topic }]); }, Math.floor(1000 / rate)); 95 | return [2 /*return*/]; 96 | }); 97 | }); } 98 | })]; 99 | case 3: 100 | _a.sent(); 101 | return [3 /*break*/, 5]; 102 | case 4: 103 | err_1 = _a.sent(); 104 | console.error('rated consumer error:', err_1); 105 | return [3 /*break*/, 5]; 106 | case 5: return [2 /*return*/]; 107 | } 108 | }); 109 | }); }; 110 | //# sourceMappingURL=consumer.js.map -------------------------------------------------------------------------------- /Utils/testbed/consumer/consumer.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"consumer.js","sourceRoot":"","sources":["consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAAoD;AAEpD,IAAM,KAAK,GAAG,IAAI,eAAK,CAAC;IACtB,QAAQ,EAAE,eAAe;IACzB,OAAO,EAAE,CAAC,YAAY,CAAC;CACxB,CAAC,CAAC;AAEH,IAAI,KAAK,GAAG,CAAC,CAAC;AAEd,IAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;AAC3D,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;AAe/B,QAAA,OAAO,GAAG,UAAO,MAAW,EAAE,KAA4B;IAA5B,sBAAA,EAAA,oBAA4B;;;;;;oBAC/D,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;oBAC3D,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,OAAA,EAAE,CAAC,CAAC;oBAC9B,qBAAM,QAAQ,CAAC,GAAG,CAAC;4BACjB,WAAW,EAAE,UAAO,OAA2B;;;oCACrC,OAAO,GAAK,OAAO,QAAZ,CAAa;oCAC5B,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;;;iCAC1D;yBACF,CAAC,EAAA;;oBALF,SAKE,CAAC;;;;;CACJ,CAAC;AAEW,QAAA,aAAa,GAAG,UAAO,MAAW,EAAE,IAAY,EAAE,KAAa;;;;;;gBAElE,kBAAgB,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACvC,qBAAM,eAAa,CAAC,OAAO,EAAE,EAAA;;gBAA7B,SAA6B,CAAC;gBAC9B,qBAAM,eAAa,CAAC,SAAS,CAAC,EAAE,KAAK,OAAA,EAAE,CAAC,EAAA;;gBAAxC,SAAwC,CAAC;gBACzC,qBAAM,eAAa,CAAC,GAAG,CAAC;wBACtB,WAAW,EAAE,UAAO,OAA2B;;;gCACrC,KAAK,GAAK,OAAO,CAAC,OAAO,MAApB,CAAqB;gCAClC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;gCACtD,eAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,OAAA,EAAE,CAAC,CAAC,CAAC;gCACjC,UAAU,CAAC,cAAM,OAAA,eAAa,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,OAAA,EAAE,CAAC,CAAC,EAAjC,CAAiC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;;;6BAC9E;qBACF,CAAC,EAAA;;gBAPF,SAOE,CAAC;;;;gBAEH,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAG,CAAC,CAAC;;;;;KAE/C,CAAC"} -------------------------------------------------------------------------------- /Utils/testbed/consumer/consumer.ts: -------------------------------------------------------------------------------- 1 | import { Kafka, EachMessagePayload } from 'kafkajs'; 2 | 3 | const kafka = new Kafka({ 4 | clientId: 'test-consumer', 5 | brokers: ['kafka:9092'], 6 | }); 7 | 8 | let count = 0; 9 | 10 | const consumer = kafka.consumer({ groupId: 'test-group' }); 11 | consumer.subscribe({ topic: 'test-topic' }); 12 | export interface ConsumeRequest { 13 | messagesCount: number; 14 | } 15 | 16 | export interface ConsumeResponse { 17 | message: { 18 | value: { 19 | name: string; 20 | quote: string; 21 | image_url: string; 22 | }; 23 | }; 24 | } 25 | 26 | export const consume = async (socket: any, topic: string = 'test-topic') => { 27 | const consumer = kafka.consumer({ groupId: 'test-group' }); 28 | consumer.subscribe({ topic }); 29 | await consumer.run({ 30 | eachMessage: async (payload: EachMessagePayload) => { 31 | const { message } = payload; 32 | socket.emit('consumeResponse', message.value.toString()); 33 | }, 34 | }); 35 | }; 36 | 37 | export const consumeAtRate = async (socket: any, rate: number, topic: string) => { 38 | try { 39 | const ratedConsumer = kafka.consumer(); 40 | await ratedConsumer.connect(); 41 | await ratedConsumer.subscribe({ topic }); 42 | await ratedConsumer.run({ 43 | eachMessage: async (payload: EachMessagePayload) => { 44 | const { value } = payload.message; 45 | socket.emit('ratedConsumeResponse', value.toString()); 46 | ratedConsumer.pause([{ topic }]); 47 | setTimeout(() => ratedConsumer.resume([{ topic }]), Math.floor(1000 / rate)); 48 | }, 49 | }); 50 | } catch (err) { 51 | console.error('rated consumer error:', err); 52 | } 53 | }; 54 | -------------------------------------------------------------------------------- /Utils/testbed/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | KAFKA TEST BED 8 | 9 | 10 | 11 |
12 |
13 | 14 | 15 |
16 | # 17 |
18 |
19 |
20 | 21 | 26 | 27 |
28 | # 29 |
30 |
31 |
32 |
33 |
34 | 35 | 36 | 37 |
38 |
39 | 40 | 41 | 42 |
43 |
44 | 45 | 46 | 47 | 48 |
49 |
50 | 51 | 52 | 53 | 54 |
55 |
56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /Utils/testbed/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | background: lightBlue; 3 | } 4 | #overall { 5 | display: flex; 6 | flex-direction: row; 7 | justify-content: space-between; 8 | margin: 1rem; 9 | } 10 | #overall > div { 11 | margin: 2rem; 12 | width: 100%; 13 | } 14 | #produce-container { 15 | font-size: x-large; 16 | padding: 2em; 17 | } 18 | #consume-container { 19 | padding: 2em; 20 | font-size: x-large; 21 | } 22 | 23 | img { 24 | height: 25px; 25 | padding-left: 6px; 26 | } 27 | 28 | .createFields { 29 | width: 100%; 30 | display: flex; 31 | margin-bottom: 0.2rem; 32 | } 33 | 34 | .createFields label { 35 | min-width: 8rem; 36 | width: 9rem; 37 | } 38 | .createFields button { 39 | min-width: 8rem; 40 | width: 9rem; 41 | } 42 | .register { 43 | color: white; 44 | background-color: lightslategrey; 45 | } 46 | -------------------------------------------------------------------------------- /Utils/testbed/producer/producer.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | exports.produceToPartition = exports.produceAtRateToPartition = exports.produce = exports.produceAtRate = void 0; 40 | var kafkajs_1 = require("kafkajs"); 41 | var kafka = new kafkajs_1.Kafka({ 42 | clientId: 'test-producer', 43 | brokers: ['kafka:9092'] 44 | }); 45 | var count = 0; 46 | var producer = kafka.producer(); 47 | var connect = function () { return __awaiter(void 0, void 0, void 0, function () { 48 | return __generator(this, function (_a) { 49 | switch (_a.label) { 50 | case 0: return [4 /*yield*/, producer.connect()]; 51 | case 1: 52 | _a.sent(); 53 | console.log('producer connected.'); 54 | return [2 /*return*/]; 55 | } 56 | }); 57 | }); }; 58 | connect(); 59 | exports.produceAtRate = function (rate) { return setInterval(exports.produce, Math.floor(1000 / rate)); }; 60 | exports.produce = function () { 61 | return new Promise(function (resolve, reject) { return __awaiter(void 0, void 0, void 0, function () { 62 | var message, stringMessage, err_1; 63 | return __generator(this, function (_a) { 64 | switch (_a.label) { 65 | case 0: 66 | _a.trys.push([0, 2, , 3]); 67 | message = { 68 | key: 'key' + count++, 69 | value: { 70 | name: 'me', 71 | quote: 'hello', 72 | image_url: 'https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/joypixels/257/smiling-face-with-sunglasses_1f60e.png' 73 | } 74 | }; 75 | stringMessage = { 76 | key: message.key, 77 | value: JSON.stringify(message.value) 78 | }; 79 | return [4 /*yield*/, producer.send({ 80 | topic: 'test-topic', 81 | messages: [stringMessage] 82 | })]; 83 | case 1: 84 | _a.sent(); 85 | resolve(message); 86 | return [3 /*break*/, 3]; 87 | case 2: 88 | err_1 = _a.sent(); 89 | reject(new Error(err_1)); 90 | return [3 /*break*/, 3]; 91 | case 3: return [2 /*return*/]; 92 | } 93 | }); 94 | }); }); 95 | }; 96 | exports.produceAtRateToPartition = function (topic, partition, rate) { 97 | if (partition === void 0) { partition = 0; } 98 | return setInterval(function () { return exports.produceToPartition(topic, partition); }, Math.floor(1000 / rate)); 99 | }; 100 | exports.produceToPartition = function (topic, partition) { 101 | if (partition === void 0) { partition = 0; } 102 | return __awaiter(void 0, void 0, void 0, function () { 103 | var message, err_2; 104 | return __generator(this, function (_a) { 105 | switch (_a.label) { 106 | case 0: 107 | message = { 108 | value: 'Besik', 109 | partition: partition 110 | }; 111 | _a.label = 1; 112 | case 1: 113 | _a.trys.push([1, 3, , 4]); 114 | return [4 /*yield*/, producer.send({ topic: topic, messages: [message] })]; 115 | case 2: 116 | _a.sent(); 117 | return [3 /*break*/, 4]; 118 | case 3: 119 | err_2 = _a.sent(); 120 | console.error('error sending to specific partition', err_2); 121 | return [3 /*break*/, 4]; 122 | case 4: return [2 /*return*/]; 123 | } 124 | }); 125 | }); 126 | }; 127 | //# sourceMappingURL=producer.js.map -------------------------------------------------------------------------------- /Utils/testbed/producer/producer.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"producer.js","sourceRoot":"","sources":["producer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAAyC;AAEzC,IAAM,KAAK,GAAG,IAAI,eAAK,CAAC;IACtB,QAAQ,EAAE,eAAe;IACzB,OAAO,EAAE,CAAC,YAAY,CAAC;CACxB,CAAC,CAAC;AAEH,IAAI,KAAK,GAAG,CAAC,CAAC;AAEd,IAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;AAalC,IAAM,OAAO,GAAG;;;oBACd,qBAAM,QAAQ,CAAC,OAAO,EAAE,EAAA;;gBAAxB,SAAwB,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;;;;KACpC,CAAC;AACF,OAAO,EAAE,CAAC;AAEG,QAAA,aAAa,GAAG,UAAC,IAAY,IAAK,OAAA,WAAW,CAAC,eAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,EAA7C,CAA6C,CAAC;AAEhF,QAAA,OAAO,GAAG;IACrB,OAAO,IAAI,OAAO,CAAC,UAAO,OAAO,EAAE,MAAM;;;;;;oBAE/B,OAAO,GAAoB;wBAC/B,GAAG,EAAE,KAAK,GAAG,KAAK,EAAE;wBACpB,KAAK,EAAE;4BACL,IAAI,EAAE,IAAI;4BACV,KAAK,EAAE,OAAO;4BACd,SAAS,EACP,4HAA4H;yBAC/H;qBACF,CAAC;oBACI,aAAa,GAAY;wBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;wBAChB,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;qBACrC,CAAC;oBAEF,qBAAM,QAAQ,CAAC,IAAI,CAAC;4BAClB,KAAK,EAAE,YAAY;4BACnB,QAAQ,EAAE,CAAC,aAAa,CAAC;yBAC1B,CAAC,EAAA;;oBAHF,SAGE,CAAC;oBACH,OAAO,CAAC,OAAO,CAAC,CAAC;;;;oBAEjB,MAAM,CAAC,IAAI,KAAK,CAAC,KAAG,CAAC,CAAC,CAAC;;;;;SAE1B,CAAC,CAAC;AACL,CAAC,CAAC;AAEW,QAAA,wBAAwB,GAAG,UAAC,KAAa,EAAE,SAAqB,EAAE,IAAY;IAAnC,0BAAA,EAAA,aAAqB;IAC3E,OAAO,WAAW,CAAC,cAAM,OAAA,0BAAkB,CAAC,KAAK,EAAE,SAAS,CAAC,EAApC,CAAoC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;AAC1F,CAAC,CAAC;AACW,QAAA,kBAAkB,GAAG,UAAO,KAAa,EAAE,SAAqB;IAArB,0BAAA,EAAA,aAAqB;;;;;;oBACrE,OAAO,GAAY;wBACvB,KAAK,EAAE,OAAO;wBACd,SAAS,WAAA;qBACV,CAAC;;;;oBAEA,qBAAM,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,OAAA,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,EAAA;;oBAAnD,SAAmD,CAAC;;;;oBAEpD,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,KAAG,CAAC,CAAC;;;;;;CAE7D,CAAC"} -------------------------------------------------------------------------------- /Utils/testbed/producer/producer.ts: -------------------------------------------------------------------------------- 1 | import { Kafka, Message } from 'kafkajs'; 2 | 3 | const kafka = new Kafka({ 4 | clientId: 'test-producer', 5 | brokers: ['kafka:9092'], 6 | }); 7 | 8 | let count = 0; 9 | 10 | const producer = kafka.producer(); 11 | export interface ProduceRequest { 12 | messagesCount: number; 13 | } 14 | 15 | export interface ProduceResponse { 16 | key: string; 17 | value: { 18 | name: string; 19 | quote: string; 20 | image_url: string; 21 | }; 22 | } 23 | const connect = async () => { 24 | await producer.connect(); 25 | console.log('producer connected.'); 26 | }; 27 | connect(); 28 | 29 | export const produceAtRate = (rate: number) => setInterval(produce, Math.floor(1000 / rate)); 30 | 31 | export const produce = (): Promise => { 32 | return new Promise(async (resolve, reject) => { 33 | try { 34 | const message: ProduceResponse = { 35 | key: 'key' + count++, 36 | value: { 37 | name: 'me', 38 | quote: 'hello', 39 | image_url: 40 | 'https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/joypixels/257/smiling-face-with-sunglasses_1f60e.png', 41 | }, 42 | }; 43 | const stringMessage: Message = { 44 | key: message.key, 45 | value: JSON.stringify(message.value), 46 | }; 47 | 48 | await producer.send({ 49 | topic: 'test-topic', 50 | messages: [stringMessage], 51 | }); 52 | resolve(message); 53 | } catch (err) { 54 | reject(new Error(err)); 55 | } 56 | }); 57 | }; 58 | 59 | export const produceAtRateToPartition = (topic: string, partition: number = 0, rate: number) => { 60 | return setInterval(() => produceToPartition(topic, partition), Math.floor(1000 / rate)); 61 | }; 62 | export const produceToPartition = async (topic: string, partition: number = 0) => { 63 | const message: Message = { 64 | value: 'Besik', 65 | partition, 66 | }; 67 | try { 68 | await producer.send({ topic, messages: [message] }); 69 | } catch (err) { 70 | console.error('error sending to specific partition', err); 71 | } 72 | }; 73 | -------------------------------------------------------------------------------- /Utils/testbed/server.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | var express = require("express"); 40 | var path = require("path"); 41 | var app = express(); 42 | var http_1 = require("http"); 43 | var server = http_1.createServer(app); 44 | var io = require('socket.io')(server); 45 | var socket_1 = require("./types/socket"); 46 | var producer_1 = require("./producer/producer"); 47 | var consumer_1 = require("./consumer/consumer"); 48 | var topics_1 = require("./topics/topics"); 49 | io.on('connection', function () { return console.log('Socket server connected...'); }); 50 | io.on('connect', function (socket) { 51 | socket.on(socket_1.SocketNames.produceNum, function (data) { 52 | // should run the produce function data.messagesCount amount of times 53 | var messagesCount = data.messagesCount; 54 | while (messagesCount) { 55 | // should send back the data on completion of each to show on the client side 56 | producer_1.produce() 57 | .then(function (resp) { return socket.emit('produceResponse', resp); })["catch"](console.error); 58 | messagesCount--; 59 | } 60 | }); 61 | socket.on(socket_1.SocketNames.produceRate, function (data) { 62 | producer_1.produceAtRate(data.rate); 63 | }); 64 | socket.on(socket_1.SocketNames.consumeRate, function (data) { 65 | var rate = data.rate, topic = data.topic; 66 | consumer_1.consumeAtRate(socket, rate, topic); 67 | }); 68 | socket.on(socket_1.SocketNames.overloadPartition, function (data) { return __awaiter(void 0, void 0, void 0, function () { 69 | var topic, numPartitions, messages; 70 | return __generator(this, function (_a) { 71 | switch (_a.label) { 72 | case 0: 73 | topic = data.topic, numPartitions = data.numPartitions; 74 | messages = data.messages; 75 | return [4 /*yield*/, topics_1.createTopicWithMultiplePartitions(topic, numPartitions)]; 76 | case 1: 77 | _a.sent(); 78 | while (messages) { 79 | // we can add in custom partitioning logic here, if needed 80 | // right now, this will overload partition 0 by default 81 | producer_1.produceToPartition(topic); 82 | messages -= 1; 83 | } 84 | return [2 /*return*/]; 85 | } 86 | }); 87 | }); }); 88 | socket.on(socket_1.SocketNames.underReplicate, function (data) { 89 | var topic = data.topic, numReplicants = data.numReplicants; 90 | // any number of replicants greater than the number of brokers will cause an under-replication event 91 | // default number of brokers is 1 92 | // default number of replicants is 2 93 | topics_1.createTopicWithMultipleReplicants(topic, numReplicants); 94 | }); 95 | socket.on(socket_1.SocketNames.createTopic, function (data) { 96 | var topic = data.topic; 97 | topics_1.createTopic(topic); 98 | }); 99 | socket.on(socket_1.SocketNames.consumeAll, function (data) { 100 | var topic = data.topic; 101 | consumer_1.consume(socket, topic); 102 | }); 103 | }); 104 | var PORT = 2022; 105 | app.get('/main.css', function (req, res) { 106 | return res.sendFile(path.resolve(__dirname, 'main.css')); 107 | }); 108 | app.get('/main.js', function (req, res) { 109 | return res.sendFile(path.resolve(__dirname, '../test-front/main.js')); 110 | }); 111 | app.get('/', function (req, res) { 112 | return res.sendFile(path.resolve(__dirname, './index.html')); 113 | }); 114 | server.listen(PORT, function () { return console.log("Server listening on port " + PORT); }); 115 | //# sourceMappingURL=server.js.map -------------------------------------------------------------------------------- /Utils/testbed/server.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"server.js","sourceRoot":"","sources":["server.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iCAAmC;AACnC,2BAA6B;AAC7B,IAAM,GAAG,GAAG,OAAO,EAAE,CAAC;AAEtB,6BAAoC;AACpC,IAAM,MAAM,GAAG,mBAAY,CAAC,GAAG,CAAC,CAAC;AACjC,IAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAExC,yCAA6D;AAE7D,gDAM6B;AAC7B,gDAA6D;AAC7D,0CAIyB;AAEzB,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,cAAM,OAAA,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,EAAzC,CAAyC,CAAC,CAAC;AACrE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,UAAC,MAAW;IAC3B,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,UAAU,EAAE,UAAC,IAA+B;QAChE,qEAAqE;QAC/D,IAAA,aAAa,GAAK,IAAI,cAAT,CAAU;QAC7B,OAAO,aAAa,EAAE;YACpB,6EAA6E;YAC7E,kBAAO,EAAE;iBACN,IAAI,CAAC,UAAC,IAAqB,IAAK,OAAA,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,EAApC,CAAoC,CAAC,CACrE,OAAK,CAAA,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACxB,aAAa,EAAE,CAAC;SACjB;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,WAAW,EAAE,UAAC,IAAgC;QAClE,wBAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,WAAW,EAAE,UAAC,IAAgC;QAC1D,IAAA,IAAI,GAAY,IAAI,KAAhB,EAAE,KAAK,GAAK,IAAI,MAAT,CAAU;QAC7B,wBAAa,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CACP,oBAAW,CAAC,iBAAiB,EAC7B,UAAO,IAAsC;;;;;oBACnC,KAAK,GAAoB,IAAI,MAAxB,EAAE,aAAa,GAAK,IAAI,cAAT,CAAU;oBAChC,QAAQ,GAAK,IAAI,SAAT,CAAU;oBACxB,qBAAM,0CAAiC,CAAC,KAAK,EAAE,aAAa,CAAC,EAAA;;oBAA7D,SAA6D,CAAC;oBAC9D,OAAO,QAAQ,EAAE;wBACf,0DAA0D;wBAC1D,uDAAuD;wBACvD,6BAAkB,CAAC,KAAK,CAAC,CAAC;wBAC1B,QAAQ,IAAI,CAAC,CAAC;qBACf;;;;SACF,CACF,CAAC;IAEF,MAAM,CAAC,EAAE,CACP,oBAAW,CAAC,cAAc,EAC1B,UAAC,IAAmC;QAC1B,IAAA,KAAK,GAAoB,IAAI,MAAxB,EAAE,aAAa,GAAK,IAAI,cAAT,CAAU;QACtC,oGAAoG;QACpG,iCAAiC;QACjC,oCAAoC;QACpC,0CAAiC,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IAC1D,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,WAAW,EAAE,UAAC,IAAgC;QAC1D,IAAA,KAAK,GAAK,IAAI,MAAT,CAAU;QACvB,oBAAW,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,UAAU,EAAE,UAAC,IAA+B;QACxD,IAAA,KAAK,GAAK,IAAI,MAAT,CAAU;QACvB,kBAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACH,IAAM,IAAI,GAAG,IAAI,CAAC;AAElB,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,UAAC,GAAG,EAAE,GAAG;IAC5B,OAAA,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAAjD,CAAiD,CAClD,CAAC;AACF,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,UAAC,GAAG,EAAE,GAAG;IAC3B,OAAA,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;AAA9D,CAA8D,CAC/D,CAAC;AACF,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,UAAC,GAAG,EAAE,GAAG;IACpB,OAAA,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAArD,CAAqD,CACtD,CAAC;AAEF,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAM,OAAA,OAAO,CAAC,GAAG,CAAC,8BAA4B,IAAM,CAAC,EAA/C,CAA+C,CAAC,CAAC"} -------------------------------------------------------------------------------- /Utils/testbed/server.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | import * as path from 'path'; 3 | const app = express(); 4 | 5 | import { createServer } from 'http'; 6 | const server = createServer(app); 7 | const io = require('socket.io')(server); 8 | 9 | import { SocketNames, SocketMessages } from './types/socket'; 10 | 11 | import { 12 | produce, 13 | produceAtRate, 14 | ProduceRequest, 15 | ProduceResponse, 16 | produceToPartition, 17 | } from './producer/producer'; 18 | import { consume, consumeAtRate } from './consumer/consumer'; 19 | import { 20 | createTopicWithMultiplePartitions, 21 | createTopicWithMultipleReplicants, 22 | createTopic, 23 | } from './topics/topics'; 24 | 25 | io.on('connection', () => console.log('Socket server connected...')); 26 | io.on('connect', (socket: any) => { 27 | socket.on(SocketNames.produceNum, (data: SocketMessages.produceNum) => { 28 | // should run the produce function data.messagesCount amount of times 29 | let { messagesCount } = data; 30 | while (messagesCount) { 31 | // should send back the data on completion of each to show on the client side 32 | produce() 33 | .then((resp: ProduceResponse) => socket.emit('produceResponse', resp)) 34 | .catch(console.error); 35 | messagesCount--; 36 | } 37 | }); 38 | 39 | socket.on(SocketNames.produceRate, (data: SocketMessages.produceRate) => { 40 | produceAtRate(data.rate); 41 | }); 42 | 43 | socket.on(SocketNames.consumeRate, (data: SocketMessages.consumeRate) => { 44 | const { rate, topic } = data; 45 | consumeAtRate(socket, rate, topic); 46 | }); 47 | 48 | socket.on( 49 | SocketNames.overloadPartition, 50 | async (data: SocketMessages.overloadPartition) => { 51 | const { topic, numPartitions } = data; 52 | let { messages } = data; 53 | await createTopicWithMultiplePartitions(topic, numPartitions); 54 | while (messages) { 55 | // we can add in custom partitioning logic here, if needed 56 | // right now, this will overload partition 0 by default 57 | produceToPartition(topic); 58 | messages -= 1; 59 | } 60 | } 61 | ); 62 | 63 | socket.on( 64 | SocketNames.underReplicate, 65 | (data: SocketMessages.underReplicate) => { 66 | const { topic, numReplicants } = data; 67 | // any number of replicants greater than the number of brokers will cause an under-replication event 68 | // default number of brokers is 1 69 | // default number of replicants is 2 70 | createTopicWithMultipleReplicants(topic, numReplicants); 71 | } 72 | ); 73 | 74 | socket.on(SocketNames.createTopic, (data: SocketMessages.createTopic) => { 75 | const { topic } = data; 76 | createTopic(topic); 77 | }); 78 | socket.on(SocketNames.consumeAll, (data: SocketMessages.consumeAll) => { 79 | const { topic } = data; 80 | consume(socket, topic); 81 | }); 82 | }); 83 | const PORT = 2022; 84 | 85 | app.get('/main.css', (req, res) => 86 | res.sendFile(path.resolve(__dirname, 'main.css')) 87 | ); 88 | app.get('/main.js', (req, res) => 89 | res.sendFile(path.resolve(__dirname, '../test-front/main.js')) 90 | ); 91 | app.get('/', (req, res) => 92 | res.sendFile(path.resolve(__dirname, './index.html')) 93 | ); 94 | 95 | server.listen(PORT, () => console.log(`Server listening on port ${PORT}`)); 96 | -------------------------------------------------------------------------------- /Utils/testbed/topics/topics.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | exports.createTopicWithMultipleReplicants = exports.createTopicWithMultiplePartitions = exports.createTopic = void 0; 40 | var kafkajs_1 = require("kafkajs"); 41 | var kafka = new kafkajs_1.Kafka({ 42 | clientId: 'test-client', 43 | brokers: ['kafka:9092'] 44 | }); 45 | var admin = kafka.admin(); 46 | var connect = function () { return __awaiter(void 0, void 0, void 0, function () { 47 | return __generator(this, function (_a) { 48 | switch (_a.label) { 49 | case 0: return [4 /*yield*/, admin.connect()]; 50 | case 1: 51 | _a.sent(); 52 | console.log('Admin client connected.'); 53 | return [2 /*return*/]; 54 | } 55 | }); 56 | }); }; 57 | connect(); 58 | exports.createTopic = function (topic, numPartitions, replicationFactor) { 59 | if (topic === void 0) { topic = 'test-topic'; } 60 | if (numPartitions === void 0) { numPartitions = 1; } 61 | if (replicationFactor === void 0) { replicationFactor = 1; } 62 | return __awaiter(void 0, void 0, void 0, function () { 63 | var topicConfig, newTopicCreated, err_1; 64 | return __generator(this, function (_a) { 65 | switch (_a.label) { 66 | case 0: 67 | topicConfig = { 68 | topic: topic, 69 | numPartitions: numPartitions, 70 | replicationFactor: replicationFactor 71 | }; 72 | _a.label = 1; 73 | case 1: 74 | _a.trys.push([1, 3, , 4]); 75 | return [4 /*yield*/, admin.createTopics({ 76 | topics: [topicConfig] 77 | })]; 78 | case 2: 79 | newTopicCreated = _a.sent(); 80 | console.log(newTopicCreated 81 | ? "Topic with name " + topic + " was created." 82 | : "Topic with name " + topic + " already exists."); 83 | return [3 /*break*/, 4]; 84 | case 3: 85 | err_1 = _a.sent(); 86 | console.error(err_1); 87 | return [3 /*break*/, 4]; 88 | case 4: return [2 /*return*/]; 89 | } 90 | }); 91 | }); 92 | }; 93 | // Create a topic with multiple partitions 94 | exports.createTopicWithMultiplePartitions = function (topic, numPartitions) { 95 | return exports.createTopic(topic, numPartitions); 96 | }; 97 | // Create a topic with multiple replicants 98 | exports.createTopicWithMultipleReplicants = function (topic, numReplicants) { 99 | if (numReplicants === void 0) { numReplicants = 2; } 100 | return exports.createTopic(topic, undefined, numReplicants); 101 | }; 102 | //# sourceMappingURL=topics.js.map -------------------------------------------------------------------------------- /Utils/testbed/topics/topics.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"topics.js","sourceRoot":"","sources":["topics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAA8C;AAE9C,IAAM,KAAK,GAAG,IAAI,eAAK,CAAC;IACtB,QAAQ,EAAE,aAAa;IACvB,OAAO,EAAE,CAAC,YAAY,CAAC;CACxB,CAAC,CAAC;AAEH,IAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;AAE5B,IAAM,OAAO,GAAG;;;oBACd,qBAAM,KAAK,CAAC,OAAO,EAAE,EAAA;;gBAArB,SAAqB,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;;;;KACxC,CAAC;AAEF,OAAO,EAAE,CAAC;AAEG,QAAA,WAAW,GAAG,UACzB,KAA4B,EAC5B,aAAyB,EACzB,iBAA6B;IAF7B,sBAAA,EAAA,oBAA4B;IAC5B,8BAAA,EAAA,iBAAyB;IACzB,kCAAA,EAAA,qBAA6B;;;;;;oBAEvB,WAAW,GAAiB;wBAChC,KAAK,OAAA;wBACL,aAAa,eAAA;wBACb,iBAAiB,mBAAA;qBAClB,CAAC;;;;oBAEwB,qBAAM,KAAK,CAAC,YAAY,CAAC;4BAC/C,MAAM,EAAE,CAAC,WAAW,CAAC;yBACtB,CAAC,EAAA;;oBAFI,eAAe,GAAG,SAEtB;oBACF,OAAO,CAAC,GAAG,CACT,eAAe;wBACb,CAAC,CAAC,qBAAmB,KAAK,kBAAe;wBACzC,CAAC,CAAC,qBAAmB,KAAK,qBAAkB,CAC/C,CAAC;;;;oBAEF,OAAO,CAAC,KAAK,CAAC,KAAG,CAAC,CAAC;;;;;;CAEtB,CAAC;AAEF,0CAA0C;AAC7B,QAAA,iCAAiC,GAAG,UAAC,KAAa,EAAE,aAAqB;IACpF,OAAA,mBAAW,CAAC,KAAK,EAAE,aAAa,CAAC;AAAjC,CAAiC,CAAC;AAEpC,0CAA0C;AAC7B,QAAA,iCAAiC,GAAG,UAAC,KAAa,EAAE,aAAyB;IAAzB,8BAAA,EAAA,iBAAyB;IACxF,OAAA,mBAAW,CAAC,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC;AAA5C,CAA4C,CAAC"} -------------------------------------------------------------------------------- /Utils/testbed/topics/topics.ts: -------------------------------------------------------------------------------- 1 | import { Kafka, ITopicConfig } from 'kafkajs'; 2 | 3 | const kafka = new Kafka({ 4 | clientId: 'test-client', 5 | brokers: ['kafka:9092'], 6 | }); 7 | 8 | const admin = kafka.admin(); 9 | 10 | const connect = async () => { 11 | await admin.connect(); 12 | console.log('Admin client connected.'); 13 | }; 14 | 15 | connect(); 16 | 17 | export const createTopic = async ( 18 | topic: string = 'test-topic', 19 | numPartitions: number = 1, 20 | replicationFactor: number = 1 21 | ): Promise => { 22 | const topicConfig: ITopicConfig = { 23 | topic, 24 | numPartitions, 25 | replicationFactor, 26 | }; 27 | try { 28 | const newTopicCreated = await admin.createTopics({ 29 | topics: [topicConfig], 30 | }); 31 | console.log( 32 | newTopicCreated 33 | ? `Topic with name ${topic} was created.` 34 | : `Topic with name ${topic} already exists.` 35 | ); 36 | } catch (err) { 37 | console.error(err); 38 | } 39 | }; 40 | 41 | // Create a topic with multiple partitions 42 | export const createTopicWithMultiplePartitions = (topic: string, numPartitions: number) => 43 | createTopic(topic, numPartitions); 44 | 45 | // Create a topic with multiple replicants 46 | export const createTopicWithMultipleReplicants = (topic: string, numReplicants: number = 2) => 47 | createTopic(topic, undefined, numReplicants); 48 | -------------------------------------------------------------------------------- /Utils/testbed/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": true, 4 | "sourceMap": true, 5 | "module": "commonjs" 6 | }, 7 | "include": ["*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /Utils/testbed/types/socket.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | exports.__esModule = true; 3 | exports.SocketNames = void 0; 4 | exports.SocketNames = { 5 | produceNum: 'produceNum', 6 | produceRate: 'produceRate', 7 | consumeRate: 'consumeRate', 8 | consumeAll: 'consumeAll', 9 | overloadPartition: 'overloadPartition', 10 | createTopic: 'createTopic', 11 | underReplicate: 'underReplicate' 12 | }; 13 | //# sourceMappingURL=socket.js.map -------------------------------------------------------------------------------- /Utils/testbed/types/socket.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"socket.js","sourceRoot":"","sources":["socket.ts"],"names":[],"mappings":";;;AASa,QAAA,WAAW,GAAgB;IACtC,UAAU,EAAE,YAAY;IACxB,WAAW,EAAE,aAAa;IAC1B,WAAW,EAAE,aAAa;IAC1B,UAAU,EAAE,YAAY;IACxB,iBAAiB,EAAE,mBAAmB;IACtC,WAAW,EAAE,aAAa;IAC1B,cAAc,EAAE,gBAAgB;CACjC,CAAC"} -------------------------------------------------------------------------------- /Utils/testbed/types/socket.ts: -------------------------------------------------------------------------------- 1 | interface SocketNames { 2 | produceNum: 'produceNum'; 3 | produceRate: 'produceRate'; 4 | consumeRate: 'consumeRate'; 5 | consumeAll: 'consumeAll'; 6 | overloadPartition: 'overloadPartition'; 7 | createTopic: 'createTopic'; 8 | underReplicate: 'underReplicate'; 9 | } 10 | export const SocketNames: SocketNames = { 11 | produceNum: 'produceNum', 12 | produceRate: 'produceRate', 13 | consumeRate: 'consumeRate', 14 | consumeAll: 'consumeAll', 15 | overloadPartition: 'overloadPartition', 16 | createTopic: 'createTopic', 17 | underReplicate: 'underReplicate', 18 | }; 19 | 20 | export namespace SocketMessages { 21 | export type produceNum = { messagesCount: number }; 22 | 23 | export type produceRate = { 24 | rate: number; 25 | }; 26 | export type consumeRate = { 27 | rate: number; 28 | topic: string; 29 | }; 30 | 31 | export type overloadPartition = { 32 | topic: string; 33 | numPartitions: number; 34 | messages: number; 35 | }; 36 | 37 | export type createTopic = { 38 | topic: string; 39 | }; 40 | 41 | export type consumeAll = { 42 | topic: string; 43 | }; 44 | export type underReplicate = { 45 | topic: string; 46 | numReplicants: number; 47 | }; 48 | } 49 | -------------------------------------------------------------------------------- /assets/consumerConsumptionRate.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/assets/consumerConsumptionRate.gif -------------------------------------------------------------------------------- /assets/consumerLag.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/assets/consumerLag.gif -------------------------------------------------------------------------------- /assets/screenRecording.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/assets/screenRecording.mov -------------------------------------------------------------------------------- /assets/testbedConnection.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/assets/testbedConnection.gif -------------------------------------------------------------------------------- /assets/testbedConsumption.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/assets/testbedConsumption.gif -------------------------------------------------------------------------------- /assets/testbedProduction.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/assets/testbedProduction.gif -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | zookeeper: 5 | image: wurstmeister/zookeeper 6 | ports: 7 | - '2181:2181' 8 | kafka: 9 | image: wurstmeister/kafka:latest 10 | ports: 11 | - '9092:9092' 12 | environment: 13 | KAFKA_ADVERTISED_HOST_NAME: kafka 14 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 15 | volumes: 16 | - /var/run/docker.sock:/var/run/docker.sock 17 | depends_on: 18 | - zookeeper 19 | 20 | kafka_exporter: 21 | image: danielqsj/kafka-exporter 22 | ports: 23 | - '9308:9308' 24 | depends_on: 25 | - kafka 26 | restart: always 27 | 28 | prometheus: 29 | image: prom/prometheus 30 | ports: 31 | - 9090:9090 32 | volumes: 33 | - $PWD/Utils/scripts/prometheus.yml:/etc/prometheus/prometheus.yml 34 | depends_on: 35 | - kafka_exporter 36 | - alertmanager 37 | restart: always 38 | 39 | alertmanager: 40 | image: prom/alertmanager 41 | ports: 42 | - 9093:9093 43 | depends_on: 44 | - kafka_exporter 45 | 46 | grafana: 47 | image: grafana/grafana 48 | ports: 49 | - '3000:3000' 50 | volumes: 51 | - $PWD/plugin:/var/lib/grafana/plugins 52 | depends_on: 53 | - prometheus 54 | volumes: 55 | - $PWD/plugin:/var/lib/grafana/plugins 56 | 57 | kafka-testbed: 58 | image: svansig/kafka-testbed 59 | ports: 60 | - '2022:2022' 61 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/index.html -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Build the consumer-lag plugin 4 | cd ./plugin/consumer-lag/ && npm i && npm run build && cd ../../ 5 | 6 | # Build the consumer_message plugin 7 | cd ./plugin/consumer-message-rate/ && npm i && npm run build && cd ../../ 8 | 9 | # Build the control-panel plugin 10 | cd ./plugin/control-panel/ && npm i && npm run build && cd ../../ 11 | 12 | # Start the docker instance 13 | docker-compose up 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MASH_testbed", 3 | "version": "1.0.0", 4 | "description": "A kafka testbed for MASH", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "jest --verbose", 8 | "testbed": "tsc --project Utils/testbed/tsconfig.json && node ./Utils/testbed/server.js", 9 | "build-testbed": "webpack test-front/main.ts -o test-front/main.js" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "dependencies": { 15 | "@types/express": "^4.17.7", 16 | "@types/jest": "^26.0.8", 17 | "@types/kafkajs": "^1.9.0", 18 | "@types/puppeteer": "^3.0.1", 19 | "@types/socket.io": "^2.1.9", 20 | "@types/socket.io-client": "^1.4.33", 21 | "express": "^4.17.1", 22 | "jest": "^26.2.1", 23 | "jest-puppeteer": "^4.4.0", 24 | "kafkajs": "^1.12.0", 25 | "puppeteer": "^5.2.1", 26 | "socket.io": "^2.3.0", 27 | "socket.io-client": "^2.3.0", 28 | "ts-loader": "^8.0.1", 29 | "typescript": "^3.9.7", 30 | "webpack": "^4.43.0", 31 | "webpack-cli": "^3.3.12", 32 | "webpack-dev-server": "^3.11.0" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/consumer-lag/.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | parameters: 4 | ssh-fingerprint: 5 | type: string 6 | default: ${GITHUB_SSH_FINGERPRINT} 7 | 8 | aliases: 9 | # Workflow filters 10 | - &filter-only-master 11 | branches: 12 | only: master 13 | - &filter-only-release 14 | branches: 15 | only: /^v[1-9]*[0-9]+\.[1-9]*[0-9]+\.x$/ 16 | 17 | workflows: 18 | plugin_workflow: 19 | jobs: 20 | - build 21 | 22 | executors: 23 | default_exec: # declares a reusable executor 24 | docker: 25 | - image: srclosson/grafana-plugin-ci-alpine:latest 26 | e2e_exec: 27 | docker: 28 | - image: srclosson/grafana-plugin-ci-e2e:latest 29 | 30 | jobs: 31 | build: 32 | executor: default_exec 33 | steps: 34 | - checkout 35 | - restore_cache: 36 | name: restore node_modules 37 | keys: 38 | - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }} 39 | - run: 40 | name: Install dependencies 41 | command: | 42 | mkdir ci 43 | [ -f ~/project/node_modules/.bin/grafana-toolkit ] || yarn install --frozen-lockfile 44 | - save_cache: 45 | name: save node_modules 46 | paths: 47 | - ~/project/node_modules 48 | key: build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }} 49 | - run: 50 | name: Build and test frontend 51 | command: ./node_modules/.bin/grafana-toolkit plugin:ci-build 52 | - run: 53 | name: Move results to ci folder 54 | command: ./node_modules/.bin/grafana-toolkit plugin:ci-build --finish 55 | - run: 56 | name: Package distribution 57 | command: | 58 | ./node_modules/.bin/grafana-toolkit plugin:ci-package 59 | - persist_to_workspace: 60 | root: . 61 | paths: 62 | - ci/jobs/package 63 | - ci/packages 64 | - ci/dist 65 | - ci/grafana-test-env 66 | - store_artifacts: 67 | path: ci 68 | -------------------------------------------------------------------------------- /plugin/consumer-lag/.codeclimate.yml: -------------------------------------------------------------------------------- 1 | exclude_patterns: 2 | - "dist" 3 | - "**/node_modules/" 4 | - "**/*.d.ts" 5 | - "src/libs/" 6 | - "src/images/" 7 | - "src/img/" 8 | - "src/screenshots/" 9 | -------------------------------------------------------------------------------- /plugin/consumer-lag/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | max_line_length = 120 11 | 12 | [*.{js,ts,tsx,scss}] 13 | quote_type = single 14 | 15 | [*.md] 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /plugin/consumer-lag/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | node_modules/ 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | *.pid.lock 15 | 16 | # Directory for instrumented libs generated by jscoverage/JSCover 17 | lib-cov 18 | 19 | # Coverage directory used by tools like istanbul 20 | coverage 21 | 22 | # Compiled binary addons (https://nodejs.org/api/addons.html) 23 | dist/ 24 | artifacts/ 25 | work/ 26 | ci/ 27 | e2e-results/ 28 | 29 | # Editors 30 | .idea 31 | 32 | -------------------------------------------------------------------------------- /plugin/consumer-lag/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require("./node_modules/@grafana/toolkit/src/config/prettier.plugin.config.json"), 3 | }; 4 | -------------------------------------------------------------------------------- /plugin/consumer-lag/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## v1.0.0 6 | 7 | Initial Release 8 | -------------------------------------------------------------------------------- /plugin/consumer-lag/README.md: -------------------------------------------------------------------------------- 1 | # Grafana Panel Plugin Template 2 | 3 | 10 | 11 | This template is a starting point for building Grafana Panel Plugins in Grafana 7.0+ 12 | 13 | 14 | ## What is Grafana Panel Plugin? 15 | Panels are the building blocks of Grafana. They allow you to visualize data in different ways. While Grafana has several types of panels already built-in, you can also build your own panel, to add support for other visualizations. 16 | 17 | For more information about panels, refer to the documentation on [Panels](https://grafana.com/docs/grafana/latest/features/panels/panels/) 18 | 19 | ## Getting started 20 | 1. Install dependencies 21 | ```BASH 22 | yarn install 23 | ``` 24 | 2. Build plugin in development mode or run in watch mode 25 | ```BASH 26 | yarn dev 27 | ``` 28 | or 29 | ```BASH 30 | yarn watch 31 | ``` 32 | 3. Build plugin in production mode 33 | ```BASH 34 | yarn build 35 | ``` 36 | 37 | ## Learn more 38 | - [Build a panel plugin tutorial](https://grafana.com/tutorials/build-a-panel-plugin) 39 | - [Grafana documentation](https://grafana.com/docs/) 40 | - [Grafana Tutorials](https://grafana.com/tutorials/) - Grafana Tutorials are step-by-step guides that help you make the most of Grafana 41 | - [Grafana UI Library](https://developers.grafana.com/ui) - UI components to help you build interfaces using Grafana Design System 42 | -------------------------------------------------------------------------------- /plugin/consumer-lag/appveyor.yml: -------------------------------------------------------------------------------- 1 | # Test against the latest version of this Node.js version 2 | environment: 3 | nodejs_version: "12" 4 | 5 | # Local NPM Modules 6 | cache: 7 | - node_modules 8 | 9 | # Install scripts. (runs after repo cloning) 10 | install: 11 | # Get the latest stable version of Node.js or io.js 12 | - ps: Install-Product node $env:nodejs_version 13 | # install modules 14 | - npm install -g yarn --quiet 15 | - yarn install --pure-lockfile 16 | 17 | # Post-install test scripts. 18 | test_script: 19 | # Output useful info for debugging. 20 | - node --version 21 | - npm --version 22 | 23 | # Run the build 24 | build_script: 25 | - yarn dev # This will also run prettier! 26 | - yarn build # make sure both scripts work 27 | -------------------------------------------------------------------------------- /plugin/consumer-lag/docs/README.md: -------------------------------------------------------------------------------- 1 | TODO: add example docs structure 2 | -------------------------------------------------------------------------------- /plugin/consumer-lag/jest.config.js: -------------------------------------------------------------------------------- 1 | // This file is needed because it is used by vscode and other tools that 2 | // call `jest` directly. However, unless you are doing anything special 3 | // do not edit this file 4 | 5 | const standard = require('@grafana/toolkit/src/config/jest.plugin.config'); 6 | 7 | // This process will use the same config that `yarn test` is using 8 | module.exports = standard.jestConfig(); 9 | -------------------------------------------------------------------------------- /plugin/consumer-lag/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "envoya-consumer-lag", 3 | "version": "1.0.0", 4 | "description": "shows consumer lag for a kafka instance", 5 | "scripts": { 6 | "build": "grafana-toolkit plugin:build", 7 | "test": "grafana-toolkit plugin:test", 8 | "dev": "grafana-toolkit plugin:dev", 9 | "watch": "grafana-toolkit plugin:dev --watch" 10 | }, 11 | "repository": "github:grafana/simple-react-panel", 12 | "author": "Steve Budarz", 13 | "license": "Apache-2.0", 14 | "bugs": { 15 | "url": "https://github.com/grafana/simple-react-panel/issues", 16 | "email": "plugins@grafana.com" 17 | }, 18 | "devDependencies": { 19 | "@grafana/data": "7.0.1", 20 | "@grafana/toolkit": "7.0.1", 21 | "@grafana/ui": "7.0.1", 22 | "emotion": "10.0.27" 23 | }, 24 | "engines": { 25 | "node": ">=12 <13" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/Components/Consumer.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import { ConsumerData } from '../SimplePanel'; 3 | 4 | interface ConsumerBarProps { 5 | consumer: ConsumerData; 6 | topicXOffset: number; 7 | childIndex: number; 8 | childWidth: number; 9 | childHeight: number; 10 | } 11 | 12 | export const ConsumerBar = (props: ConsumerBarProps) => { 13 | const { childIndex, childWidth, topicXOffset, consumer, childHeight } = props; 14 | const [isHovered, setIsHovered] = useState(false); 15 | 16 | return ( 17 | 18 | setIsHovered(true)} 20 | onMouseLeave={() => setIsHovered(false)} 21 | x={topicXOffset + 2 + childIndex * childWidth} 22 | y={25} 23 | width={childWidth - 10 - 6} 24 | height={childHeight} 25 | fill="#0042ff" 26 | filter="url(#f3)" 27 | /> 28 | {isHovered && ( 29 | 41 | )} 42 | 43 | 51 | {isHovered ? consumer.offset : null} 52 | 53 | 54 | ); 55 | }; 56 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/Components/Topic.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { TopicData } from '../SimplePanel'; 3 | import { ConsumerBar } from './Consumer'; 4 | 5 | interface TopicProps { 6 | topic: TopicData; 7 | width: number; 8 | totalNumOfTopics: number; 9 | index: number; 10 | height: number; 11 | } 12 | 13 | export const TopicBar = (props: TopicProps) => { 14 | const { totalNumOfTopics, width, index, topic, height } = props; 15 | const topicWidth = width / totalNumOfTopics; 16 | const topicXOffset = topicWidth * index; 17 | const childWidth = topicWidth / topic.children.length; 18 | const scale = (messagesOffset: number, topicOffset: number): number => 19 | ((messagesOffset + 1) / (topicOffset + 1)) * height; 20 | return ( 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | {topic.children.map((child, j) => ( 31 | 38 | ))} 39 | 40 | ); 41 | }; 42 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/SimplePanel.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { PanelProps, PanelData, DataFrame } from '@grafana/data'; 3 | import { SimpleOptions } from 'types'; 4 | import { css, cx } from 'emotion'; 5 | import { stylesFactory, useTheme } from '@grafana/ui'; 6 | import { TopicBar } from './Components/Topic'; 7 | 8 | interface Props extends PanelProps {} 9 | 10 | export interface ConsumerData { 11 | topic: string; 12 | offset: number; 13 | consumergroup: string; 14 | } 15 | 16 | export interface TopicData { 17 | topic: string; 18 | partition: number; 19 | children: ConsumerData[]; 20 | offset: number; 21 | } 22 | 23 | const getLastElement = (arr: number[]) => arr[arr.length - 1]; 24 | 25 | /* for each producer/topic, create a bar with the length of the producer offset: 26 | filter by name.includes('kafka_topic_partition_current_offset') 27 | create an object with the relevant topic, add data from partition (partition, and topic_offset) 28 | */ 29 | 30 | const filterTopic = (data: PanelData) => { 31 | return data.series 32 | .filter(series => series.name?.includes('kafka_topic_partition_current_offset')) 33 | .filter(series => (series.fields[1].labels ? series.fields[1].labels.topic !== '__consumer_offsets' : false)); 34 | }; 35 | 36 | const getTopicData = (series: DataFrame[]): TopicData[] => { 37 | return series.map(s => { 38 | const { topic, partition } = s.fields[1].labels; 39 | const children: ConsumerData[] = []; 40 | const offset = getLastElement(s.fields[1].values.toArray()); 41 | return { 42 | topic, 43 | partition, 44 | children, 45 | offset, 46 | }; 47 | }); 48 | }; 49 | 50 | /* for each consumer/topic, create a bar on top of the producer bar with the length of the consumer offset: 51 | filter by name.includes('kafka_consumergroup_current_offset') 52 | find the partition object with the matching topic and partition 53 | add to "children" array consumer group name and offset 54 | */ 55 | 56 | const filterConsumers = (data: PanelData) => { 57 | return data.series.filter(series => series.name?.includes('kafka_consumergroup_current_offset')); 58 | }; 59 | 60 | const getConsumerData = (series: DataFrame[]): ConsumerData[] => { 61 | return series.map(s => { 62 | const { topic, consumergroup } = s.fields[1].labels; 63 | const offset = getLastElement(s.fields[1].values.toArray()); 64 | return { 65 | topic, 66 | offset, 67 | consumergroup, 68 | }; 69 | }); 70 | }; 71 | 72 | const attachConsumerToTopic = (topic: TopicData[], consumer: ConsumerData[]) => { 73 | consumer.forEach(c => { 74 | const selected = topic.filter(v => v.topic === c.topic)[0]; 75 | console.log('consumer for selection: ', c); 76 | console.log('topic for selection: ', topic); 77 | console.log('selected: ', selected); 78 | selected.children.push(c); 79 | }); 80 | }; 81 | 82 | export const SimplePanel: React.FC = ({ options, data, width, height }) => { 83 | const theme = useTheme(); 84 | const styles = getStyles(); 85 | 86 | console.log(data); 87 | 88 | let topics = filterTopic(data); 89 | let topicsData = getTopicData(topics); 90 | let consumers = filterConsumers(data); 91 | let consumersData = getConsumerData(consumers); 92 | attachConsumerToTopic(topicsData, consumersData); 93 | console.log('topics data:', topicsData); 94 | 95 | /* 96 | Add the values for the total consumer offset by topic / partition 97 | Should be vsible on hover 98 | 99 | */ 100 | 101 | return ( 102 | // {/* for each consumer/topic, on hover / click, expand the bar and add data*/} 103 | // {topicsData 104 | // .map( 105 | // topic => 106 | // topic.topic + 107 | // ' - ' + 108 | // topic.offset + 109 | // // const scale = d3 110 | // // .scaleLinear() 111 | // // .domain([0, topic.offset]) 112 | // // .range([0, width]); 113 | // ' : ' + 114 | // topic.children.map(child => child.consumergroup + ' - ' + child.offset).join(', ') 115 | // ) 116 | // .map(s => ( 117 | //
{s}
118 | // ))} 119 | // 120 | 121 | 122 | 123 | {topicsData.map((topic, i) => { 124 | return ( 125 | 126 | ); 127 | })} 128 | 129 | 130 | ); 131 | }; 132 | 133 | const getStyles = stylesFactory(() => { 134 | return { 135 | wrapper: css` 136 | position: relative; 137 | `, 138 | svg: css` 139 | position: absolute; 140 | top: 0; 141 | left: 0; 142 | `, 143 | textBox: css` 144 | position: absolute; 145 | bottom: 0; 146 | left: 0; 147 | padding: 10px; 148 | `, 149 | }; 150 | }); 151 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/img/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/module.test.ts: -------------------------------------------------------------------------------- 1 | // Just a stub test 2 | describe('placeholder test', () => { 3 | it('should return true', () => { 4 | expect(true).toBeTruthy(); 5 | }); 6 | }); 7 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/module.ts: -------------------------------------------------------------------------------- 1 | import { PanelPlugin } from '@grafana/data'; 2 | import { SimpleOptions } from './types'; 3 | import { SimplePanel } from './SimplePanel'; 4 | 5 | export const plugin = new PanelPlugin(SimplePanel).setPanelOptions(builder => { 6 | return builder 7 | .addTextInput({ 8 | path: 'text', 9 | name: 'Simple text option', 10 | description: 'Description of panel option', 11 | defaultValue: 'Default value of text input option', 12 | }) 13 | .addBooleanSwitch({ 14 | path: 'showSeriesCount', 15 | name: 'Show series counter', 16 | defaultValue: false, 17 | }) 18 | .addRadio({ 19 | path: 'seriesCountSize', 20 | defaultValue: 'sm', 21 | name: 'Series counter size', 22 | settings: { 23 | options: [ 24 | { 25 | value: 'sm', 26 | label: 'Small', 27 | }, 28 | { 29 | value: 'md', 30 | label: 'Medium', 31 | }, 32 | { 33 | value: 'lg', 34 | label: 'Large', 35 | }, 36 | ], 37 | }, 38 | showIf: config => config.showSeriesCount, 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "panel", 3 | "name": "consumer_lag", 4 | "id": "envoya-consumer-lag", 5 | "info": { 6 | "description": "shows consumer lag for a kafka instance", 7 | "author": { 8 | "name": "Steve Budarz", 9 | "url": "" 10 | }, 11 | "keywords": ["kafka"], 12 | "logos": { 13 | "small": "img/logo.svg", 14 | "large": "img/logo.svg" 15 | }, 16 | "links": [ 17 | { 18 | "name": "Website", 19 | "url": "https://github.com/grafana/simple-react-panel" 20 | }, 21 | { 22 | "name": "License", 23 | "url": "https://github.com/grafana/simple-react-panel/blob/master/LICENSE" 24 | } 25 | ], 26 | "screenshots": [], 27 | "version": "0.1.0", 28 | "updated": "08-04-20" 29 | }, 30 | "dependencies": { 31 | "grafanaVersion": "6.3.x", 32 | "plugins": [] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/consumer-lag/src/types.ts: -------------------------------------------------------------------------------- 1 | type SeriesSize = 'sm' | 'md' | 'lg'; 2 | 3 | export interface SimpleOptions { 4 | text: string; 5 | showSeriesCount: boolean; 6 | seriesCountSize: SeriesSize; 7 | } 8 | -------------------------------------------------------------------------------- /plugin/consumer-lag/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/@grafana/toolkit/src/config/tsconfig.plugin.json", 3 | "include": ["src", "types"], 4 | "compilerOptions": { 5 | "rootDir": "./src", 6 | "baseUrl": "./src", 7 | "typeRoots": ["./node_modules/@types"] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | parameters: 4 | ssh-fingerprint: 5 | type: string 6 | default: ${GITHUB_SSH_FINGERPRINT} 7 | 8 | aliases: 9 | # Workflow filters 10 | - &filter-only-master 11 | branches: 12 | only: master 13 | - &filter-only-release 14 | branches: 15 | only: /^v[1-9]*[0-9]+\.[1-9]*[0-9]+\.x$/ 16 | 17 | workflows: 18 | plugin_workflow: 19 | jobs: 20 | - build 21 | 22 | executors: 23 | default_exec: # declares a reusable executor 24 | docker: 25 | - image: srclosson/grafana-plugin-ci-alpine:latest 26 | e2e_exec: 27 | docker: 28 | - image: srclosson/grafana-plugin-ci-e2e:latest 29 | 30 | jobs: 31 | build: 32 | executor: default_exec 33 | steps: 34 | - checkout 35 | - restore_cache: 36 | name: restore node_modules 37 | keys: 38 | - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }} 39 | - run: 40 | name: Install dependencies 41 | command: | 42 | mkdir ci 43 | [ -f ~/project/node_modules/.bin/grafana-toolkit ] || yarn install --frozen-lockfile 44 | - save_cache: 45 | name: save node_modules 46 | paths: 47 | - ~/project/node_modules 48 | key: build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }} 49 | - run: 50 | name: Build and test frontend 51 | command: ./node_modules/.bin/grafana-toolkit plugin:ci-build 52 | - run: 53 | name: Move results to ci folder 54 | command: ./node_modules/.bin/grafana-toolkit plugin:ci-build --finish 55 | - run: 56 | name: Package distribution 57 | command: | 58 | ./node_modules/.bin/grafana-toolkit plugin:ci-package 59 | - persist_to_workspace: 60 | root: . 61 | paths: 62 | - ci/jobs/package 63 | - ci/packages 64 | - ci/dist 65 | - ci/grafana-test-env 66 | - store_artifacts: 67 | path: ci 68 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/.codeclimate.yml: -------------------------------------------------------------------------------- 1 | exclude_patterns: 2 | - "dist" 3 | - "**/node_modules/" 4 | - "**/*.d.ts" 5 | - "src/libs/" 6 | - "src/images/" 7 | - "src/img/" 8 | - "src/screenshots/" 9 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | max_line_length = 120 11 | 12 | [*.{js,ts,tsx,scss}] 13 | quote_type = single 14 | 15 | [*.md] 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | node_modules/ 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | *.pid.lock 15 | 16 | # Directory for instrumented libs generated by jscoverage/JSCover 17 | lib-cov 18 | 19 | # Coverage directory used by tools like istanbul 20 | coverage 21 | 22 | # Compiled binary addons (https://nodejs.org/api/addons.html) 23 | dist/ 24 | artifacts/ 25 | work/ 26 | ci/ 27 | e2e-results/ 28 | 29 | # Editors 30 | .idea 31 | 32 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require("./node_modules/@grafana/toolkit/src/config/prettier.plugin.config.json"), 3 | }; 4 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## v1.0.0 6 | 7 | Initial Release 8 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/README.md: -------------------------------------------------------------------------------- 1 | # Grafana Panel Plugin Template 2 | 3 | 10 | 11 | This template is a starting point for building Grafana Panel Plugins in Grafana 7.0+ 12 | 13 | 14 | ## What is Grafana Panel Plugin? 15 | Panels are the building blocks of Grafana. They allow you to visualize data in different ways. While Grafana has several types of panels already built-in, you can also build your own panel, to add support for other visualizations. 16 | 17 | For more information about panels, refer to the documentation on [Panels](https://grafana.com/docs/grafana/latest/features/panels/panels/) 18 | 19 | ## Getting started 20 | 1. Install dependencies 21 | ```BASH 22 | yarn install 23 | ``` 24 | 2. Build plugin in development mode or run in watch mode 25 | ```BASH 26 | yarn dev 27 | ``` 28 | or 29 | ```BASH 30 | yarn watch 31 | ``` 32 | 3. Build plugin in production mode 33 | ```BASH 34 | yarn build 35 | ``` 36 | 37 | ## Learn more 38 | - [Build a panel plugin tutorial](https://grafana.com/tutorials/build-a-panel-plugin) 39 | - [Grafana documentation](https://grafana.com/docs/) 40 | - [Grafana Tutorials](https://grafana.com/tutorials/) - Grafana Tutorials are step-by-step guides that help you make the most of Grafana 41 | - [Grafana UI Library](https://developers.grafana.com/ui) - UI components to help you build interfaces using Grafana Design System 42 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/appveyor.yml: -------------------------------------------------------------------------------- 1 | # Test against the latest version of this Node.js version 2 | environment: 3 | nodejs_version: "12" 4 | 5 | # Local NPM Modules 6 | cache: 7 | - node_modules 8 | 9 | # Install scripts. (runs after repo cloning) 10 | install: 11 | # Get the latest stable version of Node.js or io.js 12 | - ps: Install-Product node $env:nodejs_version 13 | # install modules 14 | - npm install -g yarn --quiet 15 | - yarn install --pure-lockfile 16 | 17 | # Post-install test scripts. 18 | test_script: 19 | # Output useful info for debugging. 20 | - node --version 21 | - npm --version 22 | 23 | # Run the build 24 | build_script: 25 | - yarn dev # This will also run prettier! 26 | - yarn build # make sure both scripts work 27 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/docs/README.md: -------------------------------------------------------------------------------- 1 | TODO: add example docs structure 2 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/jest.config.js: -------------------------------------------------------------------------------- 1 | // This file is needed because it is used by vscode and other tools that 2 | // call `jest` directly. However, unless you are doing anything special 3 | // do not edit this file 4 | 5 | const standard = require('@grafana/toolkit/src/config/jest.plugin.config'); 6 | 7 | // This process will use the same config that `yarn test` is using 8 | module.exports = standard.jestConfig(); 9 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "envoya-consumer-message-rate", 3 | "version": "1.0.0", 4 | "description": "visualization of the consumer message consumption rate", 5 | "scripts": { 6 | "build": "grafana-toolkit plugin:build", 7 | "test": "grafana-toolkit plugin:test", 8 | "dev": "grafana-toolkit plugin:dev", 9 | "watch": "grafana-toolkit plugin:dev --watch" 10 | }, 11 | "repository": "github:grafana/simple-react-panel", 12 | "author": "Steve Budarz", 13 | "license": "Apache-2.0", 14 | "bugs": { 15 | "url": "https://github.com/grafana/simple-react-panel/issues", 16 | "email": "plugins@grafana.com" 17 | }, 18 | "devDependencies": { 19 | "@grafana/data": "7.0.1", 20 | "@grafana/toolkit": "7.0.1", 21 | "@grafana/ui": "7.0.1", 22 | "emotion": "10.0.27" 23 | }, 24 | "engines": { 25 | "node": ">=12 <13" 26 | }, 27 | "dependencies": { 28 | "@types/stats-lite": "^2.2.0", 29 | "stats-lite": "^2.2.0" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/src/SimplePanel.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { PanelProps } from '@grafana/data'; 3 | import { SimpleOptions } from 'types'; 4 | import { css, cx } from 'emotion'; 5 | import { stylesFactory, useTheme } from '@grafana/ui'; 6 | 7 | import * as stats from 'stats-lite'; 8 | 9 | // what do we need? 10 | // 1. we need the messaging rate per consumer 11 | // 2. we need the highest message rate per (per 30 seconds) 12 | // 3. we need the highest 25% message rate per (per 30 seconds) 13 | // 4. we need the lowest 25% message rate per (per 30 seconds) 14 | // 5. we need the lowest message rate per (per 30 seconds) 15 | 16 | interface MessagingRateQuarters { 17 | top: number; 18 | topMid: number; 19 | bottomMid: number; 20 | bottom: number; 21 | stdev: number; 22 | names: string; 23 | } 24 | 25 | // takes in an array of consumer offsets, spaced at 30 seconds apart 26 | const getMessagingRate = (array: number[]): number[] => { 27 | const rateResponse: number[] = [1]; 28 | for (let i = 1; i < array.length; i++) { 29 | rateResponse[i] = array[i] - array[i - 1]; // 10 - 300 = 241 // [555, 990, 1000] 30 | } 31 | return rateResponse; 32 | }; 33 | 34 | const getQuarters = (array: number[], names: string): MessagingRateQuarters => { 35 | array = array.sort((a, b) => a - b).filter(x => x > 1); 36 | // console.log(names); 37 | // console.log(array); 38 | let bottomMid; 39 | let topMid; 40 | if (array.length % 2 !== 0) { 41 | topMid = Math.floor(array[array.length / 2 + 1]); 42 | bottomMid = Math.floor(array[array.length / 2]); 43 | } else { 44 | topMid = array[array.length / 2]; 45 | bottomMid = array[array.length / 2 - 1]; 46 | } 47 | 48 | const response: MessagingRateQuarters = { 49 | top: array[array.length - 1] || 0, 50 | topMid: topMid || 0, 51 | bottomMid: bottomMid || 0, 52 | bottom: array[0] || 0, 53 | stdev: Math.floor(stats.stdev(array)), 54 | names: names, 55 | }; 56 | return response; 57 | }; 58 | 59 | const getColor = (index: number, length: number) => { 60 | let color = parseInt('FFFFFF', 16); 61 | color = Math.floor((color / length) * index); 62 | return `#${color.toString(16)}`; 63 | }; 64 | interface Props extends PanelProps {} 65 | 66 | export const SimplePanel: React.FC = ({ options, data, width, height }) => { 67 | const theme = useTheme(); 68 | const styles = getStyles(); 69 | // console.log(data.series[3].name); 70 | let xOffset = 0; 71 | let maxHeight = 0; 72 | 73 | return ( 74 |
75 | 84 | {data.series 85 | .map(series => { 86 | let tNmaes = series.name?.split(' '); 87 | // console.log(tNmaes[5].substring(7).slice(0, -2)); 88 | let finalNames = tNmaes[5].substring(7).slice(0, -2); 89 | // console.log(finalNames); 90 | 91 | return getQuarters( 92 | getMessagingRate(series.fields[1].values.toArray().filter(x => x < 30000)).reverse(), 93 | finalNames 94 | ); 95 | }) 96 | .map(group => { 97 | // console.log(group); 98 | maxHeight = Math.max(group.top, maxHeight); 99 | return group; 100 | }) 101 | .map((consumergroup, i) => { 102 | // console.log(consumergroup); 103 | xOffset = Math.floor(width / data.series.length) * i; 104 | return ( 105 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | {/* 128 | 129 | 130 | 131 | */} 132 | 143 | 156 | 169 | 179 | 185 |
186 |

{`TPName: ${consumergroup.names}`}

187 |

{`BOTT ${consumergroup.bottom}`}

188 |

{`BMD ${consumergroup.bottomMid}`}

189 |

{`TMD ${consumergroup.topMid}`}

190 |

{`TOP ${consumergroup.top}`}

191 |

{`STDEV ${consumergroup.stdev}`}

192 |
193 | 194 | 195 | 196 | ); 197 | })} 198 | 199 |
200 | ); 201 | }; 202 | 203 | const getStyles = stylesFactory(() => { 204 | return { 205 | wrapper: css` 206 | position: relative; 207 | `, 208 | svg: css` 209 | position: relative; 210 | top: 0; 211 | left: 0; 212 | `, 213 | textBox: css` 214 | position: absolute; 215 | bottom: 0; 216 | left: 0; 217 | padding: 10px; 218 | `, 219 | }; 220 | }); 221 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/src/img/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/src/module.test.ts: -------------------------------------------------------------------------------- 1 | // Just a stub test 2 | describe('placeholder test', () => { 3 | it('should return true', () => { 4 | expect(true).toBeTruthy(); 5 | }); 6 | }); 7 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/src/module.ts: -------------------------------------------------------------------------------- 1 | import { PanelPlugin } from '@grafana/data'; 2 | import { SimpleOptions } from './types'; 3 | import { SimplePanel } from './SimplePanel'; 4 | 5 | export const plugin = new PanelPlugin(SimplePanel).setPanelOptions(builder => { 6 | return builder 7 | .addTextInput({ 8 | path: 'text', 9 | name: 'Simple text option', 10 | description: 'Description of panel option', 11 | defaultValue: 'Default value of text input option', 12 | }) 13 | .addBooleanSwitch({ 14 | path: 'showSeriesCount', 15 | name: 'Show series counter', 16 | defaultValue: false, 17 | }) 18 | .addRadio({ 19 | path: 'seriesCountSize', 20 | defaultValue: 'sm', 21 | name: 'Series counter size', 22 | settings: { 23 | options: [ 24 | { 25 | value: 'sm', 26 | label: 'Small', 27 | }, 28 | { 29 | value: 'md', 30 | label: 'Medium', 31 | }, 32 | { 33 | value: 'lg', 34 | label: 'Large', 35 | }, 36 | ], 37 | }, 38 | showIf: config => config.showSeriesCount, 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/src/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "panel", 3 | "name": "consumer-message-rate", 4 | "id": "envoya-consumer-message-rate", 5 | "info": { 6 | "description": "visualization of the consumer message consumption rate", 7 | "author": { 8 | "name": "Steve Budarz", 9 | "url": "" 10 | }, 11 | "keywords": ["kafka"], 12 | "logos": { 13 | "small": "img/logo.svg", 14 | "large": "img/logo.svg" 15 | }, 16 | "links": [ 17 | { 18 | "name": "Website", 19 | "url": "https://github.com/grafana/simple-react-panel" 20 | }, 21 | { 22 | "name": "License", 23 | "url": "https://github.com/grafana/simple-react-panel/blob/master/LICENSE" 24 | } 25 | ], 26 | "screenshots": [], 27 | "version": "0.1.0", 28 | "updated": "08-04-20" 29 | }, 30 | "dependencies": { 31 | "grafanaVersion": "6.3.x", 32 | "plugins": [] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/src/types.ts: -------------------------------------------------------------------------------- 1 | type SeriesSize = 'sm' | 'md' | 'lg'; 2 | 3 | export interface SimpleOptions { 4 | text: string; 5 | showSeriesCount: boolean; 6 | seriesCountSize: SeriesSize; 7 | } 8 | -------------------------------------------------------------------------------- /plugin/consumer-message-rate/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/@grafana/toolkit/src/config/tsconfig.plugin.json", 3 | "include": ["src", "types"], 4 | "compilerOptions": { 5 | "rootDir": "./src", 6 | "baseUrl": "./src", 7 | "typeRoots": ["./node_modules/@types"] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /plugin/control-panel/.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | parameters: 4 | ssh-fingerprint: 5 | type: string 6 | default: ${GITHUB_SSH_FINGERPRINT} 7 | 8 | aliases: 9 | # Workflow filters 10 | - &filter-only-master 11 | branches: 12 | only: master 13 | - &filter-only-release 14 | branches: 15 | only: /^v[1-9]*[0-9]+\.[1-9]*[0-9]+\.x$/ 16 | 17 | workflows: 18 | plugin_workflow: 19 | jobs: 20 | - build 21 | 22 | executors: 23 | default_exec: # declares a reusable executor 24 | docker: 25 | - image: srclosson/grafana-plugin-ci-alpine:latest 26 | e2e_exec: 27 | docker: 28 | - image: srclosson/grafana-plugin-ci-e2e:latest 29 | 30 | jobs: 31 | build: 32 | executor: default_exec 33 | steps: 34 | - checkout 35 | - restore_cache: 36 | name: restore node_modules 37 | keys: 38 | - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }} 39 | - run: 40 | name: Install dependencies 41 | command: | 42 | mkdir ci 43 | [ -f ~/project/node_modules/.bin/grafana-toolkit ] || yarn install --frozen-lockfile 44 | - save_cache: 45 | name: save node_modules 46 | paths: 47 | - ~/project/node_modules 48 | key: build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }} 49 | - run: 50 | name: Build and test frontend 51 | command: ./node_modules/.bin/grafana-toolkit plugin:ci-build 52 | - run: 53 | name: Move results to ci folder 54 | command: ./node_modules/.bin/grafana-toolkit plugin:ci-build --finish 55 | - run: 56 | name: Package distribution 57 | command: | 58 | ./node_modules/.bin/grafana-toolkit plugin:ci-package 59 | - persist_to_workspace: 60 | root: . 61 | paths: 62 | - ci/jobs/package 63 | - ci/packages 64 | - ci/dist 65 | - ci/grafana-test-env 66 | - store_artifacts: 67 | path: ci 68 | -------------------------------------------------------------------------------- /plugin/control-panel/.codeclimate.yml: -------------------------------------------------------------------------------- 1 | exclude_patterns: 2 | - "dist" 3 | - "**/node_modules/" 4 | - "**/*.d.ts" 5 | - "src/libs/" 6 | - "src/images/" 7 | - "src/img/" 8 | - "src/screenshots/" 9 | -------------------------------------------------------------------------------- /plugin/control-panel/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | max_line_length = 120 11 | 12 | [*.{js,ts,tsx,scss}] 13 | quote_type = single 14 | 15 | [*.md] 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /plugin/control-panel/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | node_modules/ 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | *.pid.lock 15 | 16 | # Directory for instrumented libs generated by jscoverage/JSCover 17 | lib-cov 18 | 19 | # Coverage directory used by tools like istanbul 20 | coverage 21 | 22 | # Compiled binary addons (https://nodejs.org/api/addons.html) 23 | dist/ 24 | artifacts/ 25 | work/ 26 | ci/ 27 | e2e-results/ 28 | 29 | # Editors 30 | .idea 31 | 32 | -------------------------------------------------------------------------------- /plugin/control-panel/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require("./node_modules/@grafana/toolkit/src/config/prettier.plugin.config.json"), 3 | }; 4 | -------------------------------------------------------------------------------- /plugin/control-panel/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## v1.0.0 6 | 7 | Initial Release 8 | -------------------------------------------------------------------------------- /plugin/control-panel/README.md: -------------------------------------------------------------------------------- 1 | # Grafana Panel Plugin Template 2 | 3 | 10 | 11 | This template is a starting point for building Grafana Panel Plugins in Grafana 7.0+ 12 | 13 | 14 | ## What is Grafana Panel Plugin? 15 | Panels are the building blocks of Grafana. They allow you to visualize data in different ways. While Grafana has several types of panels already built-in, you can also build your own panel, to add support for other visualizations. 16 | 17 | For more information about panels, refer to the documentation on [Panels](https://grafana.com/docs/grafana/latest/features/panels/panels/) 18 | 19 | ## Getting started 20 | 1. Install dependencies 21 | ```BASH 22 | yarn install 23 | ``` 24 | 2. Build plugin in development mode or run in watch mode 25 | ```BASH 26 | yarn dev 27 | ``` 28 | or 29 | ```BASH 30 | yarn watch 31 | ``` 32 | 3. Build plugin in production mode 33 | ```BASH 34 | yarn build 35 | ``` 36 | 37 | ## Learn more 38 | - [Build a panel plugin tutorial](https://grafana.com/tutorials/build-a-panel-plugin) 39 | - [Grafana documentation](https://grafana.com/docs/) 40 | - [Grafana Tutorials](https://grafana.com/tutorials/) - Grafana Tutorials are step-by-step guides that help you make the most of Grafana 41 | - [Grafana UI Library](https://developers.grafana.com/ui) - UI components to help you build interfaces using Grafana Design System 42 | -------------------------------------------------------------------------------- /plugin/control-panel/appveyor.yml: -------------------------------------------------------------------------------- 1 | # Test against the latest version of this Node.js version 2 | environment: 3 | nodejs_version: "12" 4 | 5 | # Local NPM Modules 6 | cache: 7 | - node_modules 8 | 9 | # Install scripts. (runs after repo cloning) 10 | install: 11 | # Get the latest stable version of Node.js or io.js 12 | - ps: Install-Product node $env:nodejs_version 13 | # install modules 14 | - npm install -g yarn --quiet 15 | - yarn install --pure-lockfile 16 | 17 | # Post-install test scripts. 18 | test_script: 19 | # Output useful info for debugging. 20 | - node --version 21 | - npm --version 22 | 23 | # Run the build 24 | build_script: 25 | - yarn dev # This will also run prettier! 26 | - yarn build # make sure both scripts work 27 | -------------------------------------------------------------------------------- /plugin/control-panel/docs/README.md: -------------------------------------------------------------------------------- 1 | TODO: add example docs structure 2 | -------------------------------------------------------------------------------- /plugin/control-panel/jest.config.js: -------------------------------------------------------------------------------- 1 | // This file is needed because it is used by vscode and other tools that 2 | // call `jest` directly. However, unless you are doing anything special 3 | // do not edit this file 4 | 5 | const standard = require('@grafana/toolkit/src/config/jest.plugin.config'); 6 | 7 | // This process will use the same config that `yarn test` is using 8 | module.exports = standard.jestConfig(); 9 | -------------------------------------------------------------------------------- /plugin/control-panel/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "envoya-control-panel", 3 | "version": "1.0.0", 4 | "description": "control panel for kafka cluster testing", 5 | "scripts": { 6 | "build": "grafana-toolkit plugin:build", 7 | "test": "grafana-toolkit plugin:test", 8 | "dev": "grafana-toolkit plugin:dev", 9 | "watch": "grafana-toolkit plugin:dev --watch" 10 | }, 11 | "repository": "github:grafana/simple-react-panel", 12 | "author": "Steve Budarz", 13 | "license": "Apache-2.0", 14 | "bugs": { 15 | "url": "https://github.com/grafana/simple-react-panel/issues", 16 | "email": "plugins@grafana.com" 17 | }, 18 | "devDependencies": { 19 | "@grafana/data": "7.0.1", 20 | "@grafana/toolkit": "7.0.1", 21 | "@grafana/ui": "7.0.1", 22 | "emotion": "10.0.27" 23 | }, 24 | "engines": { 25 | "node": ">=12 <13" 26 | }, 27 | "dependencies": { 28 | "kafkajs": "^1.12.0", 29 | "socket.io-client": "^2.3.0" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /plugin/control-panel/src/Components/Alerts/kafkaConnection.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/MASH/e930daeb82bb90f62dde4101520f48e85c4aec80/plugin/control-panel/src/Components/Alerts/kafkaConnection.tsx -------------------------------------------------------------------------------- /plugin/control-panel/src/Components/Buttons/Produce.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Kafka } from 'kafkajs'; 4 | 5 | interface ProduceButtonProps { 6 | topic: string; 7 | numMessages: number; 8 | kafkaBroker: string; 9 | } 10 | export const ProduceButton: React.FC = (props: ProduceButtonProps) => { 11 | // needs to receive a topic and an amount of messages to create 12 | // should update the state of the button with how many messages were created 13 | }; 14 | -------------------------------------------------------------------------------- /plugin/control-panel/src/Components/Containers/Consumer.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { SocketNames } from '../../types'; 3 | interface ConsumerContainerProps { 4 | socket: SocketIOClient.Socket; 5 | width: number; 6 | topic: string; 7 | setTopic: React.Dispatch>; 8 | } 9 | 10 | let messages = 0; 11 | 12 | export const ConsumerContainer: React.FC = props => { 13 | const { socket, width, topic, setTopic } = props; 14 | const [consumedMessages, setConsumedMessages] = useState(0); 15 | const handleClick = () => { 16 | socket.emit(SocketNames.consumeAll, { topic }); 17 | messages = 0; 18 | setConsumedMessages(0); 19 | }; 20 | useEffect(() => { 21 | socket.on('consumeResponse', () => { 22 | setConsumedMessages(messages++); 23 | }); 24 | }, [socket]); 25 | 26 | return ( 27 |
28 |
29 |

Consume Messages:

30 | 36 |
37 |
{consumedMessages ? consumedMessages + ' messages consumed' : null}
38 |
39 |
40 | ); 41 | }; 42 | -------------------------------------------------------------------------------- /plugin/control-panel/src/Components/Containers/Producer.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { SocketMessages, SocketNames } from '../../types'; 3 | 4 | interface ProducerContainerProps { 5 | socket: SocketIOClient.Socket; 6 | width: number; 7 | topic: string; 8 | setTopic: React.Dispatch>; 9 | isConnected: boolean; 10 | } 11 | 12 | let messagesSent = 0; 13 | let messagesReceived = 0; 14 | 15 | const updatePercentage = (percentDone: number, setPercentDone: React.Dispatch>) => { 16 | messagesReceived++; 17 | const newPercent = (messagesReceived / messagesSent) * 100; 18 | if (Math.floor(newPercent) > percentDone) { 19 | setPercentDone(newPercent); 20 | } 21 | if (newPercent === 100) { 22 | setTimeout(() => setPercentDone(0), 2000); 23 | } 24 | }; 25 | 26 | const handleClick = ( 27 | topic: string, 28 | numMessages: number, 29 | socket: SocketIOClient.Socket, 30 | setTopic: React.Dispatch> 31 | ) => { 32 | const message: SocketMessages.produceNum = { 33 | topic, 34 | messagesCount: numMessages, 35 | }; 36 | socket.emit(SocketNames.produceNum, message); 37 | messagesSent = numMessages; 38 | messagesReceived = 0; 39 | }; 40 | 41 | export const ProducerContainer: React.FC = props => { 42 | const { socket, width, topic, setTopic, isConnected } = props; 43 | const [numMessages, setNumMessages] = useState(1000); 44 | const [percentDone, setPercentDone] = useState(0); 45 | const handleNumMessagesChange = (e: React.ChangeEvent) => setNumMessages(+e.target.value); 46 | useEffect(() => { 47 | socket.on('produceResponse', () => updatePercentage(percentDone, setPercentDone)); 48 | }, [socket]); 49 | 50 | return ( 51 |
52 |
53 |
54 |

Produce Messages:

55 |
56 | 63 | {isConnected && ( 64 | 70 | )} 71 |
{percentDone ? percentDone + '%' : null}
72 |
73 | {percentDone ? ( 74 | 75 | 76 | 77 | 78 | 79 | ) : null} 80 |
81 | ); 82 | }; 83 | -------------------------------------------------------------------------------- /plugin/control-panel/src/Components/Containers/TopicBox.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | 3 | interface TopicBoxProps { 4 | topic: string; 5 | setTopic: React.Dispatch>; 6 | isSelected: boolean; 7 | } 8 | export const TopicBox = (props: TopicBoxProps) => { 9 | const { topic, setTopic, isSelected } = props; 10 | // const [mousePosition, setMousePosition] = useState({ x: 0, y: 0 }); 11 | const [isHovered, setIsHovered] = useState(false); 12 | 13 | // const handleMouseMove = (e: React.MouseEvent) => 14 | // setMousePosition({ x: e.clientX, y: e.clientY }); 15 | 16 | return ( 17 |
setTopic(topic)} 28 | onMouseEnter={() => setIsHovered(true)} 29 | onMouseLeave={() => setIsHovered(false)} 30 | // onMouseMove={handleMouseMove} 31 | > 32 | {topic} 33 | {/* {isHovered && ( 34 |
35 | {topic} 36 |
37 | )} */} 38 |
39 | ); 40 | }; 41 | -------------------------------------------------------------------------------- /plugin/control-panel/src/ControlPanel.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect, useMemo } from 'react'; 2 | import io from 'socket.io-client'; 3 | import { PanelProps, PanelData } from '@grafana/data'; 4 | import { SimpleOptions } from 'types'; 5 | import { ProducerContainer } from './Components/Containers/Producer'; 6 | import { ConsumerContainer } from 'Components/Containers/Consumer'; 7 | import { TopicBox } from './Components/Containers/TopicBox'; 8 | 9 | interface Props extends PanelProps {} 10 | 11 | const TESTBED_SOCKET_ADDRESS = 'localhost:2022'; 12 | const CANNON_SOCKET_ADDRESS = 'localhost:2024'; 13 | 14 | const connectToSocket = (socketAddress: string) => { 15 | const socket = io(socketAddress); 16 | return socket; 17 | }; 18 | 19 | const getTopics = (data: PanelData): Array => { 20 | return data.series.map(series => series.fields[1].labels?.topic).filter(topic => topic !== '__consumer_offsets'); 21 | }; 22 | 23 | export const ControlPanel: React.FC = ({ options, data, width, height }) => { 24 | const socket = useMemo(() => connectToSocket(TESTBED_SOCKET_ADDRESS), [TESTBED_SOCKET_ADDRESS]); 25 | const [topic, setTopic] = useState('test-topic'); 26 | const [topicsList, setTopicsList] = useState(getTopics(data)); 27 | const [showStatus, setShowStatus] = useState(false); 28 | const [isConnected, setIsConnected] = useState(false); 29 | const handleTopicChange = (e: React.ChangeEvent) => setTopic(e.target.value); 30 | const toggleStatus = () => setShowStatus(!showStatus); 31 | 32 | console.log(data); 33 | 34 | useEffect(() => { 35 | socket.on('connect', () => setIsConnected(true)); 36 | socket.on('disconnect', () => setIsConnected(false)); 37 | }, [socket]); 38 | 39 | useEffect(() => { 40 | setTopicsList(getTopics(data)); 41 | }, [data]); 42 | 43 | return ( 44 |
45 |
46 |

Welcome to the Control Panel

47 |
48 | {showStatus && `Status: ${isConnected ? 'Connected' : 'Not Connected'}`} 49 | 50 | 51 | 52 | 53 | 54 |
55 |
56 |
57 |
58 |

Please choose a topic:

59 |
60 | 61 |
62 | 63 | 64 |
74 | {topicsList.map(t => ( 75 | 76 | ))} 77 |
78 |
79 | ); 80 | }; 81 | -------------------------------------------------------------------------------- /plugin/control-panel/src/img/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugin/control-panel/src/module.test.ts: -------------------------------------------------------------------------------- 1 | // Just a stub test 2 | describe('placeholder test', () => { 3 | it('should return true', () => { 4 | expect(true).toBeTruthy(); 5 | }); 6 | }); 7 | -------------------------------------------------------------------------------- /plugin/control-panel/src/module.ts: -------------------------------------------------------------------------------- 1 | import { PanelPlugin } from '@grafana/data'; 2 | import { SimpleOptions } from './types'; 3 | import { ControlPanel } from './ControlPanel'; 4 | 5 | export const plugin = new PanelPlugin(ControlPanel).setPanelOptions(builder => { 6 | return builder.addTextInput({ 7 | path: 'text', 8 | name: 'Kafka Cluster Port', 9 | description: 'Port of your running Kafka Instance', 10 | defaultValue: 'kafka:9092', 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /plugin/control-panel/src/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "panel", 3 | "name": "control_panel", 4 | "id": "envoya-control-panel", 5 | "info": { 6 | "description": "control panel for kafka cluster testing", 7 | "author": { 8 | "name": "Steve Budarz", 9 | "url": "" 10 | }, 11 | "keywords": ["kafka"], 12 | "logos": { 13 | "small": "img/logo.svg", 14 | "large": "img/logo.svg" 15 | }, 16 | "links": [ 17 | { 18 | "name": "Website", 19 | "url": "https://github.com/grafana/simple-react-panel" 20 | }, 21 | { 22 | "name": "License", 23 | "url": "https://github.com/grafana/simple-react-panel/blob/master/LICENSE" 24 | } 25 | ], 26 | "screenshots": [], 27 | "version": "0.1.0", 28 | "updated": "08-04-20" 29 | }, 30 | "dependencies": { 31 | "grafanaVersion": "6.3.x", 32 | "plugins": [] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/control-panel/src/types.ts: -------------------------------------------------------------------------------- 1 | type SeriesSize = 'sm' | 'md' | 'lg'; 2 | 3 | export interface SimpleOptions { 4 | text: string; 5 | showSeriesCount: boolean; 6 | seriesCountSize: SeriesSize; 7 | } 8 | 9 | interface SocketNames { 10 | produceNum: 'produceNum'; 11 | produceRate: 'produceRate'; 12 | consumeRate: 'consumeRate'; 13 | consumeAll: 'consumeAll'; 14 | overloadPartition: 'overloadPartition'; 15 | createTopic: 'createTopic'; 16 | underReplicate: 'underReplicate'; 17 | } 18 | export const SocketNames: SocketNames = { 19 | produceNum: 'produceNum', 20 | produceRate: 'produceRate', 21 | consumeRate: 'consumeRate', 22 | consumeAll: 'consumeAll', 23 | overloadPartition: 'overloadPartition', 24 | createTopic: 'createTopic', 25 | underReplicate: 'underReplicate', 26 | }; 27 | 28 | // eslint-disable-next-line 29 | export namespace SocketMessages { 30 | export type produceNum = { messagesCount: number; topic: string }; 31 | 32 | export type produceRate = { 33 | rate: number; 34 | topic: string; 35 | }; 36 | export type consumeRate = { 37 | rate: number; 38 | topic: string; 39 | }; 40 | 41 | export type overloadPartition = { 42 | topic: string; 43 | numPartitions: number; 44 | messages: number; 45 | }; 46 | 47 | export type createTopic = { 48 | topic: string; 49 | }; 50 | 51 | export type consumeAll = { 52 | topic: string; 53 | }; 54 | export type underReplicate = { 55 | topic: string; 56 | numReplicants: number; 57 | }; 58 | } 59 | export interface ProduceResponse { 60 | key: string; 61 | value: { 62 | name: string; 63 | quote: string; 64 | image_url: string; 65 | }; 66 | } 67 | -------------------------------------------------------------------------------- /plugin/control-panel/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/@grafana/toolkit/src/config/tsconfig.plugin.json", 3 | "include": ["src", "types"], 4 | "compilerOptions": { 5 | "rootDir": "./src", 6 | "baseUrl": "./src", 7 | "typeRoots": ["./node_modules/@types"] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testbed/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:latest 2 | 3 | COPY . /image 4 | 5 | WORKDIR /image 6 | 7 | RUN npm i 8 | 9 | EXPOSE 2022 10 | 11 | CMD ["npm", "run", "testbed"] -------------------------------------------------------------------------------- /testbed/__tests__/frontSocket.test.ts: -------------------------------------------------------------------------------- 1 | import { Socket } from 'socket.io-client'; 2 | 3 | import { SocketNames } from '../testbed/types/socket'; 4 | 5 | import { 6 | sendConsumeAtRateSignal, 7 | sendConsumeSignal, 8 | sendOverloadPartitionSignal, 9 | sendProduceAtRateSignal, 10 | sendProduceSignal, 11 | sendUnderReplicateSignal, 12 | } from '../test-front/socketEvents'; 13 | 14 | describe('front end socket test suite', () => { 15 | let mockSocket = { 16 | emit: (eventName, message) => { 17 | socketMessage = { eventName, message }; 18 | }, 19 | }; 20 | interface Message { 21 | [key: string]: any; 22 | } 23 | 24 | interface SocketMessage { 25 | eventName: string; 26 | message: Message; 27 | } 28 | 29 | let socketMessage: SocketMessage = { eventName: '', message: {} }; 30 | 31 | describe('send produce signal', () => { 32 | const topic = 'test-topic'; 33 | const messagesCount = 1000; 34 | beforeAll(() => { 35 | sendProduceSignal(mockSocket as typeof Socket, topic, messagesCount); 36 | }); 37 | 38 | afterAll(() => { 39 | socketMessage = { eventName: '', message: {} }; 40 | }); 41 | it('sends produce signal with the correct event name', () => { 42 | expect(socketMessage.eventName).toEqual(SocketNames.produceNum); 43 | }); 44 | it('send the produce signal with the passed-in message', () => { 45 | expect(socketMessage.message).toEqual({ topic, messagesCount }); 46 | }); 47 | describe('produce defaults', () => { 48 | beforeAll(() => { 49 | sendProduceSignal(mockSocket as typeof Socket, '', 100); 50 | }); 51 | 52 | afterAll(() => { 53 | socketMessage = { eventName: '', message: {} }; 54 | }); 55 | it("includes 'test-topic' as the default topic", () => { 56 | expect(socketMessage.message.topic).toEqual('test-topic'); 57 | }); 58 | }); 59 | }); 60 | 61 | describe('send consume signal', () => { 62 | const topic = 'test-topic'; 63 | beforeAll(() => { 64 | sendConsumeSignal(mockSocket as typeof Socket, topic); 65 | }); 66 | 67 | afterAll(() => { 68 | socketMessage = { eventName: '', message: {} }; 69 | }); 70 | it('sends consume signal with the correct event name', () => { 71 | expect(socketMessage.eventName).toEqual(SocketNames.consumeAll); 72 | }); 73 | it('send the consume signal with the passed-in message', () => { 74 | expect(socketMessage.message).toEqual({ topic }); 75 | }); 76 | }); 77 | describe('send produce at rate signal', () => { 78 | const topic = 'test-topic'; 79 | const rate = 10; 80 | beforeAll(() => { 81 | sendProduceAtRateSignal(mockSocket as typeof Socket, topic, rate); 82 | }); 83 | 84 | afterAll(() => { 85 | socketMessage = { eventName: '', message: {} }; 86 | }); 87 | it('sends produce at rate signal with the correct event name', () => { 88 | expect(socketMessage.eventName).toEqual(SocketNames.produceRate); 89 | }); 90 | it('send the produce at rate signal with the passed-in message', () => { 91 | expect(socketMessage.message).toEqual({ topic, rate }); 92 | }); 93 | describe('produce at rate defaults', () => { 94 | beforeAll(() => { 95 | sendProduceAtRateSignal(mockSocket as typeof Socket, '', 100); 96 | }); 97 | 98 | afterAll(() => { 99 | socketMessage = { eventName: '', message: {} }; 100 | }); 101 | it("includes 'test-topic' as the default topic", () => { 102 | expect(socketMessage.message.topic).toEqual('test-topic'); 103 | }); 104 | }); 105 | }); 106 | describe('send consume at rate signal', () => { 107 | const topic = 'test-topic'; 108 | const rate = 10; 109 | beforeAll(() => { 110 | sendConsumeAtRateSignal(mockSocket as typeof Socket, topic, rate); 111 | }); 112 | 113 | afterAll(() => { 114 | socketMessage = { eventName: '', message: {} }; 115 | }); 116 | it('sends consume at rate signal with the correct event name', () => { 117 | expect(socketMessage.eventName).toEqual(SocketNames.consumeRate); 118 | }); 119 | it('send the consume at rate signal with the passed-in message', () => { 120 | expect(socketMessage.message).toEqual({ topic, rate }); 121 | }); 122 | }); 123 | describe('send under replicate signal', () => { 124 | const topic = 'test-topic'; 125 | const numReplicants = 10; 126 | beforeAll(() => { 127 | sendUnderReplicateSignal(mockSocket as typeof Socket, topic, numReplicants); 128 | }); 129 | 130 | afterAll(() => { 131 | socketMessage = { eventName: '', message: {} }; 132 | }); 133 | it('sends under replicate signal with the correct event name', () => { 134 | expect(socketMessage.eventName).toEqual(SocketNames.underReplicate); 135 | }); 136 | it('send the under replicate signal with the passed-in message', () => { 137 | expect(socketMessage.message).toEqual({ topic, numReplicants }); 138 | }); 139 | }); 140 | describe('send overload partition signal', () => { 141 | const topic = 'testing-topic'; 142 | const numPartitions = 10; 143 | const messages = 1000; 144 | beforeAll(() => { 145 | sendOverloadPartitionSignal(mockSocket as typeof Socket, topic, messages, numPartitions); 146 | }); 147 | 148 | afterAll(() => { 149 | socketMessage = { eventName: '', message: {} }; 150 | }); 151 | it('sends overload partition signal with the correct event name', () => { 152 | expect(socketMessage.eventName).toEqual(SocketNames.overloadPartition); 153 | }); 154 | it('send the overload partition signal with the passed-in message', () => { 155 | expect(socketMessage.message).toEqual({ topic, messages, numPartitions }); 156 | }); 157 | describe('overload parition defaults', () => { 158 | beforeAll(() => { 159 | sendOverloadPartitionSignal(mockSocket as typeof Socket); 160 | }); 161 | 162 | afterAll(() => { 163 | socketMessage = { eventName: '', message: {} }; 164 | }); 165 | it("includes 'test-topic' as the default topic", () => { 166 | expect(socketMessage.message.topic).toEqual('test-topic'); 167 | }); 168 | it('includes 100 as the default number of messages', () => { 169 | expect(socketMessage.message.messages).toEqual(100); 170 | }); 171 | it('includes 2 as the default number of partitions', () => { 172 | expect(socketMessage.message.numPartitions).toEqual(2); 173 | }); 174 | }); 175 | }); 176 | }); 177 | -------------------------------------------------------------------------------- /testbed/__tests__/puppeteer.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require('puppeteer'); 2 | const server = require('../testbed/server'); 3 | 4 | // const APP = `http://localhost:${process.env.PORT || 2022}/`; 5 | const APP = `http://localhost:2022/`; 6 | // const APP = server; 7 | 8 | describe('Front-end Integration/Features', () => { 9 | let browser; 10 | let page; 11 | 12 | beforeAll(async () => { 13 | browser = await puppeteer.launch({ 14 | args: ['--no-sandbox', '--disable-setuid-sandbox'], 15 | }); 16 | page = await browser.newPage(); 17 | }); 18 | 19 | afterAll(() => { 20 | browser.close(); 21 | }); 22 | 23 | // simulate button click Producer 24 | const buttonProducer = async () => { 25 | await page.waitForSelector('#message-num'); 26 | await page.focus('#message-num'); 27 | await page.keyboard.type('1'); 28 | await page.waitForSelector('#produce-button'); 29 | await page.click('#produce-button'); 30 | }; 31 | 32 | describe('Initial display', () => { 33 | it('loads successfully', async () => { 34 | // We navigate to the page at the beginning of each case so we have a 35 | // fresh start 36 | await page.goto(APP); 37 | await page.waitForSelector('#header'); 38 | const title = await page.$eval('#header', (el) => el.innerHTML); 39 | expect(title).toBe('Select Option To Generate Data'); 40 | }); 41 | 42 | // checkes Input field of Producer 43 | it('displays a usable input field for Producer', async () => { 44 | await page.waitForSelector('#message-num'); 45 | await page.focus('#message-num'); 46 | await page.keyboard.type('100'); 47 | const inputValue = await page.$eval('#message-num', (el) => el.value); 48 | expect(inputValue).toBe('100'); 49 | }); 50 | 51 | // checkes Input field of Consumer 52 | it('displays a usable input field for Consumer', async () => { 53 | await page.waitForSelector('#consume-input'); 54 | await page.focus('#consume-input'); 55 | await page.keyboard.type('100'); 56 | const inputValue = await page.$eval('#consume-input', (el) => el.value); 57 | expect(inputValue).toBe('100'); 58 | }); 59 | 60 | // // Simulate button click and run msg Producer 61 | // it('renders the Producer MSG section reciver is socket.io', async () => { 62 | // await buttonProducer(); 63 | 64 | // }); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /testbed/__tests__/socketIo_test.js: -------------------------------------------------------------------------------- 1 | const io = require('socket.io-client'); 2 | const http = require('http'); 3 | const ioBack = require('socket.io'); 4 | const { connect } = require('net'); 5 | // const formatMessage = require('../utils/messages'); 6 | 7 | let socket; 8 | let httpServer; 9 | let httpServerAddr; 10 | let ioServer; 11 | // const testMsg = 'Admin'; 12 | 13 | /** 14 | * Setup WS & HTTP servers 15 | */ 16 | beforeAll((done) => { 17 | httpServer = http.createServer().listen(); 18 | 19 | httpServerAddr = httpServer.address(); 20 | 21 | ioServer = ioBack(httpServer); 22 | done(); 23 | }); 24 | 25 | /** 26 | * Cleanup WS & HTTP servers 27 | */ 28 | afterAll((done) => { 29 | ioServer.close(); 30 | httpServer.close(); 31 | done(); 32 | }); 33 | 34 | /** 35 | * Run before each test 36 | */ 37 | beforeEach((done) => { 38 | // Setup 39 | // Do not hardcode server port and address, square brackets are used for IPv6 40 | socket = io.connect( 41 | `http://[${httpServerAddr.address}]:${httpServerAddr.port}`, 42 | { 43 | 'reconnection delay': 0, 44 | 'reopen delay': 0, 45 | 'force new connection': true, 46 | transports: ['websocket'], 47 | } 48 | ); 49 | 50 | // console.log(socket); 51 | 52 | socket.on('connect', () => { 53 | done(); 54 | }); 55 | }); 56 | 57 | /** 58 | * Run after each test 59 | */ 60 | afterEach((done) => { 61 | // Cleanup 62 | if (socket.connected) { 63 | socket.disconnect(); 64 | } 65 | done(); 66 | }); 67 | 68 | describe('basic socket.io example', () => { 69 | test('should communicate', (done) => { 70 | // once connected, emit Hello World 71 | ioServer.emit('echo', 'Hello World'); 72 | socket.once('echo', (message) => { 73 | // Check that the message matches 74 | expect(message).toBe('Hello World'); 75 | done(); 76 | }); 77 | ioServer.on('connection', (mySocket) => { 78 | // console.log(mySocket); 79 | expect(mySocket).toBeDefined(); 80 | }); 81 | }); 82 | 83 | test('should communicate', (done) => { 84 | // once connected, emit Besik 85 | ioServer.emit('message1', 'Besik'); 86 | socket.once('message1', (message) => { 87 | // Check that the message matches 88 | expect(message).toBe('Besik'); 89 | done(); 90 | }); 91 | ioServer.on('connection', (mySocket) => { 92 | expect(mySocket).toBeDefined(); 93 | }); 94 | }); 95 | 96 | test('should communicate with waiting for socket.io handshakes', (done) => { 97 | // Emit sth from Client do Server 98 | // socket.emit('message1', formatMessage(testMsg, 'Welcome to my test')); 99 | socket.emit('examlpe', 'some messages'); 100 | // Use timeout to wait for socket.io server handshakes 101 | setTimeout(() => { 102 | done(); 103 | }, 50); 104 | }); 105 | }); 106 | -------------------------------------------------------------------------------- /testbed/babel.config.js: -------------------------------------------------------------------------------- 1 | // babel.config.js 2 | module.exports = { 3 | presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'], 4 | }; 5 | -------------------------------------------------------------------------------- /testbed/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MASH_testbed", 3 | "version": "1.0.0", 4 | "description": "A kafka testbed for MASH", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "jest", 8 | "testbed": "node ./testbed/server.js", 9 | "build-testbed": "webpack test-front/main.ts -o test-front/main.js" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "dependencies": { 15 | "@types/express": "^4.17.7", 16 | "@types/jest": "^26.0.8", 17 | "@types/kafkajs": "^1.9.0", 18 | "@types/puppeteer": "^3.0.1", 19 | "@types/socket.io": "^2.1.9", 20 | "@types/socket.io-client": "^1.4.33", 21 | "express": "^4.17.1", 22 | "jest": "^26.2.2", 23 | "kafkajs": "^1.12.0", 24 | "puppeteer": "^5.2.1", 25 | "socket.io": "^2.3.0", 26 | "socket.io-client": "^2.3.0", 27 | "ts-loader": "^8.0.1", 28 | "typescript": "^3.9.7", 29 | "webpack": "^4.43.0", 30 | "webpack-cli": "^3.3.12", 31 | "webpack-dev-server": "^3.11.0" 32 | }, 33 | "devDependencies": { 34 | "@babel/core": "^7.11.0", 35 | "@babel/preset-env": "^7.10.4", 36 | "@babel/preset-typescript": "^7.10.4", 37 | "babel-jest": "^26.2.2" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /testbed/scripts/prometheus.yml: -------------------------------------------------------------------------------- 1 | scrape_configs: 2 | - job_name: 'kafka_exporter' 3 | scrape_interval: 5s 4 | static_configs: 5 | - targets: ['kafka_exporter:9308'] 6 | labels: 7 | group: 'kafka_exporter' 8 | -------------------------------------------------------------------------------- /testbed/test-front/main.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"main.js","sourceRoot":"","sources":["main.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAkB,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtE,MAAM,MAAM,GAAG,EAAE,EAAE,CAAC;AACpB,IAAI,YAAY,GAAG,CAAC,CAAC;AACrB,IAAI,YAAY,GAAG,CAAC,CAAC;AACrB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;IACxB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,EAAE;IACxC,MAAM,UAAU,GAA8B;QAC5C,aAAa,EAAE,GAAG;KACnB,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;AAClD,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAa,EAAE,EAAE;IAC1C,MAAM,UAAU,GAA8B;QAC5C,KAAK;KACN,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;AAClD,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,CAClC,QAAgB,YAAY,EAC5B,WAAmB,GAAG,EACtB,gBAAwB,CAAC,EACzB,EAAE;IACF,MAAM,QAAQ,GAAqC;QACjD,QAAQ;QACR,aAAa;QACb,KAAK;KACN,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAAC,IAAY,EAAE,EAAE;IAC/C,MAAM,aAAa,GAA+B;QAChD,IAAI;KACL,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAAC,KAAa,EAAE,IAAY,EAAE,EAAE;IAC9D,MAAM,aAAa,GAA+B;QAChD,IAAI;QACJ,KAAK;KACN,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,wBAAwB,GAAG,CAAC,KAAa,EAAE,gBAAwB,CAAC,EAAE,EAAE;IAC5E,MAAM,kBAAkB,GAAkC;QACxD,KAAK;QACL,aAAa;KACd,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,IAAS,EAAE,EAAE;IACxC,YAAY,EAAE,CAAC;IACf,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IACnD,SAAS,CAAC,SAAS,GAAG,aAAa,YAAY,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5F,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,GAAG,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACzF,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAChD,CAAC,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAO,IAAS,EAAE,EAAE;IAC9C,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,YAAY,EAAE,CAAC;IACf,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,QAAQ,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IACnD,SAAS,CAAC,SAAS,GAAG,cAAc,YAAY,MAAM,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;IACjF,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,GAAG,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACzF,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;AAC1C,CAAC,CAAA,CAAC;AAEF,MAAM,aAAa,GAAG,QAAQ,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;AAChE,MAAM,aAAa,GAAG,QAAQ,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;AAChE,MAAM,2BAA2B,GAAG,QAAQ,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAC;AACxF,MAAM,uBAAuB,GAAG,QAAQ,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;AAC9E,MAAM,uBAAuB,GAAG,QAAQ,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;AAC9E,MAAM,wBAAwB,GAAG,QAAQ,CAAC,cAAc,CAAC,uBAAuB,CAAC,CAAC;AAElF,aAAa,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;IAC3C,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC,aAAa,CAAqB,CAAC;IAC9E,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC;IACrC,iBAAiB,CAAC,CAAC,WAAW,CAAC,CAAC;IAChC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC;AACxB,CAAC,CAAC,CAAC;AAEH,aAAa,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;IAC3C,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC,eAAe,CAAqB,CAAC;IAChF,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;IAC/B,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC;IACtB,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC,CAAC,CAAC;AAEH,2BAA2B,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;IACzD,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAqB,CAAC;IACpF,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;IAC/B,2BAA2B,CAAC,KAAK,CAAC,CAAC;IACnC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC;AACxB,CAAC,CAAC,CAAC;AAEH,uBAAuB,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;IACrD,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC,aAAa,CAAqB,CAAC;IAC9E,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC;IACrC,uBAAuB,CAAC,CAAC,WAAW,CAAC,CAAC;IACtC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC;AACxB,CAAC,CAAC,CAAC;AAEH,uBAAuB,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;IACrD,MAAM,WAAW,GAAG,QAAQ,CAAC,cAAc,CAAC,eAAe,CAAqB,CAAC;IACjF,MAAM,WAAW,GAAG,QAAQ,CAAC,cAAc,CAAC,gBAAgB,CAAqB,CAAC;IAClF,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC;IACzC,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC;IACzC,uBAAuB,CAAC,cAAc,EAAE,CAAC,cAAc,CAAC,CAAC;IACzD,WAAW,CAAC,KAAK,GAAG,EAAE,CAAC;AACzB,CAAC,CAAC,CAAC;AAEH,wBAAwB,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;IACtD,MAAM,WAAW,GAAG,QAAQ,CAAC,cAAc,CAAC,kBAAkB,CAAqB,CAAC;IACpF,MAAM,WAAW,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAqB,CAAC;IACrF,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC;IACzC,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC;IACzC,wBAAwB,CAAC,cAAc,EAAE,CAAC,cAAc,CAAC,CAAC;IAC1D,WAAW,CAAC,KAAK,GAAG,EAAE,CAAC;AACzB,CAAC,CAAC,CAAC;AAEH,MAAM,CAAC,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC;AAClD,MAAM,CAAC,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC"} -------------------------------------------------------------------------------- /testbed/test-front/main.ts: -------------------------------------------------------------------------------- 1 | import * as io from 'socket.io-client'; 2 | import { 3 | sendConsumeAtRateSignal, 4 | sendConsumeSignal, 5 | sendOverloadPartitionSignal, 6 | sendProduceAtRateSignal, 7 | sendProduceSignal, 8 | sendUnderReplicateSignal, 9 | } from './socketEvents'; 10 | 11 | const socket = io(); 12 | let consumeCount = 0; 13 | let produceCount = 0; 14 | socket.on('connect', () => { 15 | console.log('socket connected'); 16 | }); 17 | 18 | const listProduceResponse = (data: any) => { 19 | produceCount++; 20 | const container = document.getElementById('produce-container'); 21 | const img = document.getElementById('produce-img'); 22 | container.innerText = `PRODUCED: ${produceCount} - ${data.value.name}: ${data.value.quote}`; 23 | container.style.border = '2px solid ' + '#' + produceCount.toString(16).padStart(6, '0'); 24 | img.setAttribute('src', data.value.image_url); 25 | }; 26 | 27 | const listConsumeResponse = async (data: any) => { 28 | data = await JSON.parse(data); 29 | consumeCount++; 30 | const container = document.getElementById('consume-container'); 31 | const img = document.getElementById('consume-img'); 32 | container.innerText = ` CONSUMED: ${consumeCount} - ${data.name}: ${data.quote}`; 33 | container.style.border = '2px solid ' + '#' + produceCount.toString(16).padStart(6, '0'); 34 | img.setAttribute('src', data.image_url); 35 | }; 36 | 37 | const produceButton = document.getElementById('produce-button'); 38 | const consumeButton = document.getElementById('consume-button'); 39 | const sendOverloadPartitionButton = document.getElementById('OverloadPartition-button'); 40 | const sendProduceAtRateButton = document.getElementById('produceRate-button'); 41 | const sendConsumeAtRateButton = document.getElementById('consumeRate-button'); 42 | const sendUnderReplicateButton = document.getElementById('underReplicate-button'); 43 | 44 | produceButton.addEventListener('click', () => { 45 | const inputField = document.getElementById('message-num') as HTMLInputElement; 46 | const numMessages = inputField.value; 47 | const topicField = document.getElementById('produce-topic') as HTMLInputElement; 48 | const topic = topicField.value; 49 | sendProduceSignal(socket, topic, +numMessages); 50 | inputField.value = ''; 51 | }); 52 | 53 | consumeButton.addEventListener('click', () => { 54 | const inputField = document.getElementById('consume-input') as HTMLInputElement; 55 | const topic = inputField.value; 56 | inputField.value = ''; 57 | sendConsumeSignal(socket, topic); 58 | }); 59 | 60 | sendOverloadPartitionButton.addEventListener('click', () => { 61 | const inputField = document.getElementById('OverloadPartition') as HTMLInputElement; 62 | const topic = inputField.value; 63 | sendOverloadPartitionSignal(socket, topic); 64 | inputField.value = ''; 65 | }); 66 | 67 | sendProduceAtRateButton.addEventListener('click', () => { 68 | const inputField = document.getElementById('produceRate') as HTMLInputElement; 69 | const numMessages = inputField.value; 70 | const produceTopic = document.getElementById('produceRate-topic') as HTMLInputElement; 71 | const topic = produceTopic.value; 72 | sendProduceAtRateSignal(socket, topic, +numMessages); 73 | inputField.value = ''; 74 | }); 75 | 76 | sendConsumeAtRateButton.addEventListener('click', () => { 77 | const inputString = document.getElementById('consumeRateSt') as HTMLInputElement; 78 | const inputNumber = document.getElementById('consumeRateNum') as HTMLInputElement; 79 | const stringMessages = inputString.value; 80 | const numberMessages = inputString.value; 81 | sendConsumeAtRateSignal(socket, stringMessages, +numberMessages); 82 | inputNumber.value = ''; 83 | }); 84 | 85 | sendUnderReplicateButton.addEventListener('click', () => { 86 | const inputString = document.getElementById('underReplicateSt') as HTMLInputElement; 87 | const inputNumber = document.getElementById('underReplicateNum') as HTMLInputElement; 88 | const stringMessages = inputString.value; 89 | const numberMessages = inputString.value; 90 | sendUnderReplicateSignal(socket, stringMessages, +numberMessages); 91 | inputNumber.value = ''; 92 | }); 93 | 94 | socket.on('consumeResponse', listConsumeResponse); 95 | socket.on('produceResponse', listProduceResponse); 96 | -------------------------------------------------------------------------------- /testbed/test-front/socketEvents.ts: -------------------------------------------------------------------------------- 1 | import { SocketMessages, SocketNames } from '../testbed/types/socket'; 2 | 3 | export const sendProduceSignal = (socket: SocketIOClient.Socket, topic: string, num: number) => { 4 | topic = topic === '' ? 'test-topic' : topic; 5 | const produceNum: SocketMessages.produceNum = { 6 | messagesCount: num, 7 | topic, 8 | }; 9 | socket.emit(SocketNames.produceNum, produceNum); 10 | }; 11 | 12 | export const sendConsumeSignal = (socket: SocketIOClient.Socket, topic: string) => { 13 | const consumeAll: SocketMessages.consumeAll = { 14 | topic, 15 | }; 16 | socket.emit(SocketNames.consumeAll, consumeAll); 17 | }; 18 | 19 | export const sendOverloadPartitionSignal = ( 20 | socket: SocketIOClient.Socket, 21 | topic: string = 'test-topic', 22 | messages: number = 100, 23 | numPartitions: number = 2 24 | ) => { 25 | const overload: SocketMessages.overloadPartition = { 26 | messages, 27 | numPartitions, 28 | topic, 29 | }; 30 | socket.emit(SocketNames.overloadPartition, overload); 31 | }; 32 | 33 | export const sendProduceAtRateSignal = ( 34 | socket: SocketIOClient.Socket, 35 | topic: string, 36 | rate: number 37 | ) => { 38 | topic = topic === '' ? 'test-topic' : topic; 39 | const produceAtRate: SocketMessages.produceRate = { 40 | rate, 41 | topic, 42 | }; 43 | socket.emit(SocketNames.produceRate, produceAtRate); 44 | }; 45 | 46 | export const sendConsumeAtRateSignal = ( 47 | socket: SocketIOClient.Socket, 48 | topic: string, 49 | rate: number 50 | ) => { 51 | const consumeAtRate: SocketMessages.consumeRate = { 52 | rate, 53 | topic, 54 | }; 55 | socket.emit(SocketNames.consumeRate, consumeAtRate); 56 | }; 57 | 58 | export const sendUnderReplicateSignal = ( 59 | socket: SocketIOClient.Socket, 60 | topic: string, 61 | numReplicants: number = 2 62 | ) => { 63 | const sendUnderReplicate: SocketMessages.underReplicate = { 64 | topic, 65 | numReplicants, 66 | }; 67 | socket.emit(SocketNames.underReplicate, sendUnderReplicate); 68 | }; 69 | -------------------------------------------------------------------------------- /testbed/test-front/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": true, 4 | "sourceMap": true, 5 | "module": "ES2015", 6 | "target": "ES2015" 7 | }, 8 | "files": ["main.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /testbed/test-front/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | mode: 'development', 5 | entry: './main.ts', 6 | output: { 7 | filename: 'main.js', 8 | path: path.resolve(__dirname), 9 | }, 10 | module: { 11 | rules: [ 12 | { 13 | test: /\.tsx?$/, 14 | use: 'ts-loader', 15 | exclude: /node_modules/, 16 | }, 17 | ], 18 | }, 19 | resolve: { 20 | extensions: ['.tsx', '.ts', '.js'], 21 | }, 22 | }; 23 | -------------------------------------------------------------------------------- /testbed/testbed/consumer/consumer.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | exports.consumeAtRate = exports.consume = void 0; 40 | var kafkajs_1 = require("kafkajs"); 41 | var kafka = new kafkajs_1.Kafka({ 42 | clientId: 'test-consumer', 43 | brokers: ['kafka:9092'] 44 | }); 45 | var count = 0; 46 | var consumer = kafka.consumer({ groupId: 'test-group' }); 47 | consumer.subscribe({ topic: 'test-topic' }); 48 | var randomLetter = function () { 49 | var letters = 'abcdefghijklmnopqrstuvwxyz'; 50 | return letters[Math.floor(Math.random() * letters.length)]; 51 | }; 52 | var randomGroupName = function () { return Array(9).fill('a').map(randomLetter).join(''); }; 53 | exports.consume = function (socket, topic) { 54 | if (topic === void 0) { topic = 'test-topic'; } 55 | return __awaiter(void 0, void 0, void 0, function () { 56 | var consumer; 57 | return __generator(this, function (_a) { 58 | switch (_a.label) { 59 | case 0: 60 | consumer = kafka.consumer({ groupId: randomGroupName() }); 61 | consumer.subscribe({ topic: topic, fromBeginning: true }); 62 | return [4 /*yield*/, consumer.run({ 63 | eachMessage: function (payload) { return __awaiter(void 0, void 0, void 0, function () { 64 | var message; 65 | return __generator(this, function (_a) { 66 | message = payload.message; 67 | socket.emit('consumeResponse', message.value.toString()); 68 | return [2 /*return*/]; 69 | }); 70 | }); } 71 | })]; 72 | case 1: 73 | _a.sent(); 74 | return [2 /*return*/]; 75 | } 76 | }); 77 | }); 78 | }; 79 | exports.consumeAtRate = function (socket, rate, topic) { return __awaiter(void 0, void 0, void 0, function () { 80 | var ratedConsumer_1, err_1; 81 | return __generator(this, function (_a) { 82 | switch (_a.label) { 83 | case 0: 84 | _a.trys.push([0, 4, , 5]); 85 | ratedConsumer_1 = kafka.consumer({ groupId: randomGroupName() }); 86 | return [4 /*yield*/, ratedConsumer_1.connect()]; 87 | case 1: 88 | _a.sent(); 89 | return [4 /*yield*/, ratedConsumer_1.subscribe({ topic: topic })]; 90 | case 2: 91 | _a.sent(); 92 | return [4 /*yield*/, ratedConsumer_1.run({ 93 | eachMessage: function (payload) { return __awaiter(void 0, void 0, void 0, function () { 94 | var value; 95 | return __generator(this, function (_a) { 96 | value = payload.message.value; 97 | socket.emit('ratedConsumeResponse', value.toString()); 98 | ratedConsumer_1.pause([{ topic: topic }]); 99 | setTimeout(function () { return ratedConsumer_1.resume([{ topic: topic }]); }, Math.floor(1000 / rate)); 100 | return [2 /*return*/]; 101 | }); 102 | }); } 103 | })]; 104 | case 3: 105 | _a.sent(); 106 | return [3 /*break*/, 5]; 107 | case 4: 108 | err_1 = _a.sent(); 109 | console.error('rated consumer error:', err_1); 110 | return [3 /*break*/, 5]; 111 | case 5: return [2 /*return*/]; 112 | } 113 | }); 114 | }); }; 115 | //# sourceMappingURL=consumer.js.map -------------------------------------------------------------------------------- /testbed/testbed/consumer/consumer.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"consumer.js","sourceRoot":"","sources":["consumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAAoD;AAEpD,IAAM,KAAK,GAAG,IAAI,eAAK,CAAC;IACtB,QAAQ,EAAE,eAAe;IACzB,OAAO,EAAE,CAAC,YAAY,CAAC;CACxB,CAAC,CAAC;AAEH,IAAI,KAAK,GAAG,CAAC,CAAC;AAEd,IAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;AAC3D,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;AAe5C,IAAM,YAAY,GAAG;IACnB,IAAM,OAAO,GAAG,4BAA4B,CAAC;IAC7C,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AAC7D,CAAC,CAAC;AAEF,IAAM,eAAe,GAAG,cAAc,OAAA,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAA7C,CAA6C,CAAC;AAEvE,QAAA,OAAO,GAAG,UAAO,MAAW,EAAE,KAA4B;IAA5B,sBAAA,EAAA,oBAA4B;;;;;;oBAC/D,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;oBAChE,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,OAAA,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBACnD,qBAAM,QAAQ,CAAC,GAAG,CAAC;4BACjB,WAAW,EAAE,UAAO,OAA2B;;;oCACrC,OAAO,GAAK,OAAO,QAAZ,CAAa;oCAC5B,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;;;iCAC1D;yBACF,CAAC,EAAA;;oBALF,SAKE,CAAC;;;;;CACJ,CAAC;AAEW,QAAA,aAAa,GAAG,UAAO,MAAW,EAAE,IAAY,EAAE,KAAa;;;;;;gBAElE,kBAAgB,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;gBACrE,qBAAM,eAAa,CAAC,OAAO,EAAE,EAAA;;gBAA7B,SAA6B,CAAC;gBAC9B,qBAAM,eAAa,CAAC,SAAS,CAAC,EAAE,KAAK,OAAA,EAAE,CAAC,EAAA;;gBAAxC,SAAwC,CAAC;gBACzC,qBAAM,eAAa,CAAC,GAAG,CAAC;wBACtB,WAAW,EAAE,UAAO,OAA2B;;;gCACrC,KAAK,GAAK,OAAO,CAAC,OAAO,MAApB,CAAqB;gCAClC,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;gCACtD,eAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,OAAA,EAAE,CAAC,CAAC,CAAC;gCACjC,UAAU,CAAC,cAAM,OAAA,eAAa,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,OAAA,EAAE,CAAC,CAAC,EAAjC,CAAiC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;;;6BAC9E;qBACF,CAAC,EAAA;;gBAPF,SAOE,CAAC;;;;gBAEH,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAG,CAAC,CAAC;;;;;KAE/C,CAAC"} -------------------------------------------------------------------------------- /testbed/testbed/consumer/consumer.ts: -------------------------------------------------------------------------------- 1 | import { Kafka, EachMessagePayload } from 'kafkajs'; 2 | 3 | const kafka = new Kafka({ 4 | clientId: 'test-consumer', 5 | brokers: ['kafka:9092'], 6 | }); 7 | 8 | let count = 0; 9 | 10 | const consumer = kafka.consumer({ groupId: 'test-group' }); 11 | consumer.subscribe({ topic: 'test-topic' }); 12 | export interface ConsumeRequest { 13 | messagesCount: number; 14 | } 15 | 16 | export interface ConsumeResponse { 17 | message: { 18 | value: { 19 | name: string; 20 | quote: string; 21 | image_url: string; 22 | }; 23 | }; 24 | } 25 | 26 | const randomLetter = (): string => { 27 | const letters = 'abcdefghijklmnopqrstuvwxyz'; 28 | return letters[Math.floor(Math.random() * letters.length)]; 29 | }; 30 | 31 | const randomGroupName = (): string => Array(9).fill('a').map(randomLetter).join(''); 32 | 33 | export const consume = async (socket: any, topic: string = 'test-topic') => { 34 | const consumer = kafka.consumer({ groupId: randomGroupName() }); 35 | consumer.subscribe({ topic, fromBeginning: true }); 36 | await consumer.run({ 37 | eachMessage: async (payload: EachMessagePayload) => { 38 | const { message } = payload; 39 | socket.emit('consumeResponse', message.value.toString()); 40 | }, 41 | }); 42 | }; 43 | 44 | export const consumeAtRate = async (socket: any, rate: number, topic: string) => { 45 | try { 46 | const ratedConsumer = kafka.consumer({ groupId: randomGroupName() }); 47 | await ratedConsumer.connect(); 48 | await ratedConsumer.subscribe({ topic }); 49 | await ratedConsumer.run({ 50 | eachMessage: async (payload: EachMessagePayload) => { 51 | const { value } = payload.message; 52 | socket.emit('ratedConsumeResponse', value.toString()); 53 | ratedConsumer.pause([{ topic }]); 54 | setTimeout(() => ratedConsumer.resume([{ topic }]), Math.floor(1000 / rate)); 55 | }, 56 | }); 57 | } catch (err) { 58 | console.error('rated consumer error:', err); 59 | } 60 | }; 61 | -------------------------------------------------------------------------------- /testbed/testbed/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | KAFKA TEST BED 8 | 9 | 10 | 11 |
12 |
13 | 14 | 19 | 24 | 25 |
26 | # 27 |
28 |
29 |
30 | 31 | 36 | 37 |
38 | # 39 |
40 |
41 |
42 |
43 |
44 | 45 | 50 | 51 |
52 |
53 | 54 | 55 | 56 | 57 |
58 |
59 | 60 | 61 | 62 | 63 |
64 |
65 | 66 | 67 | 68 | 69 |
70 |
71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /testbed/testbed/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | background: lightBlue; 3 | } 4 | #overall { 5 | display: flex; 6 | flex-direction: row; 7 | justify-content: space-between; 8 | margin: 1rem; 9 | } 10 | #overall > div { 11 | margin: 2rem; 12 | width: 100%; 13 | } 14 | #produce-container { 15 | font-size: x-large; 16 | padding: 2em; 17 | } 18 | #consume-container { 19 | padding: 2em; 20 | font-size: x-large; 21 | } 22 | 23 | img { 24 | height: 25px; 25 | padding-left: 6px; 26 | } 27 | 28 | .createFields { 29 | width: 100%; 30 | display: flex; 31 | margin-bottom: 0.2rem; 32 | } 33 | 34 | .createFields label { 35 | min-width: 8rem; 36 | width: 9rem; 37 | } 38 | .createFields button { 39 | min-width: 8rem; 40 | width: 9rem; 41 | } 42 | .register { 43 | color: white; 44 | background-color: lightslategrey; 45 | } 46 | -------------------------------------------------------------------------------- /testbed/testbed/producer/producer.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | exports.produceToPartition = exports.produceAtRateToPartition = exports.produce = exports.produceAtRate = void 0; 40 | var kafkajs_1 = require("kafkajs"); 41 | var kafka = new kafkajs_1.Kafka({ 42 | clientId: 'test-producer', 43 | brokers: ['kafka:9092'] 44 | }); 45 | var count = 0; 46 | var producer = kafka.producer(); 47 | var connect = function () { return __awaiter(void 0, void 0, void 0, function () { 48 | return __generator(this, function (_a) { 49 | switch (_a.label) { 50 | case 0: return [4 /*yield*/, producer.connect()]; 51 | case 1: 52 | _a.sent(); 53 | console.log('producer connected.'); 54 | return [2 /*return*/]; 55 | } 56 | }); 57 | }); }; 58 | connect(); 59 | exports.produceAtRate = function (topic, rate) { 60 | return setInterval(function () { return exports.produce(topic); }, Math.floor(1000 / rate)); 61 | }; 62 | exports.produce = function (topic) { 63 | return new Promise(function (resolve, reject) { return __awaiter(void 0, void 0, void 0, function () { 64 | var message, stringMessage, err_1; 65 | return __generator(this, function (_a) { 66 | switch (_a.label) { 67 | case 0: 68 | _a.trys.push([0, 2, , 3]); 69 | message = { 70 | key: 'key' + count++, 71 | value: { 72 | name: 'me', 73 | quote: 'hello', 74 | image_url: 'https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/joypixels/257/smiling-face-with-sunglasses_1f60e.png' 75 | } 76 | }; 77 | stringMessage = { 78 | key: message.key, 79 | value: JSON.stringify(message.value) 80 | }; 81 | return [4 /*yield*/, producer.send({ 82 | topic: topic, 83 | messages: [stringMessage] 84 | })]; 85 | case 1: 86 | _a.sent(); 87 | resolve(message); 88 | return [3 /*break*/, 3]; 89 | case 2: 90 | err_1 = _a.sent(); 91 | reject(new Error(err_1)); 92 | return [3 /*break*/, 3]; 93 | case 3: return [2 /*return*/]; 94 | } 95 | }); 96 | }); }); 97 | }; 98 | exports.produceAtRateToPartition = function (topic, partition, rate) { 99 | if (partition === void 0) { partition = 0; } 100 | return setInterval(function () { return exports.produceToPartition(topic, partition); }, Math.floor(1000 / rate)); 101 | }; 102 | exports.produceToPartition = function (topic, partition) { 103 | if (partition === void 0) { partition = 0; } 104 | return __awaiter(void 0, void 0, void 0, function () { 105 | var message, err_2; 106 | return __generator(this, function (_a) { 107 | switch (_a.label) { 108 | case 0: 109 | message = { 110 | value: 'Besik', 111 | partition: partition 112 | }; 113 | _a.label = 1; 114 | case 1: 115 | _a.trys.push([1, 3, , 4]); 116 | return [4 /*yield*/, producer.send({ topic: topic, messages: [message] })]; 117 | case 2: 118 | _a.sent(); 119 | return [3 /*break*/, 4]; 120 | case 3: 121 | err_2 = _a.sent(); 122 | console.error('error sending to specific partition', err_2); 123 | return [3 /*break*/, 4]; 124 | case 4: return [2 /*return*/]; 125 | } 126 | }); 127 | }); 128 | }; 129 | //# sourceMappingURL=producer.js.map -------------------------------------------------------------------------------- /testbed/testbed/producer/producer.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"producer.js","sourceRoot":"","sources":["producer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAAyC;AAEzC,IAAM,KAAK,GAAG,IAAI,eAAK,CAAC;IACtB,QAAQ,EAAE,eAAe;IACzB,OAAO,EAAE,CAAC,YAAY,CAAC;CACxB,CAAC,CAAC;AAEH,IAAI,KAAK,GAAG,CAAC,CAAC;AAEd,IAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;AAalC,IAAM,OAAO,GAAG;;;oBACd,qBAAM,QAAQ,CAAC,OAAO,EAAE,EAAA;;gBAAxB,SAAwB,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;;;;KACpC,CAAC;AACF,OAAO,EAAE,CAAC;AAEG,QAAA,aAAa,GAAG,UAAC,KAAa,EAAE,IAAY;IACvD,OAAA,WAAW,CAAC,cAAM,OAAA,eAAO,CAAC,KAAK,CAAC,EAAd,CAAc,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;AAA1D,CAA0D,CAAC;AAEhD,QAAA,OAAO,GAAG,UAAC,KAAa;IACnC,OAAO,IAAI,OAAO,CAAC,UAAO,OAAO,EAAE,MAAM;;;;;;oBAE/B,OAAO,GAAoB;wBAC/B,GAAG,EAAE,KAAK,GAAG,KAAK,EAAE;wBACpB,KAAK,EAAE;4BACL,IAAI,EAAE,IAAI;4BACV,KAAK,EAAE,OAAO;4BACd,SAAS,EACP,4HAA4H;yBAC/H;qBACF,CAAC;oBACI,aAAa,GAAY;wBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;wBAChB,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;qBACrC,CAAC;oBAEF,qBAAM,QAAQ,CAAC,IAAI,CAAC;4BAClB,KAAK,EAAE,KAAK;4BACZ,QAAQ,EAAE,CAAC,aAAa,CAAC;yBAC1B,CAAC,EAAA;;oBAHF,SAGE,CAAC;oBACH,OAAO,CAAC,OAAO,CAAC,CAAC;;;;oBAEjB,MAAM,CAAC,IAAI,KAAK,CAAC,KAAG,CAAC,CAAC,CAAC;;;;;SAE1B,CAAC,CAAC;AACL,CAAC,CAAC;AAEW,QAAA,wBAAwB,GAAG,UAAC,KAAa,EAAE,SAAqB,EAAE,IAAY;IAAnC,0BAAA,EAAA,aAAqB;IAC3E,OAAO,WAAW,CAAC,cAAM,OAAA,0BAAkB,CAAC,KAAK,EAAE,SAAS,CAAC,EAApC,CAAoC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;AAC1F,CAAC,CAAC;AACW,QAAA,kBAAkB,GAAG,UAAO,KAAa,EAAE,SAAqB;IAArB,0BAAA,EAAA,aAAqB;;;;;;oBACrE,OAAO,GAAY;wBACvB,KAAK,EAAE,OAAO;wBACd,SAAS,WAAA;qBACV,CAAC;;;;oBAEA,qBAAM,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,OAAA,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,EAAA;;oBAAnD,SAAmD,CAAC;;;;oBAEpD,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,KAAG,CAAC,CAAC;;;;;;CAE7D,CAAC"} -------------------------------------------------------------------------------- /testbed/testbed/producer/producer.ts: -------------------------------------------------------------------------------- 1 | import { Kafka, Message } from 'kafkajs'; 2 | 3 | const kafka = new Kafka({ 4 | clientId: 'test-producer', 5 | brokers: ['kafka:9092'], 6 | }); 7 | 8 | let count = 0; 9 | 10 | const producer = kafka.producer(); 11 | export interface ProduceRequest { 12 | messagesCount: number; 13 | } 14 | 15 | export interface ProduceResponse { 16 | key: string; 17 | value: { 18 | name: string; 19 | quote: string; 20 | image_url: string; 21 | }; 22 | } 23 | const connect = async () => { 24 | await producer.connect(); 25 | console.log('producer connected.'); 26 | }; 27 | connect(); 28 | 29 | export const produceAtRate = (topic: string, rate: number) => 30 | setInterval(() => produce(topic), Math.floor(1000 / rate)); 31 | 32 | export const produce = (topic: string): Promise => { 33 | return new Promise(async (resolve, reject) => { 34 | try { 35 | const message: ProduceResponse = { 36 | key: 'key' + count++, 37 | value: { 38 | name: 'me', 39 | quote: 'hello', 40 | image_url: 41 | 'https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/joypixels/257/smiling-face-with-sunglasses_1f60e.png', 42 | }, 43 | }; 44 | const stringMessage: Message = { 45 | key: message.key, 46 | value: JSON.stringify(message.value), 47 | }; 48 | 49 | await producer.send({ 50 | topic: topic, 51 | messages: [stringMessage], 52 | }); 53 | resolve(message); 54 | } catch (err) { 55 | reject(new Error(err)); 56 | } 57 | }); 58 | }; 59 | 60 | export const produceAtRateToPartition = (topic: string, partition: number = 0, rate: number) => { 61 | return setInterval(() => produceToPartition(topic, partition), Math.floor(1000 / rate)); 62 | }; 63 | export const produceToPartition = async (topic: string, partition: number = 0) => { 64 | const message: Message = { 65 | value: 'Besik', 66 | partition, 67 | }; 68 | try { 69 | await producer.send({ topic, messages: [message] }); 70 | } catch (err) { 71 | console.error('error sending to specific partition', err); 72 | } 73 | }; 74 | -------------------------------------------------------------------------------- /testbed/testbed/server.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | var express = require("express"); 40 | var path = require("path"); 41 | var app = express(); 42 | var http_1 = require("http"); 43 | var server = http_1.createServer(app); 44 | var io = require('socket.io')(server); 45 | var socket_1 = require("./types/socket"); 46 | var producer_1 = require("./producer/producer"); 47 | var consumer_1 = require("./consumer/consumer"); 48 | var topics_1 = require("./topics/topics"); 49 | io.on('connection', function () { return console.log('Socket server connected...'); }); 50 | io.on('connect', function (socket) { 51 | socket.on(socket_1.SocketNames.produceNum, function (data) { 52 | // should run the produce function data.messagesCount amount of times 53 | var messagesCount = data.messagesCount, topic = data.topic; 54 | topic = topic === '' ? 'test-topic' : topic; 55 | while (messagesCount) { 56 | // should send back the data on completion of each to show on the client side 57 | producer_1.produce(topic) 58 | .then(function (resp) { return socket.emit('produceResponse', resp); })["catch"](console.error); 59 | messagesCount--; 60 | } 61 | }); 62 | socket.on(socket_1.SocketNames.produceRate, function (data) { 63 | var topic = data.topic, rate = data.rate; 64 | producer_1.produceAtRate(topic, rate); 65 | }); 66 | socket.on(socket_1.SocketNames.consumeRate, function (data) { 67 | var rate = data.rate, topic = data.topic; 68 | consumer_1.consumeAtRate(socket, rate, topic); 69 | }); 70 | socket.on(socket_1.SocketNames.overloadPartition, function (data) { return __awaiter(void 0, void 0, void 0, function () { 71 | var topic, numPartitions, messages; 72 | return __generator(this, function (_a) { 73 | switch (_a.label) { 74 | case 0: 75 | topic = data.topic, numPartitions = data.numPartitions; 76 | messages = data.messages; 77 | return [4 /*yield*/, topics_1.createTopicWithMultiplePartitions(topic, numPartitions)]; 78 | case 1: 79 | _a.sent(); 80 | while (messages) { 81 | // we can add in custom partitioning logic here, if needed 82 | // right now, this will overload partition 0 by default 83 | producer_1.produceToPartition(topic); 84 | messages -= 1; 85 | } 86 | return [2 /*return*/]; 87 | } 88 | }); 89 | }); }); 90 | socket.on(socket_1.SocketNames.underReplicate, function (data) { 91 | var topic = data.topic, numReplicants = data.numReplicants; 92 | // any number of replicants greater than the number of brokers will cause an under-replication event 93 | // default number of brokers is 1 94 | // default number of replicants is 2 95 | topics_1.createTopicWithMultipleReplicants(topic, numReplicants); 96 | }); 97 | socket.on(socket_1.SocketNames.createTopic, function (data) { 98 | var topic = data.topic; 99 | topics_1.createTopic(topic); 100 | }); 101 | socket.on(socket_1.SocketNames.consumeAll, function (data) { 102 | var topic = data.topic; 103 | console.log('consumeAll called with topic', topic); 104 | consumer_1.consume(socket, topic !== '' ? topic : undefined); 105 | }); 106 | }); 107 | var PORT = 2022; 108 | app.get('/main.css', function (req, res) { return res.sendFile(path.resolve(__dirname, 'main.css')); }); 109 | app.get('/main.js', function (req, res) { return res.sendFile(path.resolve(__dirname, '../test-front/main.js')); }); 110 | app.get('/', function (req, res) { return res.sendFile(path.resolve(__dirname, './index.html')); }); 111 | server.listen(PORT, function () { return console.log("Server listening on port " + PORT); }); 112 | //# sourceMappingURL=server.js.map -------------------------------------------------------------------------------- /testbed/testbed/server.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"server.js","sourceRoot":"","sources":["server.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iCAAmC;AACnC,2BAA6B;AAC7B,IAAM,GAAG,GAAG,OAAO,EAAE,CAAC;AAEtB,6BAAoC;AACpC,IAAM,MAAM,GAAG,mBAAY,CAAC,GAAG,CAAC,CAAC;AACjC,IAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAExC,yCAA6D;AAE7D,gDAM6B;AAC7B,gDAA6D;AAC7D,0CAIyB;AAEzB,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,cAAM,OAAA,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,EAAzC,CAAyC,CAAC,CAAC;AACrE,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,UAAC,MAAW;IAC3B,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,UAAU,EAAE,UAAC,IAA+B;QAChE,qEAAqE;QAC/D,IAAA,aAAa,GAAY,IAAI,cAAhB,EAAE,KAAK,GAAK,IAAI,MAAT,CAAU;QACpC,KAAK,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5C,OAAO,aAAa,EAAE;YACpB,6EAA6E;YAC7E,kBAAO,CAAC,KAAK,CAAC;iBACX,IAAI,CAAC,UAAC,IAAqB,IAAK,OAAA,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,EAApC,CAAoC,CAAC,CACrE,OAAK,CAAA,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACxB,aAAa,EAAE,CAAC;SACjB;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,WAAW,EAAE,UAAC,IAAgC;QAC1D,IAAA,KAAK,GAAW,IAAI,MAAf,EAAE,IAAI,GAAK,IAAI,KAAT,CAAU;QAC7B,wBAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,WAAW,EAAE,UAAC,IAAgC;QAC1D,IAAA,IAAI,GAAY,IAAI,KAAhB,EAAE,KAAK,GAAK,IAAI,MAAT,CAAU;QAC7B,wBAAa,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,iBAAiB,EAAE,UAAO,IAAsC;;;;;oBAC5E,KAAK,GAAoB,IAAI,MAAxB,EAAE,aAAa,GAAK,IAAI,cAAT,CAAU;oBAChC,QAAQ,GAAK,IAAI,SAAT,CAAU;oBACxB,qBAAM,0CAAiC,CAAC,KAAK,EAAE,aAAa,CAAC,EAAA;;oBAA7D,SAA6D,CAAC;oBAC9D,OAAO,QAAQ,EAAE;wBACf,0DAA0D;wBAC1D,uDAAuD;wBACvD,6BAAkB,CAAC,KAAK,CAAC,CAAC;wBAC1B,QAAQ,IAAI,CAAC,CAAC;qBACf;;;;SACF,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,cAAc,EAAE,UAAC,IAAmC;QAChE,IAAA,KAAK,GAAoB,IAAI,MAAxB,EAAE,aAAa,GAAK,IAAI,cAAT,CAAU;QACtC,oGAAoG;QACpG,iCAAiC;QACjC,oCAAoC;QACpC,0CAAiC,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,WAAW,EAAE,UAAC,IAAgC;QAC1D,IAAA,KAAK,GAAK,IAAI,MAAT,CAAU;QACvB,oBAAW,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,EAAE,CAAC,oBAAW,CAAC,UAAU,EAAE,UAAC,IAA+B;QACxD,IAAA,KAAK,GAAK,IAAI,MAAT,CAAU;QACvB,OAAO,CAAC,GAAG,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC;QACnD,kBAAO,CAAC,MAAM,EAAE,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACH,IAAM,IAAI,GAAG,IAAI,CAAC;AAElB,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,UAAC,GAAG,EAAE,GAAG,IAAK,OAAA,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,EAAjD,CAAiD,CAAC,CAAC;AACtF,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,UAAC,GAAG,EAAE,GAAG,IAAK,OAAA,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC,EAA9D,CAA8D,CAAC,CAAC;AAClG,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,UAAC,GAAG,EAAE,GAAG,IAAK,OAAA,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,EAArD,CAAqD,CAAC,CAAC;AAElF,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAM,OAAA,OAAO,CAAC,GAAG,CAAC,8BAA4B,IAAM,CAAC,EAA/C,CAA+C,CAAC,CAAC"} -------------------------------------------------------------------------------- /testbed/testbed/server.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | import * as path from 'path'; 3 | const app = express(); 4 | 5 | import { createServer } from 'http'; 6 | const server = createServer(app); 7 | const io = require('socket.io')(server); 8 | 9 | import { SocketNames, SocketMessages } from './types/socket'; 10 | 11 | import { 12 | produce, 13 | produceAtRate, 14 | ProduceRequest, 15 | ProduceResponse, 16 | produceToPartition, 17 | } from './producer/producer'; 18 | import { consume, consumeAtRate } from './consumer/consumer'; 19 | import { 20 | createTopicWithMultiplePartitions, 21 | createTopicWithMultipleReplicants, 22 | createTopic, 23 | } from './topics/topics'; 24 | 25 | io.on('connection', () => console.log('Socket server connected...')); 26 | io.on('connect', (socket: any) => { 27 | socket.on(SocketNames.produceNum, (data: SocketMessages.produceNum) => { 28 | // should run the produce function data.messagesCount amount of times 29 | let { messagesCount, topic } = data; 30 | topic = topic === '' ? 'test-topic' : topic; 31 | let remaining = messagesCount; 32 | while (remaining) { 33 | // should send back the data on completion of each to show on the client side 34 | produce(topic) 35 | .then((resp: ProduceResponse) => 36 | socket.emit('produceResponse', resp, Math.floor(1 - remaining / messagesCount) * 100) 37 | ) 38 | .catch(console.error); 39 | messagesCount--; 40 | } 41 | }); 42 | 43 | socket.on(SocketNames.produceRate, (data: SocketMessages.produceRate) => { 44 | const { topic, rate } = data; 45 | produceAtRate(topic, rate); 46 | }); 47 | 48 | socket.on(SocketNames.consumeRate, (data: SocketMessages.consumeRate) => { 49 | const { rate, topic } = data; 50 | consumeAtRate(socket, rate, topic); 51 | }); 52 | 53 | socket.on( 54 | SocketNames.overloadPartition, 55 | async (data: SocketMessages.overloadPartition) => { 56 | const { topic, numPartitions } = data; 57 | let { messages } = data; 58 | await createTopicWithMultiplePartitions(topic, numPartitions); 59 | while (messages) { 60 | // we can add in custom partitioning logic here, if needed 61 | // right now, this will overload partition 0 by default 62 | produceToPartition(topic); 63 | messages -= 1; 64 | } 65 | } 66 | ); 67 | 68 | socket.on( 69 | SocketNames.underReplicate, 70 | (data: SocketMessages.underReplicate) => { 71 | const { topic, numReplicants } = data; 72 | // any number of replicants greater than the number of brokers will cause an under-replication event 73 | // default number of brokers is 1 74 | // default number of replicants is 2 75 | createTopicWithMultipleReplicants(topic, numReplicants); 76 | } 77 | ); 78 | 79 | socket.on(SocketNames.createTopic, (data: SocketMessages.createTopic) => { 80 | const { topic } = data; 81 | createTopic(topic); 82 | }); 83 | socket.on(SocketNames.consumeAll, (data: SocketMessages.consumeAll) => { 84 | const { topic } = data; 85 | console.log('consumeAll called with topic', topic); 86 | consume(socket, topic !== '' ? topic : undefined); 87 | }); 88 | }); 89 | const PORT = 2022; 90 | 91 | app.get('/main.css', (req, res) => 92 | res.sendFile(path.resolve(__dirname, 'main.css')) 93 | ); 94 | app.get('/main.js', (req, res) => 95 | res.sendFile(path.resolve(__dirname, '../test-front/main.js')) 96 | ); 97 | app.get('/', (req, res) => 98 | res.sendFile(path.resolve(__dirname, './index.html')) 99 | ); 100 | 101 | export = server.listen(PORT, () => 102 | console.log(`Server listening on port ${PORT}`) 103 | ); 104 | -------------------------------------------------------------------------------- /testbed/testbed/topics/topics.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | exports.createTopicWithMultipleReplicants = exports.createTopicWithMultiplePartitions = exports.createTopic = void 0; 40 | var kafkajs_1 = require("kafkajs"); 41 | var kafka = new kafkajs_1.Kafka({ 42 | clientId: 'test-client', 43 | brokers: ['kafka:9092'] 44 | }); 45 | var admin = kafka.admin(); 46 | var connect = function () { return __awaiter(void 0, void 0, void 0, function () { 47 | return __generator(this, function (_a) { 48 | switch (_a.label) { 49 | case 0: return [4 /*yield*/, admin.connect()]; 50 | case 1: 51 | _a.sent(); 52 | console.log('Admin client connected.'); 53 | return [2 /*return*/]; 54 | } 55 | }); 56 | }); }; 57 | connect(); 58 | exports.createTopic = function (topic, numPartitions, replicationFactor) { 59 | if (topic === void 0) { topic = 'test-topic'; } 60 | if (numPartitions === void 0) { numPartitions = 1; } 61 | if (replicationFactor === void 0) { replicationFactor = 1; } 62 | return __awaiter(void 0, void 0, void 0, function () { 63 | var topicConfig, newTopicCreated, err_1; 64 | return __generator(this, function (_a) { 65 | switch (_a.label) { 66 | case 0: 67 | topicConfig = { 68 | topic: topic, 69 | numPartitions: numPartitions, 70 | replicationFactor: replicationFactor 71 | }; 72 | _a.label = 1; 73 | case 1: 74 | _a.trys.push([1, 3, , 4]); 75 | return [4 /*yield*/, admin.createTopics({ 76 | topics: [topicConfig] 77 | })]; 78 | case 2: 79 | newTopicCreated = _a.sent(); 80 | console.log(newTopicCreated 81 | ? "Topic with name " + topic + " was created." 82 | : "Topic with name " + topic + " already exists."); 83 | return [3 /*break*/, 4]; 84 | case 3: 85 | err_1 = _a.sent(); 86 | console.error(err_1); 87 | return [3 /*break*/, 4]; 88 | case 4: return [2 /*return*/]; 89 | } 90 | }); 91 | }); 92 | }; 93 | // Create a topic with multiple partitions 94 | exports.createTopicWithMultiplePartitions = function (topic, numPartitions) { 95 | return exports.createTopic(topic, numPartitions); 96 | }; 97 | // Create a topic with multiple replicants 98 | exports.createTopicWithMultipleReplicants = function (topic, numReplicants) { 99 | if (numReplicants === void 0) { numReplicants = 2; } 100 | return exports.createTopic(topic, undefined, numReplicants); 101 | }; 102 | //# sourceMappingURL=topics.js.map -------------------------------------------------------------------------------- /testbed/testbed/topics/topics.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"topics.js","sourceRoot":"","sources":["topics.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAA8C;AAE9C,IAAM,KAAK,GAAG,IAAI,eAAK,CAAC;IACtB,QAAQ,EAAE,aAAa;IACvB,OAAO,EAAE,CAAC,YAAY,CAAC;CACxB,CAAC,CAAC;AAEH,IAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;AAE5B,IAAM,OAAO,GAAG;;;oBACd,qBAAM,KAAK,CAAC,OAAO,EAAE,EAAA;;gBAArB,SAAqB,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;;;;KACxC,CAAC;AAEF,OAAO,EAAE,CAAC;AAEG,QAAA,WAAW,GAAG,UACzB,KAA4B,EAC5B,aAAyB,EACzB,iBAA6B;IAF7B,sBAAA,EAAA,oBAA4B;IAC5B,8BAAA,EAAA,iBAAyB;IACzB,kCAAA,EAAA,qBAA6B;;;;;;oBAEvB,WAAW,GAAiB;wBAChC,KAAK,OAAA;wBACL,aAAa,eAAA;wBACb,iBAAiB,mBAAA;qBAClB,CAAC;;;;oBAEwB,qBAAM,KAAK,CAAC,YAAY,CAAC;4BAC/C,MAAM,EAAE,CAAC,WAAW,CAAC;yBACtB,CAAC,EAAA;;oBAFI,eAAe,GAAG,SAEtB;oBACF,OAAO,CAAC,GAAG,CACT,eAAe;wBACb,CAAC,CAAC,qBAAmB,KAAK,kBAAe;wBACzC,CAAC,CAAC,qBAAmB,KAAK,qBAAkB,CAC/C,CAAC;;;;oBAEF,OAAO,CAAC,KAAK,CAAC,KAAG,CAAC,CAAC;;;;;;CAEtB,CAAC;AAEF,0CAA0C;AAC7B,QAAA,iCAAiC,GAAG,UAAC,KAAa,EAAE,aAAqB;IACpF,OAAA,mBAAW,CAAC,KAAK,EAAE,aAAa,CAAC;AAAjC,CAAiC,CAAC;AAEpC,0CAA0C;AAC7B,QAAA,iCAAiC,GAAG,UAAC,KAAa,EAAE,aAAyB;IAAzB,8BAAA,EAAA,iBAAyB;IACxF,OAAA,mBAAW,CAAC,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC;AAA5C,CAA4C,CAAC"} -------------------------------------------------------------------------------- /testbed/testbed/topics/topics.ts: -------------------------------------------------------------------------------- 1 | import { Kafka, ITopicConfig } from 'kafkajs'; 2 | 3 | const kafka = new Kafka({ 4 | clientId: 'test-client', 5 | brokers: ['kafka:9092'], 6 | }); 7 | 8 | const admin = kafka.admin(); 9 | 10 | const connect = async () => { 11 | await admin.connect(); 12 | console.log('Admin client connected.'); 13 | }; 14 | 15 | connect(); 16 | 17 | export const createTopic = async ( 18 | topic: string = 'test-topic', 19 | numPartitions: number = 1, 20 | replicationFactor: number = 1 21 | ): Promise => { 22 | const topicConfig: ITopicConfig = { 23 | topic, 24 | numPartitions, 25 | replicationFactor, 26 | }; 27 | try { 28 | const newTopicCreated = await admin.createTopics({ 29 | topics: [topicConfig], 30 | }); 31 | console.log( 32 | newTopicCreated 33 | ? `Topic with name ${topic} was created.` 34 | : `Topic with name ${topic} already exists.` 35 | ); 36 | } catch (err) { 37 | console.error(err); 38 | } 39 | }; 40 | 41 | // Create a topic with multiple partitions 42 | export const createTopicWithMultiplePartitions = (topic: string, numPartitions: number) => 43 | createTopic(topic, numPartitions); 44 | 45 | // Create a topic with multiple replicants 46 | export const createTopicWithMultipleReplicants = (topic: string, numReplicants: number = 2) => 47 | createTopic(topic, undefined, numReplicants); 48 | -------------------------------------------------------------------------------- /testbed/testbed/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": true, 4 | "sourceMap": true, 5 | "module": "commonjs" 6 | }, 7 | "include": ["*.ts"], 8 | "exclude": ["/types/"] 9 | } 10 | -------------------------------------------------------------------------------- /testbed/testbed/types/socket.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | exports.__esModule = true; 3 | exports.SocketNames = void 0; 4 | exports.SocketNames = { 5 | produceNum: 'produceNum', 6 | produceRate: 'produceRate', 7 | consumeRate: 'consumeRate', 8 | consumeAll: 'consumeAll', 9 | overloadPartition: 'overloadPartition', 10 | createTopic: 'createTopic', 11 | underReplicate: 'underReplicate' 12 | }; 13 | //# sourceMappingURL=socket.js.map -------------------------------------------------------------------------------- /testbed/testbed/types/socket.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"socket.js","sourceRoot":"","sources":["socket.ts"],"names":[],"mappings":";;;AASa,QAAA,WAAW,GAAgB;IACtC,UAAU,EAAE,YAAY;IACxB,WAAW,EAAE,aAAa;IAC1B,WAAW,EAAE,aAAa;IAC1B,UAAU,EAAE,YAAY;IACxB,iBAAiB,EAAE,mBAAmB;IACtC,WAAW,EAAE,aAAa;IAC1B,cAAc,EAAE,gBAAgB;CACjC,CAAC"} -------------------------------------------------------------------------------- /testbed/testbed/types/socket.ts: -------------------------------------------------------------------------------- 1 | interface SocketNames { 2 | produceNum: 'produceNum'; 3 | produceRate: 'produceRate'; 4 | consumeRate: 'consumeRate'; 5 | consumeAll: 'consumeAll'; 6 | overloadPartition: 'overloadPartition'; 7 | createTopic: 'createTopic'; 8 | underReplicate: 'underReplicate'; 9 | } 10 | export const SocketNames: SocketNames = { 11 | produceNum: 'produceNum', 12 | produceRate: 'produceRate', 13 | consumeRate: 'consumeRate', 14 | consumeAll: 'consumeAll', 15 | overloadPartition: 'overloadPartition', 16 | createTopic: 'createTopic', 17 | underReplicate: 'underReplicate', 18 | }; 19 | 20 | export namespace SocketMessages { 21 | export type produceNum = { messagesCount: number; topic: string }; 22 | 23 | export type produceRate = { 24 | rate: number; 25 | topic: string; 26 | }; 27 | export type consumeRate = { 28 | rate: number; 29 | topic: string; 30 | }; 31 | 32 | export type overloadPartition = { 33 | topic: string; 34 | numPartitions: number; 35 | messages: number; 36 | }; 37 | 38 | export type createTopic = { 39 | topic: string; 40 | }; 41 | 42 | export type consumeAll = { 43 | topic: string; 44 | }; 45 | export type underReplicate = { 46 | topic: string; 47 | numReplicants: number; 48 | }; 49 | } 50 | -------------------------------------------------------------------------------- /testbed/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | mode: 'development', 5 | entry: './main.ts', 6 | output: { 7 | filename: 'main.js', 8 | path: path.resolve(__dirname), 9 | }, 10 | module: { 11 | rules: [ 12 | { 13 | test: /\.tsx?$/, 14 | use: 'ts-loader', 15 | exclude: /node_modules/, 16 | }, 17 | ], 18 | }, 19 | resolve: { 20 | extensions: ['.tsx', '.ts', '.js'], 21 | }, 22 | }; 23 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./Build/", 4 | "noImplicitAny": true, 5 | "module": "es6", 6 | "target": "es5", 7 | "jsx": "react", 8 | "allowJs": true 9 | } 10 | } -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | mode: process.env.NODE_ENV, 5 | entry: './App/Main.ts', 6 | output: { 7 | filename: 'Main.js', 8 | path: path.resolve(__dirname, 'Build'), 9 | }, 10 | module: { 11 | rules: [ 12 | { 13 | test: /\.tsx?$/, 14 | use: 'ts-loader', 15 | exclude: /node_modules/, 16 | }, 17 | ], 18 | }, 19 | resolve: { 20 | extensions: [ '.tsx', '.ts', '.js' ], 21 | }, 22 | }; --------------------------------------------------------------------------------