├── .DS_Store
├── .babelrc
├── .github
└── ISSUE_TEMPLATE
│ ├── bug_report.md
│ └── feature_request.md
├── .gitignore
├── CODE_OF_CONDUCT.md
├── LICENSE
├── README.md
├── __tests__
├── socketIOtest.js
└── supertest.js
├── client
├── App.jsx
├── assets
│ ├── CogIcon.jsx
│ ├── GraphIcon.jsx
│ ├── HomeIcon.jsx
│ ├── KafkaPeekLogo.jsx
│ ├── Logout.jsx
│ ├── UserIcon.jsx
│ └── images
│ │ ├── kafkaPeek-demo.png
│ │ └── kafkaPeek-logo.png
├── components
│ ├── HealthBinBout.jsx
│ ├── HealthJVM.jsx
│ ├── HealthMsgIn.jsx
│ ├── PerfReqPerSec.jsx
│ ├── PerfReqTotalTime.jsx
│ ├── PerfResQueueTime.jsx
│ ├── PerfResSendTime.jsx
│ ├── StaticMetricDisplay.jsx
│ └── notificationComponents
│ │ ├── BytesInPer.jsx
│ │ ├── BytesOutPer.jsx
│ │ ├── OfflinePart.jsx
│ │ ├── OngoingMetrics.jsx
│ │ └── UnderRep.jsx
├── containers
│ ├── Header.jsx
│ ├── HealthDashboard.jsx
│ ├── Landing.jsx
│ ├── Notifications.jsx
│ ├── PerfDashboard.jsx
│ └── Sidebar.jsx
├── database
│ └── db.json
├── index.html
├── index.js
└── styles
│ ├── _dashboard.scss
│ ├── _header.scss
│ ├── _landing.scss
│ ├── _login.scss
│ ├── _notifications.scss
│ ├── _ongoingMetrics.scss
│ ├── _sidebar.scss
│ └── application.scss
├── consumer
└── index.js
├── docker-compose.yml
├── etc
├── jmx_exporter
│ └── config.yml
└── prometheus
│ └── prometheus.yml
├── grafana
├── dashboards
│ ├── grafana_dashboard_broker_hard_disk_usage.json
│ ├── grafana_dashboard_broker_jvm_os.json
│ ├── grafana_dashboard_broker_performance.json
│ ├── grafana_dashboard_broker_zookeeper.json
│ ├── grafana_dashboard_client_consumers_fetch_lag.json
│ ├── grafana_dashboard_cluster_healthcheck.json
│ ├── grafana_dashboard_cluster_replication.json
│ └── grafana_dashboard_topics_logs.json
└── provisioning
│ ├── dashboards
│ └── kafka.yaml
│ └── datasources
│ └── datasource.yaml
├── index.html
├── jest.config.js
├── package-lock.json
├── package.json
├── producer
└── index.js
├── server
├── controllers
│ ├── oauthController.js
│ ├── slackController.js
│ └── utils.js
├── queries.js
├── router
│ └── routes.js
└── server.js
├── test-teardown-globals.js
└── webpack.config.js
/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/KafkaPeek/8e7529f6375fb8170dc300f5e38eac608c9efc13/.DS_Store
--------------------------------------------------------------------------------
/.babelrc:
--------------------------------------------------------------------------------
1 | {
2 | presets: ["@babel/preset-env", "@babel/preset-react"]
3 | }
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Bug report
3 | about: Create a report to help us improve
4 | title: ''
5 | labels: ''
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Describe the bug**
11 | A clear and concise description of what the bug is.
12 |
13 | **To Reproduce**
14 | Steps to reproduce the behavior:
15 | 1. Go to '...'
16 | 2. Click on '....'
17 | 3. Scroll down to '....'
18 | 4. See error
19 |
20 | **Expected behavior**
21 | A clear and concise description of what you expected to happen.
22 |
23 | **Screenshots**
24 | If applicable, add screenshots to help explain your problem.
25 |
26 | **Desktop (please complete the following information):**
27 | - OS: [e.g. iOS]
28 | - Browser [e.g. chrome, safari]
29 | - Version [e.g. 22]
30 |
31 | **Smartphone (please complete the following information):**
32 | - Device: [e.g. iPhone6]
33 | - OS: [e.g. iOS8.1]
34 | - Browser [e.g. stock browser, safari]
35 | - Version [e.g. 22]
36 |
37 | **Additional context**
38 | Add any other context about the problem here.
39 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Feature request
3 | about: Suggest an idea for this project
4 | title: ''
5 | labels: ''
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12 |
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 |
16 | **Describe alternatives you've considered**
17 | A clear and concise description of any alternative solutions or features you've considered.
18 |
19 | **Additional context**
20 | Add any other context or screenshots about the feature request here.
21 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 | lerna-debug.log*
8 |
9 | # Diagnostic reports (https://nodejs.org/api/report.html)
10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
11 |
12 | # Runtime data
13 | pids
14 | *.pid
15 | *.seed
16 | *.pid.lock
17 |
18 | # Directory for instrumented libs generated by jscoverage/JSCover
19 | lib-cov
20 |
21 | # Coverage directory used by tools like istanbul
22 | coverage
23 | *.lcov
24 |
25 | # nyc test coverage
26 | .nyc_output
27 |
28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
29 | .grunt
30 |
31 | # Bower dependency directory (https://bower.io/)
32 | bower_components
33 |
34 | # node-waf configuration
35 | .lock-wscript
36 |
37 | # Compiled binary addons (https://nodejs.org/api/addons.html)
38 | build/Release
39 |
40 | # Dependency directories
41 | node_modules/
42 | jspm_packages/
43 |
44 | # TypeScript v1 declaration files
45 | typings/
46 |
47 | # TypeScript cache
48 | *.tsbuildinfo
49 |
50 | # Optional npm cache directory
51 | .npm
52 |
53 | # Optional eslint cache
54 | .eslintcache
55 |
56 | # Microbundle cache
57 | .rpt2_cache/
58 | .rts2_cache_cjs/
59 | .rts2_cache_es/
60 | .rts2_cache_umd/
61 |
62 | # Optional REPL history
63 | .node_repl_history
64 |
65 | # Output of 'npm pack'
66 | *.tgz
67 |
68 | # Yarn Integrity file
69 | .yarn-integrity
70 |
71 | # dotenv environment variables file
72 | .env
73 | .env.test
74 |
75 | # parcel-bundler cache (https://parceljs.org/)
76 | .cache
77 |
78 | # Next.js build output
79 | .next
80 |
81 | # Nuxt.js build / generate output
82 | .nuxt
83 | dist
84 |
85 | # Gatsby files
86 | .cache/
87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js
88 | # https://nextjs.org/blog/next-9-1#public-directory-support
89 | # public
90 |
91 | # vuepress build output
92 | .vuepress/dist
93 |
94 | # Serverless directories
95 | .serverless/
96 |
97 | # FuseBox cache
98 | .fusebox/
99 |
100 | # DynamoDB Local files
101 | .dynamodb/
102 |
103 | # TernJS port file
104 | .tern-port
105 |
106 | #Security risk
107 | .DS_store
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Covenant Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | We as members, contributors, and leaders pledge to make participation in our
6 | community a harassment-free experience for everyone, regardless of age, body
7 | size, visible or invisible disability, ethnicity, sex characteristics, gender
8 | identity and expression, level of experience, education, socio-economic status,
9 | nationality, personal appearance, race, religion, or sexual identity
10 | and orientation.
11 |
12 | We pledge to act and interact in ways that contribute to an open, welcoming,
13 | diverse, inclusive, and healthy community.
14 |
15 | ## Our Standards
16 |
17 | Examples of behavior that contributes to a positive environment for our
18 | community include:
19 |
20 | * Demonstrating empathy and kindness toward other people
21 | * Being respectful of differing opinions, viewpoints, and experiences
22 | * Giving and gracefully accepting constructive feedback
23 | * Accepting responsibility and apologizing to those affected by our mistakes,
24 | and learning from the experience
25 | * Focusing on what is best not just for us as individuals, but for the
26 | overall community
27 |
28 | Examples of unacceptable behavior include:
29 |
30 | * The use of sexualized language or imagery, and sexual attention or
31 | advances of any kind
32 | * Trolling, insulting or derogatory comments, and personal or political attacks
33 | * Public or private harassment
34 | * Publishing others' private information, such as a physical or email
35 | address, without their explicit permission
36 | * Other conduct which could reasonably be considered inappropriate in a
37 | professional setting
38 |
39 | ## Enforcement Responsibilities
40 |
41 | Community leaders are responsible for clarifying and enforcing our standards of
42 | acceptable behavior and will take appropriate and fair corrective action in
43 | response to any behavior that they deem inappropriate, threatening, offensive,
44 | or harmful.
45 |
46 | Community leaders have the right and responsibility to remove, edit, or reject
47 | comments, commits, code, wiki edits, issues, and other contributions that are
48 | not aligned to this Code of Conduct, and will communicate reasons for moderation
49 | decisions when appropriate.
50 |
51 | ## Scope
52 |
53 | This Code of Conduct applies within all community spaces, and also applies when
54 | an individual is officially representing the community in public spaces.
55 | Examples of representing our community include using an official e-mail address,
56 | posting via an official social media account, or acting as an appointed
57 | representative at an online or offline event.
58 |
59 | ## Enforcement
60 |
61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
62 | reported to the community leaders responsible for enforcement at
63 | dipennagpal@gmail.com.
64 | All complaints will be reviewed and investigated promptly and fairly.
65 |
66 | All community leaders are obligated to respect the privacy and security of the
67 | reporter of any incident.
68 |
69 | ## Enforcement Guidelines
70 |
71 | Community leaders will follow these Community Impact Guidelines in determining
72 | the consequences for any action they deem in violation of this Code of Conduct:
73 |
74 | ### 1. Correction
75 |
76 | **Community Impact**: Use of inappropriate language or other behavior deemed
77 | unprofessional or unwelcome in the community.
78 |
79 | **Consequence**: A private, written warning from community leaders, providing
80 | clarity around the nature of the violation and an explanation of why the
81 | behavior was inappropriate. A public apology may be requested.
82 |
83 | ### 2. Warning
84 |
85 | **Community Impact**: A violation through a single incident or series
86 | of actions.
87 |
88 | **Consequence**: A warning with consequences for continued behavior. No
89 | interaction with the people involved, including unsolicited interaction with
90 | those enforcing the Code of Conduct, for a specified period of time. This
91 | includes avoiding interactions in community spaces as well as external channels
92 | like social media. Violating these terms may lead to a temporary or
93 | permanent ban.
94 |
95 | ### 3. Temporary Ban
96 |
97 | **Community Impact**: A serious violation of community standards, including
98 | sustained inappropriate behavior.
99 |
100 | **Consequence**: A temporary ban from any sort of interaction or public
101 | communication with the community for a specified period of time. No public or
102 | private interaction with the people involved, including unsolicited interaction
103 | with those enforcing the Code of Conduct, is allowed during this period.
104 | Violating these terms may lead to a permanent ban.
105 |
106 | ### 4. Permanent Ban
107 |
108 | **Community Impact**: Demonstrating a pattern of violation of community
109 | standards, including sustained inappropriate behavior, harassment of an
110 | individual, or aggression toward or disparagement of classes of individuals.
111 |
112 | **Consequence**: A permanent ban from any sort of public interaction within
113 | the community.
114 |
115 | ## Attribution
116 |
117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage],
118 | version 2.0, available at
119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
120 |
121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct
122 | enforcement ladder](https://github.com/mozilla/diversity).
123 |
124 | [homepage]: https://www.contributor-covenant.org
125 |
126 | For answers to common questions about this code of conduct, see the FAQ at
127 | https://www.contributor-covenant.org/faq. Translations are available at
128 | https://www.contributor-covenant.org/translations.
129 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Mozilla Public License Version 2.0
2 | ==================================
3 |
4 | 1. Definitions
5 | --------------
6 |
7 | 1.1. "Contributor"
8 | means each individual or legal entity that creates, contributes to
9 | the creation of, or owns Covered Software.
10 |
11 | 1.2. "Contributor Version"
12 | means the combination of the Contributions of others (if any) used
13 | by a Contributor and that particular Contributor's Contribution.
14 |
15 | 1.3. "Contribution"
16 | means Covered Software of a particular Contributor.
17 |
18 | 1.4. "Covered Software"
19 | means Source Code Form to which the initial Contributor has attached
20 | the notice in Exhibit A, the Executable Form of such Source Code
21 | Form, and Modifications of such Source Code Form, in each case
22 | including portions thereof.
23 |
24 | 1.5. "Incompatible With Secondary Licenses"
25 | means
26 |
27 | (a) that the initial Contributor has attached the notice described
28 | in Exhibit B to the Covered Software; or
29 |
30 | (b) that the Covered Software was made available under the terms of
31 | version 1.1 or earlier of the License, but not also under the
32 | terms of a Secondary License.
33 |
34 | 1.6. "Executable Form"
35 | means any form of the work other than Source Code Form.
36 |
37 | 1.7. "Larger Work"
38 | means a work that combines Covered Software with other material, in
39 | a separate file or files, that is not Covered Software.
40 |
41 | 1.8. "License"
42 | means this document.
43 |
44 | 1.9. "Licensable"
45 | means having the right to grant, to the maximum extent possible,
46 | whether at the time of the initial grant or subsequently, any and
47 | all of the rights conveyed by this License.
48 |
49 | 1.10. "Modifications"
50 | means any of the following:
51 |
52 | (a) any file in Source Code Form that results from an addition to,
53 | deletion from, or modification of the contents of Covered
54 | Software; or
55 |
56 | (b) any new file in Source Code Form that contains any Covered
57 | Software.
58 |
59 | 1.11. "Patent Claims" of a Contributor
60 | means any patent claim(s), including without limitation, method,
61 | process, and apparatus claims, in any patent Licensable by such
62 | Contributor that would be infringed, but for the grant of the
63 | License, by the making, using, selling, offering for sale, having
64 | made, import, or transfer of either its Contributions or its
65 | Contributor Version.
66 |
67 | 1.12. "Secondary License"
68 | means either the GNU General Public License, Version 2.0, the GNU
69 | Lesser General Public License, Version 2.1, the GNU Affero General
70 | Public License, Version 3.0, or any later versions of those
71 | licenses.
72 |
73 | 1.13. "Source Code Form"
74 | means the form of the work preferred for making modifications.
75 |
76 | 1.14. "You" (or "Your")
77 | means an individual or a legal entity exercising rights under this
78 | License. For legal entities, "You" includes any entity that
79 | controls, is controlled by, or is under common control with You. For
80 | purposes of this definition, "control" means (a) the power, direct
81 | or indirect, to cause the direction or management of such entity,
82 | whether by contract or otherwise, or (b) ownership of more than
83 | fifty percent (50%) of the outstanding shares or beneficial
84 | ownership of such entity.
85 |
86 | 2. License Grants and Conditions
87 | --------------------------------
88 |
89 | 2.1. Grants
90 |
91 | Each Contributor hereby grants You a world-wide, royalty-free,
92 | non-exclusive license:
93 |
94 | (a) under intellectual property rights (other than patent or trademark)
95 | Licensable by such Contributor to use, reproduce, make available,
96 | modify, display, perform, distribute, and otherwise exploit its
97 | Contributions, either on an unmodified basis, with Modifications, or
98 | as part of a Larger Work; and
99 |
100 | (b) under Patent Claims of such Contributor to make, use, sell, offer
101 | for sale, have made, import, and otherwise transfer either its
102 | Contributions or its Contributor Version.
103 |
104 | 2.2. Effective Date
105 |
106 | The licenses granted in Section 2.1 with respect to any Contribution
107 | become effective for each Contribution on the date the Contributor first
108 | distributes such Contribution.
109 |
110 | 2.3. Limitations on Grant Scope
111 |
112 | The licenses granted in this Section 2 are the only rights granted under
113 | this License. No additional rights or licenses will be implied from the
114 | distribution or licensing of Covered Software under this License.
115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a
116 | Contributor:
117 |
118 | (a) for any code that a Contributor has removed from Covered Software;
119 | or
120 |
121 | (b) for infringements caused by: (i) Your and any other third party's
122 | modifications of Covered Software, or (ii) the combination of its
123 | Contributions with other software (except as part of its Contributor
124 | Version); or
125 |
126 | (c) under Patent Claims infringed by Covered Software in the absence of
127 | its Contributions.
128 |
129 | This License does not grant any rights in the trademarks, service marks,
130 | or logos of any Contributor (except as may be necessary to comply with
131 | the notice requirements in Section 3.4).
132 |
133 | 2.4. Subsequent Licenses
134 |
135 | No Contributor makes additional grants as a result of Your choice to
136 | distribute the Covered Software under a subsequent version of this
137 | License (see Section 10.2) or under the terms of a Secondary License (if
138 | permitted under the terms of Section 3.3).
139 |
140 | 2.5. Representation
141 |
142 | Each Contributor represents that the Contributor believes its
143 | Contributions are its original creation(s) or it has sufficient rights
144 | to grant the rights to its Contributions conveyed by this License.
145 |
146 | 2.6. Fair Use
147 |
148 | This License is not intended to limit any rights You have under
149 | applicable copyright doctrines of fair use, fair dealing, or other
150 | equivalents.
151 |
152 | 2.7. Conditions
153 |
154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
155 | in Section 2.1.
156 |
157 | 3. Responsibilities
158 | -------------------
159 |
160 | 3.1. Distribution of Source Form
161 |
162 | All distribution of Covered Software in Source Code Form, including any
163 | Modifications that You create or to which You contribute, must be under
164 | the terms of this License. You must inform recipients that the Source
165 | Code Form of the Covered Software is governed by the terms of this
166 | License, and how they can obtain a copy of this License. You may not
167 | attempt to alter or restrict the recipients' rights in the Source Code
168 | Form.
169 |
170 | 3.2. Distribution of Executable Form
171 |
172 | If You distribute Covered Software in Executable Form then:
173 |
174 | (a) such Covered Software must also be made available in Source Code
175 | Form, as described in Section 3.1, and You must inform recipients of
176 | the Executable Form how they can obtain a copy of such Source Code
177 | Form by reasonable means in a timely manner, at a charge no more
178 | than the cost of distribution to the recipient; and
179 |
180 | (b) You may distribute such Executable Form under the terms of this
181 | License, or sublicense it under different terms, provided that the
182 | license for the Executable Form does not attempt to limit or alter
183 | the recipients' rights in the Source Code Form under this License.
184 |
185 | 3.3. Distribution of a Larger Work
186 |
187 | You may create and distribute a Larger Work under terms of Your choice,
188 | provided that You also comply with the requirements of this License for
189 | the Covered Software. If the Larger Work is a combination of Covered
190 | Software with a work governed by one or more Secondary Licenses, and the
191 | Covered Software is not Incompatible With Secondary Licenses, this
192 | License permits You to additionally distribute such Covered Software
193 | under the terms of such Secondary License(s), so that the recipient of
194 | the Larger Work may, at their option, further distribute the Covered
195 | Software under the terms of either this License or such Secondary
196 | License(s).
197 |
198 | 3.4. Notices
199 |
200 | You may not remove or alter the substance of any license notices
201 | (including copyright notices, patent notices, disclaimers of warranty,
202 | or limitations of liability) contained within the Source Code Form of
203 | the Covered Software, except that You may alter any license notices to
204 | the extent required to remedy known factual inaccuracies.
205 |
206 | 3.5. Application of Additional Terms
207 |
208 | You may choose to offer, and to charge a fee for, warranty, support,
209 | indemnity or liability obligations to one or more recipients of Covered
210 | Software. However, You may do so only on Your own behalf, and not on
211 | behalf of any Contributor. You must make it absolutely clear that any
212 | such warranty, support, indemnity, or liability obligation is offered by
213 | You alone, and You hereby agree to indemnify every Contributor for any
214 | liability incurred by such Contributor as a result of warranty, support,
215 | indemnity or liability terms You offer. You may include additional
216 | disclaimers of warranty and limitations of liability specific to any
217 | jurisdiction.
218 |
219 | 4. Inability to Comply Due to Statute or Regulation
220 | ---------------------------------------------------
221 |
222 | If it is impossible for You to comply with any of the terms of this
223 | License with respect to some or all of the Covered Software due to
224 | statute, judicial order, or regulation then You must: (a) comply with
225 | the terms of this License to the maximum extent possible; and (b)
226 | describe the limitations and the code they affect. Such description must
227 | be placed in a text file included with all distributions of the Covered
228 | Software under this License. Except to the extent prohibited by statute
229 | or regulation, such description must be sufficiently detailed for a
230 | recipient of ordinary skill to be able to understand it.
231 |
232 | 5. Termination
233 | --------------
234 |
235 | 5.1. The rights granted under this License will terminate automatically
236 | if You fail to comply with any of its terms. However, if You become
237 | compliant, then the rights granted under this License from a particular
238 | Contributor are reinstated (a) provisionally, unless and until such
239 | Contributor explicitly and finally terminates Your grants, and (b) on an
240 | ongoing basis, if such Contributor fails to notify You of the
241 | non-compliance by some reasonable means prior to 60 days after You have
242 | come back into compliance. Moreover, Your grants from a particular
243 | Contributor are reinstated on an ongoing basis if such Contributor
244 | notifies You of the non-compliance by some reasonable means, this is the
245 | first time You have received notice of non-compliance with this License
246 | from such Contributor, and You become compliant prior to 30 days after
247 | Your receipt of the notice.
248 |
249 | 5.2. If You initiate litigation against any entity by asserting a patent
250 | infringement claim (excluding declaratory judgment actions,
251 | counter-claims, and cross-claims) alleging that a Contributor Version
252 | directly or indirectly infringes any patent, then the rights granted to
253 | You by any and all Contributors for the Covered Software under Section
254 | 2.1 of this License shall terminate.
255 |
256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all
257 | end user license agreements (excluding distributors and resellers) which
258 | have been validly granted by You or Your distributors under this License
259 | prior to termination shall survive termination.
260 |
261 | ************************************************************************
262 | * *
263 | * 6. Disclaimer of Warranty *
264 | * ------------------------- *
265 | * *
266 | * Covered Software is provided under this License on an "as is" *
267 | * basis, without warranty of any kind, either expressed, implied, or *
268 | * statutory, including, without limitation, warranties that the *
269 | * Covered Software is free of defects, merchantable, fit for a *
270 | * particular purpose or non-infringing. The entire risk as to the *
271 | * quality and performance of the Covered Software is with You. *
272 | * Should any Covered Software prove defective in any respect, You *
273 | * (not any Contributor) assume the cost of any necessary servicing, *
274 | * repair, or correction. This disclaimer of warranty constitutes an *
275 | * essential part of this License. No use of any Covered Software is *
276 | * authorized under this License except under this disclaimer. *
277 | * *
278 | ************************************************************************
279 |
280 | ************************************************************************
281 | * *
282 | * 7. Limitation of Liability *
283 | * -------------------------- *
284 | * *
285 | * Under no circumstances and under no legal theory, whether tort *
286 | * (including negligence), contract, or otherwise, shall any *
287 | * Contributor, or anyone who distributes Covered Software as *
288 | * permitted above, be liable to You for any direct, indirect, *
289 | * special, incidental, or consequential damages of any character *
290 | * including, without limitation, damages for lost profits, loss of *
291 | * goodwill, work stoppage, computer failure or malfunction, or any *
292 | * and all other commercial damages or losses, even if such party *
293 | * shall have been informed of the possibility of such damages. This *
294 | * limitation of liability shall not apply to liability for death or *
295 | * personal injury resulting from such party's negligence to the *
296 | * extent applicable law prohibits such limitation. Some *
297 | * jurisdictions do not allow the exclusion or limitation of *
298 | * incidental or consequential damages, so this exclusion and *
299 | * limitation may not apply to You. *
300 | * *
301 | ************************************************************************
302 |
303 | 8. Litigation
304 | -------------
305 |
306 | Any litigation relating to this License may be brought only in the
307 | courts of a jurisdiction where the defendant maintains its principal
308 | place of business and such litigation shall be governed by laws of that
309 | jurisdiction, without reference to its conflict-of-law provisions.
310 | Nothing in this Section shall prevent a party's ability to bring
311 | cross-claims or counter-claims.
312 |
313 | 9. Miscellaneous
314 | ----------------
315 |
316 | This License represents the complete agreement concerning the subject
317 | matter hereof. If any provision of this License is held to be
318 | unenforceable, such provision shall be reformed only to the extent
319 | necessary to make it enforceable. Any law or regulation which provides
320 | that the language of a contract shall be construed against the drafter
321 | shall not be used to construe this License against a Contributor.
322 |
323 | 10. Versions of the License
324 | ---------------------------
325 |
326 | 10.1. New Versions
327 |
328 | Mozilla Foundation is the license steward. Except as provided in Section
329 | 10.3, no one other than the license steward has the right to modify or
330 | publish new versions of this License. Each version will be given a
331 | distinguishing version number.
332 |
333 | 10.2. Effect of New Versions
334 |
335 | You may distribute the Covered Software under the terms of the version
336 | of the License under which You originally received the Covered Software,
337 | or under the terms of any subsequent version published by the license
338 | steward.
339 |
340 | 10.3. Modified Versions
341 |
342 | If you create software not governed by this License, and you want to
343 | create a new license for such software, you may create and use a
344 | modified version of this License if you rename the license and remove
345 | any references to the name of the license steward (except to note that
346 | such modified license differs from this License).
347 |
348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary
349 | Licenses
350 |
351 | If You choose to distribute Source Code Form that is Incompatible With
352 | Secondary Licenses under the terms of this version of the License, the
353 | notice described in Exhibit B of this License must be attached.
354 |
355 | Exhibit A - Source Code Form License Notice
356 | -------------------------------------------
357 |
358 | This Source Code Form is subject to the terms of the Mozilla Public
359 | License, v. 2.0. If a copy of the MPL was not distributed with this
360 | file, You can obtain one at http://mozilla.org/MPL/2.0/.
361 |
362 | If it is not possible or desirable to put the notice in a particular
363 | file, then You may include the notice in a location (such as a LICENSE
364 | file in a relevant directory) where a recipient would be likely to look
365 | for such a notice.
366 |
367 | You may add additional accurate notices of copyright ownership.
368 |
369 | Exhibit B - "Incompatible With Secondary Licenses" Notice
370 | ---------------------------------------------------------
371 |
372 | This Source Code Form is "Incompatible With Secondary Licenses", as
373 | defined by the Mozilla Public License, v. 2.0.
374 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | 
4 |
5 |
6 |
7 | KafkaPeek
8 |
9 | KafkaPeek is a developer tool that monitors key metrics in your local Kafka deployment.
10 |
11 | #### Key Health Metrics Tracked
12 |
13 | - Bytes In Per Sec
14 | - Bytes Out Per Sec
15 | - JVM Heap Usage (MB)
16 | - Under Replicated Partitions
17 | - Offline Partition Count
18 | - Active Controller Count
19 | - Brokers Running
20 |
21 | #### Key Performance Metrics Tracked
22 |
23 | - Requests Per Sec
24 | - Request Total Time (s)
25 | - Request Queue Time (ms)
26 | - Response Send Time (ms)
27 | ## Tech Stack
28 |
29 | **Client:** [React](https://reactjs.org/), [React Router](https://reactrouter.com/en/main), [ApexCharts](https://apexcharts.com), [Socket.io](https://socket.io)
30 |
31 | **Server:** [Node](https://nodejs.org/en/), [Express](https://expressjs.com/), [Socket.io](https://socket.io) [KafkaJS](https://kafka.js.org)
32 |
33 |
34 | **DevOps:** [Apache Kafka](https://kafka.apache.org/), [Prometheus](https://prometheus.io/), [JMX Exporter](https://github.com/prometheus/jmx_exporter), [Grafana](https://grafana.com), [Docker](https://www.docker.com), [Git](https://git-scm.com/), [Github](https://github.com/)
35 |
36 |
37 | ## Features
38 |
39 | - Live monitoring of key health metrics of a Kafka Broker
40 | - Secured login through Github OAuth
41 | - Slack alerts if metrics go beyond user set specifications
42 | - Customization of dashboard metrics based on the organizations needs
43 | - Ability to save graphs in real time
44 |
45 |
46 | ## Installation
47 |
48 | Install KafkaPeek with npm to install all dependencies
49 |
50 | ```bash
51 | npm install
52 | ```
53 |
54 | ## Deployment
55 |
56 | To deploy this project run the following commands in the terminal:
57 |
58 | 1. Deploy your kafka cluster
59 | ```bash
60 | docker-compose up
61 | ```
62 |
63 | 2. In a new terminal window, deploy the producer:
64 | ```bash
65 | npm run start:producer
66 | ```
67 |
68 | 3. In a new terminal window, deploy the consumer:
69 | ```bash
70 | npm run start:consumer
71 | ```
72 |
73 | 4. In a new terminal window, deploy the application on:
74 |
75 | Deploy application on dev server (localhost:8080)
76 | ```bash
77 | npm run dev
78 | ```
79 |
80 | OR
81 |
82 | Deploy application on production server (localhost:4000)
83 | ```bash
84 | npm run build
85 | ```
86 | then
87 | ```bash
88 | npm start
89 | ```
90 | Now you have a fully functioning Kafka broker with one demo producer and consumer.
91 |
92 | ## Demo
93 |
94 | Main Dashboard
95 |
96 |
97 | 
98 |
99 | Slack Notifications
100 |
101 |
102 | 
103 |
104 |
105 | ## Running Tests
106 |
107 | To run tests, run the following command
108 |
109 | ```bash
110 | npm run test socketIOtest.js
111 | ```
112 | ```bash
113 | npm run test supertest.js
114 | ```
115 |
116 | ## Roadmap
117 |
118 | - Web Based Application
119 | - Customized Developer Options
120 | - Github OAuth Authentication
121 | - User Account Information
122 | - User Settings
123 | - NoSql Database - Error Logs/Accounts/Archived Metrics
124 |
125 |
126 | ## Acknowledgements
127 |
128 | - [Awesome Readme Templates](https://awesomeopensource.com/project/elangosundar/awesome-README-templates)
129 | - [Awesome README](https://github.com/matiassingers/awesome-readme)
130 | - [How to write a Good readme](https://bulldogjob.com/news/449-how-to-write-a-good-readme-for-your-github-project)
131 |
132 |
133 | ## Authors
134 |
135 | - Dipen Nagpal: [Github](https://github.com/dnagpal1) | [LinkedIn](www.linkedin.com/in/dipen-nagpal)
136 | - Nicholas Echevarria: [Github](https://github.com/nick-echevarria) | [LinkedIn](https://www.linkedin.com/in/nicholasechevarria/)
137 | - Luis Navarro: [Github](https://github.com/luis-e-navarro) | [LinkedIn](https://linkedin.com/in/luis-e-navarro)
138 | - Juan Kang: [Github](https://github.com/juanpakang) | [LinkedIn](https://www.linkedin.com/in/juankang/)
139 |
--------------------------------------------------------------------------------
/__tests__/socketIOtest.js:
--------------------------------------------------------------------------------
1 | const { createServer } = require("http");
2 | const { Server } = require("socket.io");
3 | const Client = require("socket.io-client");
4 | const { httpServer } = require('../server/server');
5 | const { fetchQuery, resetCounter } = require('../server/queries');
6 | // const {params} = require('../client/containers/Health_Dashboard')
7 | describe("Testing KafkaPeek websocket", () => {
8 | let io, serverSocket, clientSocket;
9 |
10 | beforeAll((done) => {
11 | const httpServer = createServer();
12 | io = new Server(httpServer);
13 | httpServer.listen(() => {
14 | const port = httpServer.address().port;
15 | clientSocket = new Client(`http://localhost:${port}`);
16 | io.on("connection", (socket) => {
17 | serverSocket = socket;
18 | });
19 | clientSocket.on("connect", done);
20 | });
21 | });
22 |
23 | afterAll(() => {
24 | io.close();
25 | clientSocket.close();
26 | });
27 |
28 | test("should return kafka metrics with the right size and value", (done) => {
29 | clientSocket.on("health", (args) => {
30 | console.log(args)
31 | expect(args.bytesInPerSec).toHaveLength(60)
32 | expect(args.bytesOutPerSec).toHaveLength(60)
33 | expect(args.messagesInPerSec).toHaveLength(60)
34 | expect(args.jvmHeapUsage).toHaveLength(60)
35 | expect(args.activeControllerCount).toBe('1')
36 | expect(args.underRepPartitions).toBe('0')
37 | expect(args.brokersRunning).toBe('1')
38 | expect(args.offlinePartitions).toBe('0')
39 | done();
40 | });
41 | const args = {
42 | 'bytesInPerSec': ['kafka_server_broker_topic_metrics_bytesinpersec_rate','[10m:10s]'],
43 | 'bytesOutPerSec': ['kafka_server_broker_topic_metrics_bytesoutpersec_rate','[10m:10s]'],
44 | 'messagesInPerSec': ['kafka_server_broker_topic_metrics_messagesinpersec_rate','[10m:10s]'],
45 | 'jvmHeapUsage': ['kafka_jvm_heap_usage{env="cluster-demo", type="used"}','[10m:10s]'],
46 | 'activeControllerCount': ["sum(kafka_controller_activecontrollercount)",""],
47 | 'underRepPartitions': ['kafka_server_replica_manager_underreplicatedpartitions',''],
48 | 'offlinePartitions': ['kafka_controller_offlinepartitionscount',''],
49 | 'brokersRunning': ['count(kafka_server_brokerstate)','']
50 | }
51 |
52 | const fetchFunc = async () => {
53 | const fetchObj = {};
54 | for (const [k, v] of Object.entries(args)) {
55 | fetchObj[k] = await fetchQuery(v[0], v[1]);
56 | }
57 | serverSocket.emit("health", fetchObj);
58 | };
59 | fetchFunc()
60 |
61 | });
62 |
63 | test("should work (with ack)", (done) => {
64 | serverSocket.on("hi", (cb) => {
65 | cb("hello");
66 | });
67 | clientSocket.emit("hi", (arg) => {
68 | expect(arg).toBe("hello");
69 | done();
70 | });
71 | });
72 | });
--------------------------------------------------------------------------------
/__tests__/supertest.js:
--------------------------------------------------------------------------------
1 | const { io } = require('socket.io-client');
2 | const request = require('supertest');
3 | const {app} = require('../server/server');
4 |
5 | const client_id = process.env.GITHUB_CLIENT_ID;
6 | const client_secret = process.env.GITHUB_CLIENT_SECRET;
7 | const callback_url = 'http://localhost:4000/auth/github/callback';
8 |
9 | describe('Route integration', () => {
10 |
11 | describe('/', () => {
12 | describe('GET', () => {
13 | //Testing to ensure server responds with status 200 and text/html content type when a GET request is sent to the main page
14 | it('responds with 200 status and text/html content type', () => {
15 | return request(app)
16 | .get('/')
17 | .expect('Content-Type', /text\/html/)
18 | .expect(200)
19 | });
20 | });
21 | });
22 |
23 | describe('/auth', () => {
24 | describe('GET', () => {
25 | test('should respond with github url', async () => {
26 | const response = await request(app)
27 | .get('/auth/github')
28 | .query({user: 'tester@test.com', oauth: 'testoauth2'})
29 | // .set('Authorization', `${token}`)
30 | .expect(302)
31 | })
32 | })
33 | })
34 | });
--------------------------------------------------------------------------------
/client/App.jsx:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect } from 'react';
2 | import { Routes, Route } from 'react-router-dom';
3 | import axios from 'axios';
4 | import Landing from './containers/Landing';
5 | import Notifications from './containers/Notifications';
6 | import PerfDashboard from './containers/PerfDashboard';
7 | import HealthDashboard from './containers/HealthDashboard';
8 | const URL_PARAMS = new URLSearchParams(window.location.search);
9 | const accessToken = URL_PARAMS.get('token');
10 |
11 | const App = () => {
12 | const [ongoingGate, setongoingGate] = useState(false)
13 | const [ongoingList, setongoingList] = useState([])
14 | const [active, setActive] = useState('health');
15 | const [user, setUser] = useState({
16 | name: '',
17 | id: '',
18 | login: '',
19 | email: '',
20 | });
21 |
22 | useEffect(() => {
23 | const getGithubUser = async (access_token) => {
24 | const res = await axios.get('https://api.github.com/user', {
25 | headers: {
26 | Authorization: `bearer ${access_token}`,
27 | },
28 | });
29 | const userObj = {
30 | name: res.data.name,
31 | id: res.data.id,
32 | login: res.data.login,
33 | email: res.data.emil
34 | };
35 | setUser(userObj);
36 | };
37 | if (accessToken) {
38 | getGithubUser(accessToken);
39 | }
40 | }, [accessToken]);
41 |
42 | return (
43 |
44 |
45 |
49 | }
50 | />
51 |
69 |
10 |
16 | );
17 | };
18 |
19 | export default CogIcon;
20 |
--------------------------------------------------------------------------------
/client/assets/GraphIcon.jsx:
--------------------------------------------------------------------------------
1 |
2 |
3 | import React from 'react';
4 |
5 | const GraphIcon = () => {
6 | return (
7 |
12 |
16 | );
17 | }
18 |
19 | export default GraphIcon;
20 |
--------------------------------------------------------------------------------
/client/assets/HomeIcon.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | const HomeIcon = () => {
4 | return (
5 |
6 |
9 | );
10 | };
11 |
12 |
13 | export default HomeIcon;
14 |
--------------------------------------------------------------------------------
/client/assets/KafkaPeekLogo.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | const KafkaPeekLogo = () => {
4 | return (
5 |
11 |
17 | );
18 | };
19 |
20 | export default KafkaPeekLogo;
21 |
--------------------------------------------------------------------------------
/client/assets/Logout.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | const Logout = () => {
4 | return (
5 |
6 |
9 | );
10 | };
11 |
12 | export default Logout;
13 |
--------------------------------------------------------------------------------
/client/assets/UserIcon.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | const UserIcon = () => {
4 | return (
5 |
10 |
16 | );
17 | };
18 |
19 | export default UserIcon;
20 |
--------------------------------------------------------------------------------
/client/assets/images/kafkaPeek-demo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/KafkaPeek/8e7529f6375fb8170dc300f5e38eac608c9efc13/client/assets/images/kafkaPeek-demo.png
--------------------------------------------------------------------------------
/client/assets/images/kafkaPeek-logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/KafkaPeek/8e7529f6375fb8170dc300f5e38eac608c9efc13/client/assets/images/kafkaPeek-logo.png
--------------------------------------------------------------------------------
/client/components/HealthBinBout.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import ReactApexChart from 'react-apexcharts';
4 |
5 | class HealthBinBout extends React.Component {
6 | constructor(props) {
7 | super(props);
8 |
9 | this.state = {
10 | series: [{
11 | name: 'Bytes In/sec',
12 | data: props.series
13 | }, {
14 | name: 'Bytes Out/sec',
15 | data: props.series
16 | }],
17 | options: {
18 | chart: {
19 | id: 'realtime',
20 | height: '100%',
21 | width: '100%',
22 | type: 'line',
23 | animations: {
24 | enabled: true,
25 | easing: 'linear',
26 | dynamicAnimation: {
27 | speed: 300,
28 | },
29 | },
30 | toolbar: {
31 | show: true,
32 | },
33 | zoom: {
34 | enabled: false,
35 | },
36 | },
37 | dataLabels: {
38 | enabled: false,
39 | },
40 | stroke: {
41 | curve: 'smooth',
42 | },
43 | title: {
44 | text: 'Bytes In/sec & Bytes Out/sec',
45 | align: 'left',
46 | },
47 |
48 | noData: {
49 | text: "CLICK TO GET METRICS...",
50 | },
51 |
52 | markers: {
53 | size: 0,
54 | hover: {
55 | size: 0
56 | }
57 | },
58 |
59 | xaxis: {
60 | type: 'datetime',
61 | range: 600000,
62 | },
63 | yaxis: {
64 | min: 0,
65 | max: 2500,
66 | decimalsInFloat: 2,
67 | opposite: true,
68 | labels: {
69 | offsetX: -10
70 | }
71 | },
72 | legend: {
73 | show: true,
74 | floating: true,
75 | horizontalAlign: 'left',
76 | onItemClick: {
77 | toggleDataSeries: false
78 | },
79 | position: 'top',
80 | offsetY: -20,
81 | offsetX: 300
82 | },
83 |
84 | },
85 | };
86 | }
87 |
88 | render() {
89 | return (
90 |
91 |
99 | );
100 | }
101 | }
102 |
103 | export default HealthBinBout;
104 |
--------------------------------------------------------------------------------
/client/components/HealthJVM.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import ReactApexChart from 'react-apexcharts';
4 |
5 | class HealthJVM extends React.Component {
6 | constructor(props) {
7 | super(props);
8 |
9 | this.state = {
10 | series: [{ data: props.series }],
11 | options: {
12 | chart: {
13 | id: 'realtime',
14 | height: '100%',
15 | width: '100%',
16 | type: 'line',
17 | animations: {
18 | enabled: true,
19 | easing: 'linear',
20 | dynamicAnimation: {
21 | speed: 300,
22 | },
23 | },
24 | toolbar: {
25 | show: true,
26 | },
27 | zoom: {
28 | enabled: false,
29 | },
30 | },
31 | dataLabels: {
32 | enabled: false,
33 | },
34 | stroke: {
35 | curve: 'smooth',
36 | },
37 | title: {
38 | text: 'JVM Heap Usage MB',
39 | align: 'left',
40 | },
41 |
42 | markers: {
43 | size: 0,
44 | hover: {
45 | size: 0
46 | }
47 | },
48 |
49 | xaxis: {
50 | type: 'datetime',
51 | range: 600000,
52 | },
53 | yaxis: {
54 | min: 0,
55 | max: 1500,
56 | decimalsInFloat: 2,
57 | opposite: true,
58 | labels: {
59 | offsetX: -10
60 | }
61 | },
62 | legend: {
63 | show: true,
64 | floating: true,
65 | horizontalAlign: 'left',
66 | onItemClick: {
67 | toggleDataSeries: false
68 | },
69 | position: 'top',
70 | offsetY: -20,
71 | offsetX: 300
72 | },
73 |
74 | },
75 | };
76 | }
77 |
78 | render() {
79 | return (
80 |
81 |
89 | );
90 | }
91 | }
92 |
93 | export default HealthJVM;
94 |
--------------------------------------------------------------------------------
/client/components/HealthMsgIn.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import ReactApexChart from 'react-apexcharts';
4 |
5 | class HealthMsgIn extends React.Component {
6 | constructor(props) {
7 | super(props);
8 |
9 | this.state = {
10 | series: [{ data: props.series }],
11 | options: {
12 | chart: {
13 | id: 'realtime',
14 | height: '100%',
15 | width: '100%',
16 | type: 'line',
17 | animations: {
18 | enabled: true,
19 | easing: 'linear',
20 | dynamicAnimation: {
21 | speed: 300,
22 | },
23 | },
24 | toolbar: {
25 | show: true,
26 | },
27 | zoom: {
28 | enabled: false,
29 | },
30 | },
31 | dataLabels: {
32 | enabled: false,
33 | },
34 | stroke: {
35 | curve: 'smooth',
36 | },
37 | title: {
38 | text: 'Messages In Per Second',
39 | align: 'left',
40 | },
41 |
42 | markers: {
43 | size: 0,
44 | hover: {
45 | size: 0
46 | }
47 | },
48 |
49 | xaxis: {
50 | type: 'datetime',
51 | range: 600000,
52 | },
53 | yaxis: {
54 | min: 0,
55 | max: 300,
56 | decimalsInFloat: 2,
57 | opposite: true,
58 | labels: {
59 | offsetX: -10
60 | }
61 | },
62 | legend: {
63 | show: true,
64 | floating: true,
65 | horizontalAlign: 'left',
66 | onItemClick: {
67 | toggleDataSeries: false
68 | },
69 | position: 'top',
70 | offsetY: -20,
71 | offsetX: 300
72 | },
73 |
74 | },
75 | };
76 | }
77 |
78 | render() {
79 | return (
80 |
81 |
89 | );
90 | }
91 | }
92 |
93 | export default HealthMsgIn;
94 |
--------------------------------------------------------------------------------
/client/components/PerfReqPerSec.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import ReactApexChart from 'react-apexcharts';
4 |
5 | class PerfReqPerSec extends React.Component {
6 | constructor(props) {
7 | super(props);
8 |
9 | this.state = {
10 | series: [{ data: props.series }],
11 | options: {
12 | chart: {
13 | id: 'realtime',
14 | height: '100%',
15 | width: '100%',
16 | type: 'line',
17 | animations: {
18 | enabled: true,
19 | easing: 'linear',
20 | dynamicAnimation: {
21 | speed: 300,
22 | },
23 | },
24 | toolbar: {
25 | show: true,
26 | },
27 | zoom: {
28 | enabled: false,
29 | },
30 | },
31 | dataLabels: {
32 | enabled: false,
33 | },
34 | stroke: {
35 | curve: 'smooth',
36 | },
37 | title: {
38 | text: 'Requests Per Second',
39 | align: 'left',
40 | },
41 |
42 | markers: {
43 | size: 0,
44 | hover: {
45 | size: 0
46 | }
47 | },
48 |
49 | xaxis: {
50 | type: 'datetime',
51 | range: 600000,
52 | },
53 | yaxis: {
54 | min: 0,
55 | max: 300,
56 | decimalsInFloat: 2,
57 | opposite: true,
58 | labels: {
59 | offsetX: -10
60 | }
61 | },
62 | legend: {
63 | show: true,
64 | floating: true,
65 | horizontalAlign: 'left',
66 | onItemClick: {
67 | toggleDataSeries: false
68 | },
69 | position: 'top',
70 | offsetY: -20,
71 | offsetX: 300
72 | },
73 |
74 | },
75 | };
76 | }
77 |
78 | render() {
79 | return (
80 |
81 |
89 | );
90 | }
91 | }
92 |
93 | export default PerfReqPerSec;
94 |
--------------------------------------------------------------------------------
/client/components/PerfReqTotalTime.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import ReactApexChart from 'react-apexcharts';
4 |
5 | class PerfReqTotalTime extends React.Component {
6 | constructor(props) {
7 | super(props);
8 |
9 | this.state = {
10 | series: [{
11 | name: 'Mean',
12 | data: props.series
13 | }, {
14 | name: '99th Percentile',
15 | data: props.series
16 | }, {
17 | name: '75th Percentile',
18 | data: props.series
19 | }],
20 | options: {
21 | chart: {
22 | id: 'realtime',
23 | height: '100%',
24 | width: '100%',
25 | type: 'line',
26 | animations: {
27 | enabled: true,
28 | easing: 'linear',
29 | dynamicAnimation: {
30 | speed: 300,
31 | },
32 | },
33 | toolbar: {
34 | show: true,
35 | },
36 | zoom: {
37 | enabled: false,
38 | },
39 | },
40 | dataLabels: {
41 | enabled: false,
42 | },
43 | stroke: {
44 | curve: 'smooth',
45 | },
46 | title: {
47 | text: 'Request Total Time',
48 | align: 'left',
49 | },
50 |
51 | noData: {
52 | text: "CLICK TO GET METRICS...",
53 | },
54 |
55 | markers: {
56 | size: 0,
57 | hover: {
58 | size: 0
59 | }
60 | },
61 |
62 | xaxis: {
63 | type: 'datetime',
64 | range: 600000,
65 | },
66 | yaxis: {
67 | min: 2,
68 | max: 6,
69 | decimalsInFloat: 2,
70 | opposite: true,
71 | labels: {
72 | offsetX: -10
73 | }
74 | },
75 | legend: {
76 | show: true,
77 | floating: true,
78 | horizontalAlign: 'left',
79 | onItemClick: {
80 | toggleDataSeries: false
81 | },
82 | position: 'top',
83 | offsetY: -20,
84 | offsetX: 300
85 | },
86 |
87 | },
88 | };
89 | }
90 |
91 | render() {
92 | return (
93 |
94 |
102 | );
103 | }
104 | }
105 |
106 | export default PerfReqTotalTime;
107 |
--------------------------------------------------------------------------------
/client/components/PerfResQueueTime.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import ReactApexChart from 'react-apexcharts';
4 | class PerfResQueueTime extends React.Component {
5 | constructor(props) {
6 | super(props);
7 |
8 | this.state = {
9 | series: [{ data: props.series }],
10 | options: {
11 | chart: {
12 | id: 'realtime',
13 | height: '100%',
14 | width: '100%',
15 | type: 'line',
16 | animations: {
17 | enabled: true,
18 | easing: 'linear',
19 | dynamicAnimation: {
20 | speed: 300,
21 | },
22 | },
23 | toolbar: {
24 | show: true,
25 | },
26 | zoom: {
27 | enabled: false,
28 | },
29 | },
30 | dataLabels: {
31 | enabled: false,
32 | },
33 | stroke: {
34 | curve: 'smooth',
35 | },
36 | title: {
37 | text: 'Response Queue Time (ms)',
38 | align: 'left',
39 | },
40 |
41 | markers: {
42 | size: 0,
43 | hover: {
44 | size: 0
45 | }
46 | },
47 |
48 | xaxis: {
49 | type: 'datetime',
50 | range: 600000,
51 | },
52 | yaxis: {
53 | min: 0.0001,
54 | max: 4,
55 | decimalsInFloat: 2,
56 | opposite: true,
57 | labels: {
58 | offsetX: -10
59 | }
60 | },
61 | legend: {
62 | show: true,
63 | floating: true,
64 | horizontalAlign: 'left',
65 | onItemClick: {
66 | toggleDataSeries: false
67 | },
68 | position: 'top',
69 | offsetY: -20,
70 | offsetX: 300
71 | },
72 |
73 | },
74 | };
75 | }
76 |
77 | render() {
78 | return (
79 |
80 |
88 | );
89 | }
90 | }
91 |
92 | export default PerfResQueueTime;
93 |
--------------------------------------------------------------------------------
/client/components/PerfResSendTime.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | import ReactApexChart from 'react-apexcharts';
4 |
5 | class PerfResSendTime extends React.Component {
6 | constructor(props) {
7 | super(props);
8 |
9 | this.state = {
10 | series: [{ data: props.series }],
11 | options: {
12 | chart: {
13 | id: 'realtime',
14 | height: '100%',
15 | width: '100%',
16 | type: 'line',
17 | animations: {
18 | enabled: true,
19 | easing: 'linear',
20 | dynamicAnimation: {
21 | speed: 300,
22 | },
23 | },
24 | toolbar: {
25 | show: true,
26 | },
27 | zoom: {
28 | enabled: false,
29 | },
30 | },
31 | dataLabels: {
32 | enabled: false,
33 | },
34 | stroke: {
35 | curve: 'smooth',
36 | },
37 | title: {
38 | text: 'Response Send Time (ms)',
39 | align: 'left',
40 | },
41 |
42 | markers: {
43 | size: 0,
44 | hover: {
45 | size: 0
46 | }
47 | },
48 |
49 | xaxis: {
50 | type: 'datetime',
51 | range: 600000,
52 | },
53 | yaxis: {
54 | min: 0.0001,
55 | max: 2,
56 | decimalsInFloat: 2,
57 | opposite: true,
58 | labels: {
59 | offsetX: -10
60 | }
61 | },
62 | legend: {
63 | show: true,
64 | floating: true,
65 | horizontalAlign: 'left',
66 | onItemClick: {
67 | toggleDataSeries: false
68 | },
69 | position: 'top',
70 | offsetY: -20,
71 | offsetX: 300
72 | },
73 |
74 | },
75 | };
76 | }
77 |
78 | render() {
79 | return (
80 |
81 |
89 | );
90 | }
91 | }
92 |
93 | export default PerfResSendTime;
94 |
--------------------------------------------------------------------------------
/client/components/StaticMetricDisplay.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | const StaticMetricDisplay = ({ metric, title, container }) => {
4 | return (
5 |
6 |
{title}
7 |
{metric}
8 |
20 |
BytesIn
21 |
Network throughput can affect Kafka’s performance if you are sending messages across data centers, if your topics have a large number of consumers, or if your replicas are catching up to their leaders. Tracking network throughput on your brokers gives you more information as to where potential bottlenecks may lie, and can inform decisions like whether or not you should enable end-to-end compression of your messages.
22 | The following options are thresholds that you can pick from for your BytesInPerSec Metric.
23 |
24 |
29 |
BytesOut
18 |
19 | The BytesOut metric shows the mean rate and one minute rate of outgoing bytes per second from each topic.
20 | This metric will increase as the volume and frequency of messages being sent to and consumed from the topic increases.
21 | If you are hitting the limit of how much your cluster can process, you should consider increasing the capacity of your cluster.
22 | Please choose a threshold for to track for your BytesOut metric:
23 |
24 |
29 |
30 | );
31 | }
32 |
33 | export default BytesOutPer;
--------------------------------------------------------------------------------
/client/components/notificationComponents/OfflinePart.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import Select from 'react-select'
3 |
4 |
5 | const OfflinePart = () => {
6 |
7 | return (
8 | Offline Partitions
9 |
10 | The Offline Partitions metric displays the number of partitions that do not have an active leader.
11 | The two most common causes of offline partitions are when all brokers hosting replicas for a partition are down, or when no in-sync replicas can become a partition leader due to a mismatch in message counts between the replicas.
12 | This is a critical metric, as any partitions that are offline (ie. without a leader) will not be accessible since read and write operations are only performed on the leader of a partition.
13 |
14 |
15 |
16 | Expected threshold is anything over 0.
17 |
18 |
19 |
20 | );
21 | }
22 |
23 | export default OfflinePart;
--------------------------------------------------------------------------------
/client/components/notificationComponents/OngoingMetrics.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import Select from 'react-select'
3 |
4 |
5 | const OngoingMetrics = ({ongoingList}) => {
6 |
7 | return (
8 |
9 |
ONGOING METRICS
10 |
15 |
Under Replicated Partitions
9 |
10 | In a healthy cluster, the number of in sync replicas (ISRs) should be exactly equal to the total number of replicas.
11 | If partition replicas fall too far behind their leaders, the follower partition is removed from the ISR pool, and you should see a corresponding increase in IsrShrinksPerSec.
12 | If a broker becomes unavailable, the value of UnderReplicatedPartitions will increase sharply.
13 | Since Kafka’s high-availability guarantees cannot be met without replication, investigation is certainly warranted should this metric value exceed zero for extended time periods.
14 |
15 |
16 |
17 |
18 | Expected threshold is anything over 0.
19 |
20 |
21 |
22 | );
23 | }
24 |
25 | export default UnderRep;
--------------------------------------------------------------------------------
/client/containers/Header.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { Link } from 'react-router-dom';
3 | import KafkaPeekLogo from '../assets/KafkaPeekLogo';
4 |
5 | const Header = ({ user }) => {
6 | const { id } = user;
7 |
8 | const signinButton = (
9 |
10 | {
11 | } Sign In With Github {'>'}
13 |
14 | );
15 |
16 | const dashboardButton = (
17 |
18 |
20 | );
21 |
22 | return (
23 |
42 | );
43 | };
44 |
45 | export default Header;
46 |
--------------------------------------------------------------------------------
/client/containers/HealthDashboard.jsx:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect, useRef } from 'react';
2 | import { io } from "socket.io-client";
3 | import Sidebar from './Sidebar';
4 | import HealthBinBout from '../components/HealthBinBout';
5 | import HealthJVM from '../components/HealthJVM';
6 | import HealthMsgIn from '../components/HealthMsgIn';
7 | import StaticMetricDisplay from '../components/StaticMetricDisplay';
8 |
9 | const socket = io('http://localhost:4000', {
10 | autoConnect: false
11 | });
12 |
13 | export const params = {
14 | 'bytesInPerSec': ['kafka_server_broker_topic_metrics_bytesinpersec_rate', '[10m:10s]'],
15 | 'bytesOutPerSec': ['kafka_server_broker_topic_metrics_bytesoutpersec_rate', '[10m:10s]'],
16 | 'messagesInPerSec': ['kafka_server_broker_topic_metrics_messagesinpersec_rate', '[10m:10s]'],
17 | 'jvmHeapUsage': ['kafka_jvm_heap_usage{env="cluster-demo", type="used"}', '[10m:10s]'],
18 | 'activeControllerCount': ["sum(kafka_controller_activecontrollercount)", ""],
19 | 'underRepPartitions': ['kafka_server_replica_manager_underreplicatedpartitions', ''],
20 | 'offlinePartitions': ['kafka_controller_offlinepartitionscount', ''],
21 | 'brokersRunning': ['count(kafka_server_brokerstate)', '']
22 | }
23 |
24 | const emitFunc = () => {
25 | socket.emit('health', params)
26 | }
27 |
28 | const stopFunc = () => {
29 | socket.emit('stop');
30 | }
31 |
32 | const HealthDashboard = ({ active, setActive}) => {
33 | let startMetric = useRef(false);
34 | const [buttonText, setButtonText] = useState('Get Metrics');
35 |
36 | // dynamic metrics
37 | const [bytesIn, setBytesIn] = useState([]);
38 | const [bytesOut, setBytesOut] = useState([]);
39 | const [msgsIn, setMsgsIn] = useState([]);
40 | const [jvmUsage, setJvmUsage] = useState([]);
41 |
42 | // static metrics
43 | const [activeControllerCount, setActiveControllerCount] = useState(0);
44 | const [offlinePartitions, setOfflinePartitions] = useState(0);
45 | const [underReplicatedPartitions, setUnderReplicatedPartitions] = useState(0);
46 | const [brokersRunning, setBrokersRunning] = useState(0);
47 |
48 | const handleClick = () => {
49 | if (!startMetric.current) {
50 | socket.connect()
51 | emitFunc();
52 | setButtonText('Pause')
53 | startMetric.current = !startMetric.current;
54 | } else {
55 | stopFunc();
56 | setButtonText('Get Metrics')
57 | startMetric.current = !startMetric.current;
58 | }
59 | };
60 |
61 | const handleHealthDisconnect = () => {
62 | socket.disconnect()
63 | }
64 |
65 | useEffect(() => {
66 | socket.on('health', (data) => {
67 | if (data.bytesInPerSec) setBytesIn(currentData => [...currentData, ...data.bytesInPerSec])
68 | if (data.bytesOutPerSec) setBytesOut(currentData => [...currentData, ...data.bytesOutPerSec]);
69 | if (data.messagesInPerSec) setMsgsIn(currentData => [...currentData, ...data.messagesInPerSec]);
70 | if (data.jvmHeapUsage) setJvmUsage(currentData => [...currentData, ...data.jvmHeapUsage]);
71 | if (data.activeControllerCount) setActiveControllerCount(data.activeControllerCount);
72 | if (data.offlinePartitions) setOfflinePartitions(data.offlinePartitions);
73 | if (data.underRepPartitions) setUnderReplicatedPartitions(data.underRepPartitions);
74 | if (data.brokersRunning) setBrokersRunning(data.brokersRunning);
75 | })
76 | }, []);
77 |
78 | return (
79 |
80 |
81 |
82 |
83 | {buttonText}
84 |
85 |
93 |
15 |
16 |
17 |
22 |
23 |
24 | Keep a keen eye on your
25 |
28 |
29 | {' '}
30 | KafkaPeek helps engineering teams of all sizes quickly and easily stay up
31 | to date on the status of deployments
32 |
33 |
34 |
Try It Out {'>'}
35 |
36 |
37 |
41 |
45 |
])
46 | }
47 |
48 | // bytesOut (state, func)
49 | function bytesOutFunc(interval) {
50 | setGate(true)
51 | setbytesInterval(interval.value.thresholdNumber)
52 | setdisplayedMetric([
53 |
54 |
58 |
])
59 | }
60 |
61 | const subComponents = {
62 | bytesInPerSec: [
87 |
91 |
])
92 | } else {
93 | setGate(false)
94 | setdisplayedMetric([
95 |
96 |
99 |
])
100 | }
101 | await setdisplayedComponent(currentNode)
102 | };
103 |
104 | // sends message to slack
105 | const trackMetrics = async () => {
106 | if (gate) {
107 | const compared = metric.value;
108 | if (!ongoingMetrics[compared]) {
109 | const objectToSend = Object.assign({}, metric, { name: user.name }, { threshold: bytesInterval })
110 | await axios.post("http://localhost:4000/auth/form-submit", objectToSend)
111 | .then(({ data }) => {
112 | console.log(data)
113 | })
114 | .catch(err => {
115 | console.error(err.toJSON());
116 | });
117 | const newOngoing = [...ongoingList]
118 | newOngoing.push({`${metric.label}`} {' > '} {`${bytesInterval}`} STOP
135 |
136 |
137 |
138 |
Set up Slack Notifications
139 |
140 |
141 |
Here, you can choose optional notificaiton in which you will receive a slack message confirming that you have selected a new metric to monitor and recieve notifications for.
144 |
145 | {displayedMetric}
146 |
147 |
148 |
149 | Track Metric
150 |
151 | {ongoingGate ?
153 |
154 |
155 |
156 |
157 | {displayedComponent}
158 |
159 |
160 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 | {buttonText}
87 |
88 |
KafkaPeek
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/client/index.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { createRoot } from 'react-dom/client';
3 | import { HashRouter } from 'react-router-dom';
4 | import App from './App';
5 | import './styles/application.scss';
6 |
7 | const container = document.getElementById('root');
8 | const root = createRoot(container);
9 |
10 | root.render(
11 |
12 |
14 | );
15 |
--------------------------------------------------------------------------------
/client/styles/_dashboard.scss:
--------------------------------------------------------------------------------
1 | #dashboard-container {
2 | width: 100%;
3 | height: 100%;
4 | display: grid;
5 | align-items: center;
6 | justify-content: center;
7 | grid-template-columns: 1fr 2fr 2fr;
8 | grid-template-rows: 1fr 4fr 4fr;
9 | grid-template-areas:
10 | 'sidebar main main'
11 | 'sidebar main main'
12 | 'sidebar main main';
13 |
14 | #dashboard-charts {
15 | width: 100%;
16 | height: 100%;
17 | grid-area: main;
18 |
19 | display: grid;
20 | align-items: center;
21 | padding: 1rem;
22 | justify-content: center;
23 | grid-template-columns: 1fr 1fr 1fr 1fr;
24 | grid-template-rows: .5fr 4fr 3fr;
25 | grid-gap: 1rem;
26 | grid-template-areas:
27 | 'metric metric-2 metric-3 metric-4'
28 | 'chart chart chart chart'
29 | 'chart-2 chart-2 chart-3 chart-3';
30 |
31 | #metric-container-1 {
32 | width: 100%;
33 | height: 100%;
34 | grid-area: metric;
35 | background-color: white;
36 | box-shadow: 0 0 5px $secondaryColor;
37 | border-radius: 1rem;
38 | padding: 10px;
39 |
40 | &:hover {
41 | background-color: aliceblue
42 | }
43 | }
44 |
45 | #metric-container-2 {
46 | width: 100%;
47 | height: 100%;
48 | grid-area: metric-2;
49 | background-color: white;
50 | box-shadow: 0 0 5px $secondaryColor;
51 | border-radius: 1rem;
52 | padding: 10px;
53 | }
54 |
55 | #metric-container-3 {
56 | width: 100%;
57 | height: 100%;
58 | grid-area: metric-3;
59 | background-color: white;
60 | box-shadow: 0 0 5px $secondaryColor;
61 | border-radius: 1rem;
62 | padding: 10px;
63 | }
64 |
65 | #metric-container-4 {
66 | width: 100%;
67 | height: 100%;
68 | grid-area: metric-4;
69 | background-color: white;
70 | box-shadow: 0 0 5px $secondaryColor;
71 | border-radius: 1rem;
72 | padding: 10px;
73 | }
74 |
75 | #chart-container {
76 | width: 100%;
77 | height: 100%;
78 | grid-area: chart;
79 | background-color: white;
80 | box-shadow: 0 0 5px $secondaryColor;
81 | border-radius: 1rem;
82 | padding: 10px;
83 | }
84 |
85 | #chart-container-2 {
86 | width: 100%;
87 | height: 100%;
88 | grid-area: chart-2;
89 | background-color: white;
90 | box-shadow: 0 0 5px $secondaryColor;
91 | border-radius: 1rem;
92 | padding: 10px;
93 | }
94 |
95 | #chart-container-3 {
96 | width: 100%;
97 | height: 100%;
98 | grid-area: chart-3;
99 | background-color: white;
100 | box-shadow: 0 0 5px $secondaryColor;
101 | border-radius: 1rem;
102 | padding: 10px;
103 | }
104 | }
105 | }
--------------------------------------------------------------------------------
/client/styles/_header.scss:
--------------------------------------------------------------------------------
1 | #header-container {
2 | width: 100%;
3 | height: 100%;
4 | display: flex;
5 | flex-direction: row;
6 | grid-area: header;
7 | padding: 2rem;
8 |
9 | #header-left {
10 | display: flex;
11 | justify-content: flex-start;
12 | align-items: center;
13 | width: 25%;
14 | height: 100%;
15 | font-weight: 800;
16 | font-size: 2rem;
17 |
18 | a {
19 | color: white;
20 | font-family: $titleFont;
21 | &:hover {
22 | color: rgba(253, 253, 253, 0.5);
23 | }
24 | cursor: pointer;
25 | }
26 | }
27 |
28 | #header-mid {
29 | display: flex;
30 | flex-direction: row;
31 | justify-content: center;
32 | align-items: center;
33 | width: 50%;
34 | height: 100%;
35 |
36 |
37 | #header-mid-left {
38 | width: 50%;
39 | height: 100%;
40 | display: flex;
41 | justify-content: flex-end;
42 | align-items: center;
43 | padding: 5%;
44 | }
45 |
46 | #header-mid-right {
47 | width: 50%;
48 | height: 100%;
49 | display: flex;
50 | justify-content: flex-start;
51 | align-items: center;
52 | padding: 5%;
53 | }
54 |
55 | a {
56 | color: white;
57 | &:hover {
58 | color: rgba(253, 253, 253, 0.5);
59 | cursor: pointer;
60 | }
61 | }
62 | }
63 |
64 | #header-right {
65 | display: flex;
66 | justify-content: flex-end;
67 | align-items: center;
68 | width: 25%;
69 | height: 100%;
70 |
71 | button {
72 | background: rgba(253, 253, 253, 0.4);
73 | border-radius: 1.2rem;
74 | padding: 0.75rem;
75 |
76 | &:hover {
77 | background: rgba(253, 253, 253, 0.5);
78 | cursor: pointer;
79 | }
80 |
81 | a {
82 | color: white;
83 | font-size: 1rem;
84 | }
85 | }
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/client/styles/_landing.scss:
--------------------------------------------------------------------------------
1 | #landing-container {
2 | width: 100%;
3 | height: 100%;
4 | display: grid;
5 | align-items: center;
6 | justify-content: center;
7 | grid-template-columns: 1fr;
8 | grid-template-rows: 1fr 7fr;
9 | grid-template-areas:
10 | 'header header header'
11 | 'main main main';
12 |
13 | .gradient-container {
14 | width: 100%;
15 | height: 800px;
16 | background: linear-gradient(150deg, #123d57 15%, #2b9fc9 70%, #2bcac8 94%);
17 | position: absolute;
18 | top: 0;
19 | transform: skewY(-12deg);
20 | transform-origin: 0;
21 | z-index: -1;
22 |
23 | span {
24 | height: 190px;
25 | position: absolute;
26 | }
27 |
28 | span:nth-child(4) {
29 | // Select the 4th span inside .block
30 | width: 33.33333%;
31 | bottom: 0;
32 | background: linear-gradient(145deg, rgba(18, 61, 87, 1) 0%, #2bcac8 75%);
33 | }
34 | }
35 |
36 | .copy-container {
37 | width: 100%;
38 | height: 100%;
39 | grid-area: main;
40 | display: flex;
41 | flex-direction: column;
42 | justify-content: center;
43 | align-items: flex-start;
44 | text-align: left;
45 | padding: 3rem;
46 |
47 | h1 {
48 | display: flex;
49 | align-items: center;
50 | width: 90%;
51 | height: 50%;
52 | font-size: 4.5rem;
53 | font-weight: 800;
54 | }
55 |
56 | p {
57 | width: 60%;
58 | height: 20%;
59 | font-weight: 400;
60 | font-size: 1.5rem;
61 | }
62 |
63 | button {
64 | background-color: black;
65 | color: white;
66 | border-radius: 1.2rem;
67 | // min-width: 100px;
68 | // max-width: 150px;
69 | // width: 100%;
70 | height: 100%;
71 | padding: .7rem;
72 |
73 | &:hover {
74 | background-color: rgba(0, 0, 0, 0.6);
75 | cursor: pointer;
76 | }
77 | a {
78 | color: white;
79 | }
80 | }
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/client/styles/_login.scss:
--------------------------------------------------------------------------------
1 | #login-container {
2 | width: 100%;
3 | height: 100%;
4 | display: grid;
5 | align-items: center;
6 | padding: 1rem;
7 | justify-content: center;
8 | grid-template-columns: 1fr 4fr 1fr;
9 | grid-template-rows: 1fr 4fr 4fr;
10 | grid-gap: 0.5rem;
11 | grid-template-areas:
12 | 'header header header'
13 | 'main main main'
14 | 'main main main';
15 |
16 | #login-main {
17 | display: flex;
18 | justify-content: center;
19 | align-items: center;
20 | width: 100%;
21 | height: 100%;
22 | grid-area: main;
23 |
24 | #login-inputs {
25 | width: 50%;
26 | height: 50%;
27 | display: flex;
28 | flex-direction: column;
29 | justify-content: space-evenly;
30 | align-items: center;
31 | }
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/client/styles/_notifications.scss:
--------------------------------------------------------------------------------
1 | #dashboard-container{
2 | .fade {
3 | margin-top: 15px;
4 | padding: 20px;
5 | height: 90px;
6 | font-size: 24px;
7 | border-radius: 4px;
8 | font-weight: bold;
9 | background-color: #123d57;
10 | color: #fff;
11 | transition: background-color .6s;
12 | }
13 |
14 | .fade:hover {
15 | background-color: #166492;
16 | }
17 |
18 | // .fade:focus,
19 | // .fade:active {
20 | // background-color: black;
21 | // transition: none;
22 | // }
23 |
24 | }
--------------------------------------------------------------------------------
/client/styles/_ongoingMetrics.scss:
--------------------------------------------------------------------------------
1 |
2 | .stop-button-container{
3 | #stop-button {
4 | border-radius: 4px;
5 | background-color: #123d57;
6 | color: #fff;
7 | transition: background-color .4s;
8 | }
9 |
10 | #stop-button:hover {
11 | background-color: #c62026;
12 | }
13 |
14 | // .fade:focus,
15 | // .fade:active {
16 | // background-color: black;
17 | // transition: none;
18 | // }
19 | }
--------------------------------------------------------------------------------
/client/styles/_sidebar.scss:
--------------------------------------------------------------------------------
1 | #sidebar-container {
2 | width: 100%;
3 | height: 100%;
4 | grid-area: sidebar;
5 | background-color: $primaryColor;
6 | background: linear-gradient(
7 | 145deg,
8 | rgba(2, 0, 36, 1) 0%,
9 | rgba(18, 61, 87, 1) 75%
10 | );
11 | color: white;
12 |
13 | #top-sidebar {
14 | width: 100%;
15 | height: 25%;
16 | display: flex;
17 | flex-direction: column;
18 | align-items: center;
19 | justify-content: center;
20 |
21 | .spacer {
22 | width: 100%;
23 | height: 15%;
24 | }
25 |
26 | #sidebar-title {
27 | font-family: $titleFont;
28 | font-size: 2rem;
29 | font-weight: 800;
30 | }
31 | }
32 |
33 | #middle-sidebar {
34 | width: 100%;
35 | height: 50%;
36 | display: flex;
37 | flex-direction: column;
38 | align-items: center;
39 | justify-content: center;
40 | padding: 15%;
41 | border-top: 1px solid rgba(211, 211, 211, 0.25);
42 | border-bottom: 1px solid rgba(211, 211, 211, 0.25);
43 |
44 | .sidebar-button {
45 | height: 20%;
46 | width: 100%;
47 | display: flex;
48 | flex-direction: row;
49 | border-radius: 1rem;
50 | padding: 1rem;
51 | margin: 0.5rem;
52 |
53 | &:hover {
54 | background-color: $secondaryColor;
55 | }
56 |
57 | div {
58 | display: flex;
59 | flex-direction: column;
60 | justify-content: center;
61 | align-items: center;
62 | width: 30%;
63 | height: 100%;
64 | }
65 |
66 | a {
67 | width: 70%;
68 | height: 100%;
69 | display: flex;
70 | justify-content: left;
71 | align-items: center;
72 | color: white;
73 | padding: 5px;
74 | font-family: $secondaryFont;
75 | font-weight: 500;
76 | font-size: 85%;
77 | }
78 | }
79 |
80 | .active-button {
81 | background-color: $secondaryColor;
82 | color: $quinaryColor;
83 |
84 | a {
85 | color: $quinaryColor;
86 | }
87 | }
88 | }
89 |
90 | #bottom-sidebar {
91 | width: 100%;
92 | height: 25%;
93 | display: flex;
94 | flex-direction: column;
95 | align-items: center;
96 | justify-content: center;
97 | padding: 15%;
98 | border-top: 1px solid rgba(211, 211, 211, 0.25);
99 | border-bottom: 1px solid rgba(211, 211, 211, 0.25);
100 |
101 | .sidebar-button {
102 | height: 50%;
103 | width: 100%;
104 | display: flex;
105 | padding: .5rem;
106 | flex-direction: row;
107 | border-radius: 1rem;
108 | &:hover {
109 | background-color: $zurauOrange
110 | }
111 |
112 | div {
113 | display: flex;
114 | flex-direction: column;
115 | justify-content: center;
116 | align-items: center;
117 | width: 30%;
118 | height: 100%;
119 | }
120 |
121 | a {
122 | width: 70%;
123 | height: 100%;
124 | display: flex;
125 | justify-content: left;
126 | align-items: center;
127 | color: white;
128 | padding: 5px;
129 | font-family: $secondaryFont;
130 | font-weight: 500;
131 | font-size: 100%;
132 | }
133 | }
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/client/styles/application.scss:
--------------------------------------------------------------------------------
1 | $primaryColor: #06172a;
2 | $secondaryColor: #123d57;
3 | $tertiaryColor: #1d6a92;
4 | $quarternaryColor: #2b9fc9;
5 | $quinaryColor: #2bcac8;
6 | $zurauOrange: #d16c0e;
7 |
8 | @import url('https://fonts.googleapis.com/css2?family=Josefin+Sans:wght@700&display=swap');
9 | @import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,100;0,300;0,400;0,700;0,900;1,100;1,300;1,400;1,700;1,900&display=swap');
10 | $titleFont: 'Josefin Sans', sans-serif;
11 | $secondaryFont: 'Lato', sans-serif;
12 |
13 | @import 'landing.scss';
14 | @import 'dashboard.scss';
15 | @import 'header.scss';
16 | @import 'sidebar.scss';
17 | @import 'login.scss';
18 | @import 'notifications.scss';
19 | @import 'ongoingMetrics.scss';
20 | //CSS Reset
21 | * {
22 | margin: 0;
23 | padding: 0;
24 | border: 0;
25 | font-size: 100%;
26 | font: inherit;
27 | vertical-align: baseline;
28 | box-sizing: border-box;
29 | font-family: 'Open Sans', sans-serif;
30 | // outline: 1px red solid;
31 | }
32 |
33 | html {
34 | width: 100vw;
35 | height: 100vh;
36 |
37 | body,
38 | #root,
39 | #app-container {
40 | width: 100%;
41 | height: 100%;
42 | min-height: 700px;
43 | min-width: 1000px;
44 | overflow: hidden;
45 |
46 | a {
47 | text-decoration: none;
48 | font-weight: 800;
49 | }
50 |
51 | input,
52 | button {
53 | border-radius: 5px;
54 | padding: 5px;
55 | }
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/consumer/index.js:
--------------------------------------------------------------------------------
1 | const ip = require('ip')
2 |
3 | const { Kafka, logLevel } = require('kafkajs')
4 |
5 | const host = process.env.HOST_IP || ip.address()
6 |
7 | const kafka = new Kafka({
8 | logLevel: logLevel.INFO,
9 | brokers: [`localhost:9092`],
10 | clientId: 'example-consumer',
11 | })
12 |
13 | const topic = 'topic-test'
14 | const consumer = kafka.consumer({ groupId: 'test-group' })
15 |
16 | const run = async () => {
17 | await consumer.connect()
18 | await consumer.subscribe({ topic, fromBeginning: true })
19 | await consumer.run({
20 | eachMessage: async ({ topic, partition, message }) => {
21 | const prefix = `${topic}[${partition} | ${message.offset}] / ${message.timestamp}`
22 | console.log(`- ${prefix} ${message.key}#${message.value}`)
23 | },
24 | })
25 | }
26 |
27 | run().catch(e => console.error(`[example/consumer] ${e.message}`, e))
28 |
29 | const errorTypes = ['unhandledRejection', 'uncaughtException']
30 | const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2']
31 |
32 | errorTypes.forEach(type => {
33 | process.on(type, async e => {
34 | try {
35 | console.log(`process.on ${type}`)
36 | console.error(e)
37 | await consumer.disconnect()
38 | process.exit(0)
39 | } catch (_) {
40 | process.exit(1)
41 | }
42 | })
43 | })
44 |
45 | signalTraps.forEach(type => {
46 | process.once(type, async () => {
47 | try {
48 | await consumer.disconnect()
49 | } finally {
50 | process.kill(process.pid, type)
51 | }
52 | })
53 | })
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | grafana:
4 | image: "grafana/grafana:latest"
5 | ports:
6 | - "3000:3000"
7 | environment:
8 | GF_PATHS_DATA : /var/lib/grafana
9 | GF_SECURITY_ADMIN_PASSWORD : kafka
10 | volumes:
11 | - ./grafana/provisioning:/etc/grafana/provisioning
12 | - ./grafana/dashboards:/var/lib/grafana/dashboards
13 | container_name: grafana
14 | depends_on:
15 | - prometheus
16 |
17 | prometheus:
18 | image: "prom/prometheus:latest"
19 | ports:
20 | - "9090:9090"
21 | volumes:
22 | - ./etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
23 | command: "--config.file=/etc/prometheus/prometheus.yml"
24 | container_name: prometheus
25 |
26 | jmx-kafka:
27 | image: "sscaling/jmx-prometheus-exporter"
28 | ports:
29 | - "5556:5556"
30 | environment:
31 | CONFIG_YML : "/etc/jmx_exporter/config.yml"
32 | volumes:
33 | - ./etc/jmx_exporter/config.yml:/etc/jmx_exporter/config.yml
34 | container_name: jmx-kafka
35 | depends_on:
36 | - kafka
37 |
38 | zk1:
39 | image: confluentinc/cp-zookeeper:latest
40 | environment:
41 | ZOOKEEPER_CLIENT_PORT: 2181
42 | ZOOKEEPER_TICK_TIME: 2000
43 | ZOOKEEPER_INIT_LIMIT: 5
44 | ZOOKEEPER_SYNC_LIMIT: 2
45 | ports:
46 | - "2181:2181"
47 | container_name: zookeeper1
48 |
49 | kafka:
50 | image: confluentinc/cp-kafka:latest
51 | depends_on:
52 | - zk1
53 | ports:
54 | - "9092:9092"
55 | - "9991:9991"
56 | container_name: kafka
57 | environment:
58 | KAFKA_BROKER_ID: 101
59 | KAFKA_JMX_PORT: 9991
60 | KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
61 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
62 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
63 | KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
64 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
65 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
66 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
67 | KAFKA_LOG_RETENTION_MS: 60000
68 | CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
69 | CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper1:2181
70 | CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
71 | CONFLUENT_METRICS_ENABLE: 'false'
72 | KAFKA_HEAP_OPTS: ${KAFKA_HEAP_OPTS}
73 | command: sh -c "((sleep 15 && kafka-topics --create --replication-factor 1 --partitions 3 --topic topic-test --bootstrap-server kafka:29092)&) && /etc/confluent/docker/run ">
--------------------------------------------------------------------------------
/etc/prometheus/prometheus.yml:
--------------------------------------------------------------------------------
1 | # my global config
2 | global:
3 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
4 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
5 | # scrape_timeout is set to the global default (10s).
6 |
7 | # Alertmanager configuration
8 | alerting:
9 | alertmanagers:
10 | - static_configs:
11 | - targets:
12 | # - alertmanager:9093
13 |
14 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
15 | rule_files:
16 | # - "first_rules.yml"
17 | # - "second_rules.yml"
18 |
19 | # A scrape configuration containing exactly one endpoint to scrape:
20 | # Here it's Prometheus itself.
21 | scrape_configs:
22 | # The job name is added as a label `job=` to any timeseries scraped from this config.
23 | - job_name: "kafka"
24 |
25 | # metrics_path defaults to '/metrics'
26 | # scheme defaults to 'http'.
27 |
28 | static_configs:
29 | - targets: ["jmx-kafka:5556"]
30 |
--------------------------------------------------------------------------------
/grafana/dashboards/grafana_dashboard_broker_hard_disk_usage.json:
--------------------------------------------------------------------------------
1 | {
2 | "__inputs": [
3 | {
4 | "name": "Prometheus",
5 | "label": "Prometheus",
6 | "description": "",
7 | "type": "datasource",
8 | "pluginId": "prometheus",
9 | "pluginName": "Prometheus"
10 | }
11 | ],
12 | "__requires": [
13 | {
14 | "type": "grafana",
15 | "id": "grafana",
16 | "name": "Grafana",
17 | "version": "5.2.1"
18 | },
19 | {
20 | "type": "panel",
21 | "id": "graph",
22 | "name": "Graph",
23 | "version": "5.0.0"
24 | },
25 | {
26 | "type": "datasource",
27 | "id": "prometheus",
28 | "name": "Prometheus",
29 | "version": "5.0.0"
30 | }
31 | ],
32 | "annotations": {
33 | "list": [
34 | {
35 | "builtIn": 1,
36 | "datasource": "-- Grafana --",
37 | "enable": true,
38 | "hide": true,
39 | "iconColor": "rgba(0, 211, 255, 1)",
40 | "name": "Annotations & Alerts",
41 | "type": "dashboard"
42 | }
43 | ]
44 | },
45 | "editable": true,
46 | "gnetId": null,
47 | "graphTooltip": 0,
48 | "id": null,
49 | "iteration": 1540328526402,
50 | "links": [],
51 | "panels": [
52 | {
53 | "aliasColors": {},
54 | "bars": false,
55 | "dashLength": 10,
56 | "dashes": false,
57 | "datasource": "Prometheus",
58 | "fill": 1,
59 | "gridPos": {
60 | "h": 9,
61 | "w": 12,
62 | "x": 0,
63 | "y": 0
64 | },
65 | "id": 2,
66 | "legend": {
67 | "alignAsTable": true,
68 | "avg": false,
69 | "current": true,
70 | "max": false,
71 | "min": false,
72 | "rightSide": true,
73 | "show": true,
74 | "sort": "current",
75 | "sortDesc": true,
76 | "total": false,
77 | "values": true
78 | },
79 | "lines": true,
80 | "linewidth": 1,
81 | "links": [],
82 | "nullPointMode": "null",
83 | "percentage": false,
84 | "pointradius": 5,
85 | "points": false,
86 | "renderer": "flot",
87 | "seriesOverrides": [],
88 | "spaceLength": 10,
89 | "stack": true,
90 | "steppedLine": false,
91 | "targets": [
92 | {
93 | "expr": "sum(kafka_log_size{env=\"$env\"}) by (topic)",
94 | "format": "time_series",
95 | "intervalFactor": 1,
96 | "legendFormat": "{{topic}}",
97 | "refId": "A"
98 | }
99 | ],
100 | "thresholds": [],
101 | "timeFrom": null,
102 | "timeShift": null,
103 | "title": "Global Topics Size (including replicas)",
104 | "tooltip": {
105 | "shared": true,
106 | "sort": 1,
107 | "value_type": "individual"
108 | },
109 | "type": "graph",
110 | "xaxis": {
111 | "buckets": null,
112 | "mode": "time",
113 | "name": null,
114 | "show": true,
115 | "values": []
116 | },
117 | "yaxes": [
118 | {
119 | "format": "decbytes",
120 | "label": null,
121 | "logBase": 1,
122 | "max": null,
123 | "min": null,
124 | "show": true
125 | },
126 | {
127 | "format": "short",
128 | "label": null,
129 | "logBase": 1,
130 | "max": null,
131 | "min": null,
132 | "show": true
133 | }
134 | ],
135 | "yaxis": {
136 | "align": false,
137 | "alignLevel": null
138 | }
139 | },
140 | {
141 | "aliasColors": {},
142 | "bars": false,
143 | "dashLength": 10,
144 | "dashes": false,
145 | "datasource": "Prometheus",
146 | "fill": 1,
147 | "gridPos": {
148 | "h": 9,
149 | "w": 12,
150 | "x": 12,
151 | "y": 0
152 | },
153 | "id": 5,
154 | "legend": {
155 | "alignAsTable": true,
156 | "avg": false,
157 | "current": true,
158 | "max": false,
159 | "min": false,
160 | "rightSide": true,
161 | "show": true,
162 | "total": false,
163 | "values": true
164 | },
165 | "lines": true,
166 | "linewidth": 1,
167 | "links": [],
168 | "nullPointMode": "null",
169 | "percentage": false,
170 | "pointradius": 5,
171 | "points": false,
172 | "renderer": "flot",
173 | "seriesOverrides": [],
174 | "spaceLength": 10,
175 | "stack": false,
176 | "steppedLine": false,
177 | "targets": [
178 | {
179 | "expr": "sum(sum(kafka_log_size{env=\"$env\"}) by (topic, partition) / sum(kafka_cluster_partition_replicascount{env=\"$env\"}) by(partition, topic)) by (topic)",
180 | "format": "time_series",
181 | "intervalFactor": 1,
182 | "legendFormat": "{{topic}}",
183 | "refId": "A"
184 | }
185 | ],
186 | "thresholds": [],
187 | "timeFrom": null,
188 | "timeShift": null,
189 | "title": "Global Topics Size (without replication)",
190 | "tooltip": {
191 | "shared": true,
192 | "sort": 0,
193 | "value_type": "individual"
194 | },
195 | "type": "graph",
196 | "xaxis": {
197 | "buckets": null,
198 | "mode": "time",
199 | "name": null,
200 | "show": true,
201 | "values": []
202 | },
203 | "yaxes": [
204 | {
205 | "format": "decbytes",
206 | "label": null,
207 | "logBase": 1,
208 | "max": null,
209 | "min": null,
210 | "show": true
211 | },
212 | {
213 | "format": "short",
214 | "label": null,
215 | "logBase": 1,
216 | "max": null,
217 | "min": null,
218 | "show": true
219 | }
220 | ],
221 | "yaxis": {
222 | "align": false,
223 | "alignLevel": null
224 | }
225 | },
226 | {
227 | "aliasColors": {},
228 | "bars": false,
229 | "dashLength": 10,
230 | "dashes": false,
231 | "datasource": "Prometheus",
232 | "fill": 1,
233 | "gridPos": {
234 | "h": 7,
235 | "w": 12,
236 | "x": 0,
237 | "y": 9
238 | },
239 | "id": 3,
240 | "legend": {
241 | "alignAsTable": false,
242 | "avg": false,
243 | "current": false,
244 | "max": false,
245 | "min": false,
246 | "rightSide": false,
247 | "show": true,
248 | "total": false,
249 | "values": false
250 | },
251 | "lines": true,
252 | "linewidth": 1,
253 | "links": [],
254 | "nullPointMode": "null",
255 | "percentage": false,
256 | "pointradius": 5,
257 | "points": false,
258 | "renderer": "flot",
259 | "seriesOverrides": [],
260 | "spaceLength": 10,
261 | "stack": true,
262 | "steppedLine": false,
263 | "targets": [
264 | {
265 | "expr": "sum(kafka_log_size{env=\"$env\"}) by (instance)",
266 | "format": "time_series",
267 | "intervalFactor": 1,
268 | "legendFormat": "{{instance}}",
269 | "refId": "A"
270 | }
271 | ],
272 | "thresholds": [],
273 | "timeFrom": null,
274 | "timeShift": null,
275 | "title": "Logs Size Per Broker (including replicas)",
276 | "tooltip": {
277 | "shared": true,
278 | "sort": 1,
279 | "value_type": "individual"
280 | },
281 | "type": "graph",
282 | "xaxis": {
283 | "buckets": null,
284 | "mode": "time",
285 | "name": null,
286 | "show": true,
287 | "values": []
288 | },
289 | "yaxes": [
290 | {
291 | "format": "decbytes",
292 | "label": null,
293 | "logBase": 1,
294 | "max": null,
295 | "min": null,
296 | "show": true
297 | },
298 | {
299 | "format": "short",
300 | "label": null,
301 | "logBase": 1,
302 | "max": null,
303 | "min": null,
304 | "show": true
305 | }
306 | ],
307 | "yaxis": {
308 | "align": false,
309 | "alignLevel": null
310 | }
311 | }
312 | ],
313 | "refresh": "5s",
314 | "schemaVersion": 16,
315 | "style": "dark",
316 | "tags": [
317 | "kafka",
318 | "topics",
319 | "disk"
320 | ],
321 | "templating": {
322 | "list": [
323 | {
324 | "allValue": null,
325 | "current": {},
326 | "datasource": "Prometheus",
327 | "hide": 0,
328 | "includeAll": false,
329 | "label": "Env",
330 | "multi": false,
331 | "name": "env",
332 | "options": [],
333 | "query": "label_values(kafka_log_size, env)",
334 | "refresh": 1,
335 | "regex": "",
336 | "sort": 0,
337 | "tagValuesQuery": "",
338 | "tags": [],
339 | "tagsQuery": "",
340 | "type": "query",
341 | "useTags": false
342 | }
343 | ]
344 | },
345 | "time": {
346 | "from": "now-1h",
347 | "to": "now"
348 | },
349 | "timepicker": {
350 | "refresh_intervals": [
351 | "5s",
352 | "10s",
353 | "30s",
354 | "1m",
355 | "5m",
356 | "15m",
357 | "30m",
358 | "1h",
359 | "2h",
360 | "1d"
361 | ],
362 | "time_options": [
363 | "5m",
364 | "15m",
365 | "1h",
366 | "6h",
367 | "12h",
368 | "24h",
369 | "2d",
370 | "7d",
371 | "30d"
372 | ]
373 | },
374 | "timezone": "",
375 | "title": "Kafka / Hard Disk Usage",
376 | "uid": "zApgMBbik",
377 | "version": 1
378 | }
--------------------------------------------------------------------------------
/grafana/dashboards/grafana_dashboard_broker_zookeeper.json:
--------------------------------------------------------------------------------
1 | {
2 | "__inputs": [
3 | {
4 | "name": "Prometheus",
5 | "label": "Prometheus",
6 | "description": "",
7 | "type": "datasource",
8 | "pluginId": "prometheus",
9 | "pluginName": "Prometheus"
10 | }
11 | ],
12 | "__requires": [
13 | {
14 | "type": "grafana",
15 | "id": "grafana",
16 | "name": "Grafana",
17 | "version": "5.2.1"
18 | },
19 | {
20 | "type": "panel",
21 | "id": "graph",
22 | "name": "Graph",
23 | "version": "5.0.0"
24 | },
25 | {
26 | "type": "datasource",
27 | "id": "prometheus",
28 | "name": "Prometheus",
29 | "version": "5.0.0"
30 | },
31 | {
32 | "type": "panel",
33 | "id": "text",
34 | "name": "Text",
35 | "version": "5.0.0"
36 | }
37 | ],
38 | "annotations": {
39 | "list": [
40 | {
41 | "builtIn": 1,
42 | "datasource": "-- Grafana --",
43 | "enable": true,
44 | "hide": true,
45 | "iconColor": "rgba(0, 211, 255, 1)",
46 | "name": "Annotations & Alerts",
47 | "type": "dashboard"
48 | }
49 | ]
50 | },
51 | "editable": true,
52 | "gnetId": null,
53 | "graphTooltip": 0,
54 | "id": null,
55 | "iteration": 1539721769054,
56 | "links": [],
57 | "panels": [
58 | {
59 | "collapsed": false,
60 | "gridPos": {
61 | "h": 1,
62 | "w": 24,
63 | "x": 0,
64 | "y": 0
65 | },
66 | "id": 8,
67 | "panels": [],
68 | "title": "Kafka/Zookeeper Session Expire Listener",
69 | "type": "row"
70 | },
71 | {
72 | "aliasColors": {},
73 | "bars": false,
74 | "dashLength": 10,
75 | "dashes": false,
76 | "datasource": "Prometheus",
77 | "editable": true,
78 | "error": false,
79 | "fill": 1,
80 | "grid": {},
81 | "gridPos": {
82 | "h": 7,
83 | "w": 8,
84 | "x": 0,
85 | "y": 1
86 | },
87 | "id": 1,
88 | "isNew": true,
89 | "legend": {
90 | "avg": false,
91 | "current": false,
92 | "max": false,
93 | "min": false,
94 | "show": true,
95 | "total": false,
96 | "values": false
97 | },
98 | "lines": true,
99 | "linewidth": 2,
100 | "links": [],
101 | "nullPointMode": "connected",
102 | "percentage": false,
103 | "pointradius": 5,
104 | "points": false,
105 | "renderer": "flot",
106 | "seriesOverrides": [],
107 | "spaceLength": 10,
108 | "stack": false,
109 | "steppedLine": false,
110 | "targets": [
111 | {
112 | "expr": "kafka_zookeeper_session_expire_listener_zookeepersyncconnectspersec{aggregate=\"OneMinute\", env=\"$env\"}",
113 | "intervalFactor": 2,
114 | "legendFormat": "{{instance}}",
115 | "refId": "A",
116 | "step": 4
117 | }
118 | ],
119 | "thresholds": [],
120 | "timeFrom": null,
121 | "timeShift": null,
122 | "title": "Zookeeper Sync Connects Per Sec",
123 | "tooltip": {
124 | "msResolution": false,
125 | "shared": true,
126 | "sort": 0,
127 | "value_type": "cumulative"
128 | },
129 | "type": "graph",
130 | "xaxis": {
131 | "buckets": null,
132 | "mode": "time",
133 | "name": null,
134 | "show": true,
135 | "values": []
136 | },
137 | "yaxes": [
138 | {
139 | "format": "short",
140 | "label": null,
141 | "logBase": 1,
142 | "max": null,
143 | "min": null,
144 | "show": true
145 | },
146 | {
147 | "format": "short",
148 | "label": null,
149 | "logBase": 1,
150 | "max": null,
151 | "min": null,
152 | "show": true
153 | }
154 | ],
155 | "yaxis": {
156 | "align": false,
157 | "alignLevel": null
158 | }
159 | },
160 | {
161 | "aliasColors": {},
162 | "bars": false,
163 | "dashLength": 10,
164 | "dashes": false,
165 | "datasource": "Prometheus",
166 | "editable": true,
167 | "error": false,
168 | "fill": 1,
169 | "grid": {},
170 | "gridPos": {
171 | "h": 7,
172 | "w": 8,
173 | "x": 8,
174 | "y": 1
175 | },
176 | "id": 2,
177 | "isNew": true,
178 | "legend": {
179 | "avg": false,
180 | "current": false,
181 | "max": false,
182 | "min": false,
183 | "show": true,
184 | "total": false,
185 | "values": false
186 | },
187 | "lines": true,
188 | "linewidth": 2,
189 | "links": [],
190 | "nullPointMode": "connected",
191 | "percentage": false,
192 | "pointradius": 5,
193 | "points": false,
194 | "renderer": "flot",
195 | "seriesOverrides": [],
196 | "spaceLength": 10,
197 | "stack": false,
198 | "steppedLine": false,
199 | "targets": [
200 | {
201 | "expr": "kafka_zookeeper_session_expire_listener_zookeeperdisconnectspersec{env=\"$env\", aggregate=\"OneMinute\"}",
202 | "intervalFactor": 2,
203 | "legendFormat": "{{instance}}",
204 | "refId": "A",
205 | "step": 4
206 | }
207 | ],
208 | "thresholds": [],
209 | "timeFrom": null,
210 | "timeShift": null,
211 | "title": "Zookeeper Disconnects Per Sec",
212 | "tooltip": {
213 | "msResolution": false,
214 | "shared": true,
215 | "sort": 0,
216 | "value_type": "cumulative"
217 | },
218 | "type": "graph",
219 | "xaxis": {
220 | "buckets": null,
221 | "mode": "time",
222 | "name": null,
223 | "show": true,
224 | "values": []
225 | },
226 | "yaxes": [
227 | {
228 | "format": "short",
229 | "label": null,
230 | "logBase": 1,
231 | "max": null,
232 | "min": null,
233 | "show": true
234 | },
235 | {
236 | "format": "short",
237 | "label": null,
238 | "logBase": 1,
239 | "max": null,
240 | "min": null,
241 | "show": true
242 | }
243 | ],
244 | "yaxis": {
245 | "align": false,
246 | "alignLevel": null
247 | }
248 | },
249 | {
250 | "content": "The ZooKeeper session has expired. **When a session expires, we can have leader changes and even a new controller.**\n\nIt is important to keep an eye on the number of such events across a Kafka cluster and if the overall number is high, then we have a few recommendations:\n\n* Check the health of your network\n* Check for garbage collection issues and tune it accordingly\n * If necessary, increase the session time out by setting the value of zookeeper.connection.timeout.ms.\n\nDocumentation : [monitoring.html#zookeeper-metrics](http://docs.confluent.io/current/kafka/monitoring.html#zookeeper-metrics)\n",
251 | "editable": true,
252 | "error": false,
253 | "gridPos": {
254 | "h": 7,
255 | "w": 8,
256 | "x": 16,
257 | "y": 1
258 | },
259 | "id": 7,
260 | "isNew": true,
261 | "links": [],
262 | "mode": "markdown",
263 | "title": "The ZooKeeper session has expired ?",
264 | "transparent": true,
265 | "type": "text"
266 | },
267 | {
268 | "aliasColors": {},
269 | "bars": false,
270 | "dashLength": 10,
271 | "dashes": false,
272 | "datasource": "Prometheus",
273 | "editable": true,
274 | "error": false,
275 | "fill": 1,
276 | "grid": {},
277 | "gridPos": {
278 | "h": 5,
279 | "w": 8,
280 | "x": 0,
281 | "y": 8
282 | },
283 | "id": 4,
284 | "isNew": true,
285 | "legend": {
286 | "avg": false,
287 | "current": false,
288 | "max": false,
289 | "min": false,
290 | "show": true,
291 | "total": false,
292 | "values": false
293 | },
294 | "lines": true,
295 | "linewidth": 2,
296 | "links": [],
297 | "nullPointMode": "connected",
298 | "percentage": false,
299 | "pointradius": 5,
300 | "points": false,
301 | "renderer": "flot",
302 | "seriesOverrides": [],
303 | "spaceLength": 10,
304 | "stack": false,
305 | "steppedLine": false,
306 | "targets": [
307 | {
308 | "expr": "kafka_zookeeper_session_expire_listener_zookeeperreadonlyconnectspersec{env=\"$env\", aggregate=\"OneMinute\"}",
309 | "intervalFactor": 2,
310 | "legendFormat": "{{instance}}",
311 | "refId": "A",
312 | "step": 4
313 | }
314 | ],
315 | "thresholds": [],
316 | "timeFrom": null,
317 | "timeShift": null,
318 | "title": "Zookeeper Read Only Connects Per Sec",
319 | "tooltip": {
320 | "msResolution": false,
321 | "shared": true,
322 | "sort": 0,
323 | "value_type": "cumulative"
324 | },
325 | "type": "graph",
326 | "xaxis": {
327 | "buckets": null,
328 | "mode": "time",
329 | "name": null,
330 | "show": true,
331 | "values": []
332 | },
333 | "yaxes": [
334 | {
335 | "format": "short",
336 | "label": null,
337 | "logBase": 1,
338 | "max": null,
339 | "min": null,
340 | "show": true
341 | },
342 | {
343 | "format": "short",
344 | "label": null,
345 | "logBase": 1,
346 | "max": null,
347 | "min": null,
348 | "show": true
349 | }
350 | ],
351 | "yaxis": {
352 | "align": false,
353 | "alignLevel": null
354 | }
355 | },
356 | {
357 | "aliasColors": {},
358 | "bars": false,
359 | "dashLength": 10,
360 | "dashes": false,
361 | "datasource": "Prometheus",
362 | "editable": true,
363 | "error": false,
364 | "fill": 1,
365 | "grid": {},
366 | "gridPos": {
367 | "h": 5,
368 | "w": 8,
369 | "x": 8,
370 | "y": 8
371 | },
372 | "id": 3,
373 | "isNew": true,
374 | "legend": {
375 | "avg": false,
376 | "current": false,
377 | "max": false,
378 | "min": false,
379 | "show": true,
380 | "total": false,
381 | "values": false
382 | },
383 | "lines": true,
384 | "linewidth": 2,
385 | "links": [],
386 | "nullPointMode": "connected",
387 | "percentage": false,
388 | "pointradius": 5,
389 | "points": false,
390 | "renderer": "flot",
391 | "seriesOverrides": [],
392 | "spaceLength": 10,
393 | "stack": false,
394 | "steppedLine": false,
395 | "targets": [
396 | {
397 | "expr": "kafka_zookeeper_session_expire_listener_zookeeperexpirespersec{env=\"$env\", aggregate=\"OneMinute\"}",
398 | "intervalFactor": 2,
399 | "legendFormat": "{{instance}}",
400 | "refId": "A",
401 | "step": 4
402 | }
403 | ],
404 | "thresholds": [],
405 | "timeFrom": null,
406 | "timeShift": null,
407 | "title": "Zookeeper Expires Per Sec",
408 | "tooltip": {
409 | "msResolution": false,
410 | "shared": true,
411 | "sort": 0,
412 | "value_type": "cumulative"
413 | },
414 | "type": "graph",
415 | "xaxis": {
416 | "buckets": null,
417 | "mode": "time",
418 | "name": null,
419 | "show": true,
420 | "values": []
421 | },
422 | "yaxes": [
423 | {
424 | "format": "short",
425 | "label": null,
426 | "logBase": 1,
427 | "max": null,
428 | "min": null,
429 | "show": true
430 | },
431 | {
432 | "format": "short",
433 | "label": null,
434 | "logBase": 1,
435 | "max": null,
436 | "min": null,
437 | "show": true
438 | }
439 | ],
440 | "yaxis": {
441 | "align": false,
442 | "alignLevel": null
443 | }
444 | },
445 | {
446 | "aliasColors": {},
447 | "bars": false,
448 | "dashLength": 10,
449 | "dashes": false,
450 | "datasource": "Prometheus",
451 | "editable": true,
452 | "error": false,
453 | "fill": 1,
454 | "grid": {},
455 | "gridPos": {
456 | "h": 5,
457 | "w": 8,
458 | "x": 0,
459 | "y": 13
460 | },
461 | "id": 6,
462 | "isNew": true,
463 | "legend": {
464 | "avg": false,
465 | "current": false,
466 | "max": false,
467 | "min": false,
468 | "show": true,
469 | "total": false,
470 | "values": false
471 | },
472 | "lines": true,
473 | "linewidth": 2,
474 | "links": [],
475 | "nullPointMode": "connected",
476 | "percentage": false,
477 | "pointradius": 5,
478 | "points": false,
479 | "renderer": "flot",
480 | "seriesOverrides": [],
481 | "spaceLength": 10,
482 | "stack": false,
483 | "steppedLine": false,
484 | "targets": [
485 | {
486 | "expr": "kafka_zookeeper_session_expire_listener_zookeeperauthfailurespersec{env=\"$env\", aggregate=\"OneMinute\"}",
487 | "intervalFactor": 2,
488 | "legendFormat": "{{instance}}",
489 | "refId": "A",
490 | "step": 4
491 | }
492 | ],
493 | "thresholds": [],
494 | "timeFrom": null,
495 | "timeShift": null,
496 | "title": "Zookeeper SASL Authentications Per Sec",
497 | "tooltip": {
498 | "msResolution": false,
499 | "shared": true,
500 | "sort": 0,
501 | "value_type": "cumulative"
502 | },
503 | "type": "graph",
504 | "xaxis": {
505 | "buckets": null,
506 | "mode": "time",
507 | "name": null,
508 | "show": true,
509 | "values": []
510 | },
511 | "yaxes": [
512 | {
513 | "format": "short",
514 | "label": null,
515 | "logBase": 1,
516 | "max": null,
517 | "min": null,
518 | "show": true
519 | },
520 | {
521 | "format": "short",
522 | "label": null,
523 | "logBase": 1,
524 | "max": null,
525 | "min": null,
526 | "show": true
527 | }
528 | ],
529 | "yaxis": {
530 | "align": false,
531 | "alignLevel": null
532 | }
533 | },
534 | {
535 | "aliasColors": {},
536 | "bars": false,
537 | "dashLength": 10,
538 | "dashes": false,
539 | "datasource": "Prometheus",
540 | "editable": true,
541 | "error": false,
542 | "fill": 1,
543 | "grid": {},
544 | "gridPos": {
545 | "h": 5,
546 | "w": 8,
547 | "x": 8,
548 | "y": 13
549 | },
550 | "id": 5,
551 | "isNew": true,
552 | "legend": {
553 | "avg": false,
554 | "current": false,
555 | "max": false,
556 | "min": false,
557 | "show": true,
558 | "total": false,
559 | "values": false
560 | },
561 | "lines": true,
562 | "linewidth": 2,
563 | "links": [],
564 | "nullPointMode": "connected",
565 | "percentage": false,
566 | "pointradius": 5,
567 | "points": false,
568 | "renderer": "flot",
569 | "seriesOverrides": [],
570 | "spaceLength": 10,
571 | "stack": false,
572 | "steppedLine": false,
573 | "targets": [
574 | {
575 | "expr": "kafka_zookeeper_session_expire_listener_zookeeperauthfailurespersec{env=\"$env\", aggregate=\"OneMinute\"}",
576 | "intervalFactor": 2,
577 | "legendFormat": "{{instance}}",
578 | "refId": "A",
579 | "step": 4
580 | }
581 | ],
582 | "thresholds": [],
583 | "timeFrom": null,
584 | "timeShift": null,
585 | "title": "Zookeeper Auth Failures Per Sec",
586 | "tooltip": {
587 | "msResolution": false,
588 | "shared": true,
589 | "sort": 0,
590 | "value_type": "cumulative"
591 | },
592 | "type": "graph",
593 | "xaxis": {
594 | "buckets": null,
595 | "mode": "time",
596 | "name": null,
597 | "show": true,
598 | "values": []
599 | },
600 | "yaxes": [
601 | {
602 | "format": "short",
603 | "label": null,
604 | "logBase": 1,
605 | "max": null,
606 | "min": null,
607 | "show": true
608 | },
609 | {
610 | "format": "short",
611 | "label": null,
612 | "logBase": 1,
613 | "max": null,
614 | "min": null,
615 | "show": true
616 | }
617 | ],
618 | "yaxis": {
619 | "align": false,
620 | "alignLevel": null
621 | }
622 | }
623 | ],
624 | "refresh": "5s",
625 | "schemaVersion": 16,
626 | "style": "dark",
627 | "tags": [
628 | "kafka",
629 | "zookeeper"
630 | ],
631 | "templating": {
632 | "list": [
633 | {
634 | "allValue": null,
635 | "current": {},
636 | "datasource": "Prometheus",
637 | "hide": 0,
638 | "includeAll": false,
639 | "label": "Env",
640 | "multi": false,
641 | "name": "env",
642 | "options": [],
643 | "query": "label_values(kafka_server_brokerstate, env)",
644 | "refresh": 1,
645 | "regex": "",
646 | "sort": 0,
647 | "tagValuesQuery": "",
648 | "tags": [],
649 | "tagsQuery": "",
650 | "type": "query",
651 | "useTags": false
652 | }
653 | ]
654 | },
655 | "time": {
656 | "from": "now-30m",
657 | "to": "now"
658 | },
659 | "timepicker": {
660 | "refresh_intervals": [
661 | "5s",
662 | "10s",
663 | "30s",
664 | "1m",
665 | "5m",
666 | "15m",
667 | "30m",
668 | "1h",
669 | "2h",
670 | "1d"
671 | ],
672 | "time_options": [
673 | "5m",
674 | "15m",
675 | "1h",
676 | "6h",
677 | "12h",
678 | "24h",
679 | "2d",
680 | "7d",
681 | "30d"
682 | ]
683 | },
684 | "timezone": "browser",
685 | "title": "Kafka Broker / Zookeeper Connection",
686 | "uid": "142Xi34mk",
687 | "version": 1
688 | }
--------------------------------------------------------------------------------
/grafana/dashboards/grafana_dashboard_client_consumers_fetch_lag.json:
--------------------------------------------------------------------------------
1 | {
2 | "__inputs": [
3 | {
4 | "name": "Prometheus",
5 | "label": "prometheus",
6 | "description": "",
7 | "type": "datasource",
8 | "pluginId": "prometheus",
9 | "pluginName": "Prometheus"
10 | }
11 | ],
12 | "__requires": [
13 | {
14 | "type": "grafana",
15 | "id": "grafana",
16 | "name": "Grafana",
17 | "version": "5.2.1"
18 | },
19 | {
20 | "type": "panel",
21 | "id": "graph",
22 | "name": "Graph",
23 | "version": "5.0.0"
24 | },
25 | {
26 | "type": "datasource",
27 | "id": "prometheus",
28 | "name": "Prometheus",
29 | "version": "5.0.0"
30 | }
31 | ],
32 | "annotations": {
33 | "list": [
34 | {
35 | "builtIn": 1,
36 | "datasource": "-- Grafana --",
37 | "enable": true,
38 | "hide": true,
39 | "iconColor": "rgba(0, 211, 255, 1)",
40 | "name": "Annotations & Alerts",
41 | "type": "dashboard"
42 | }
43 | ]
44 | },
45 | "description": "",
46 | "editable": true,
47 | "gnetId": null,
48 | "graphTooltip": 0,
49 | "id": null,
50 | "iteration": 1532113236152,
51 | "links": [],
52 | "panels": [
53 | {
54 | "aliasColors": {},
55 | "bars": false,
56 | "dashLength": 10,
57 | "dashes": false,
58 | "datasource": "Prometheus",
59 | "description": "The latest lag of each client",
60 | "fill": 1,
61 | "gridPos": {
62 | "h": 6,
63 | "w": 12,
64 | "x": 0,
65 | "y": 0
66 | },
67 | "id": 8,
68 | "legend": {
69 | "alignAsTable": true,
70 | "avg": false,
71 | "current": true,
72 | "max": false,
73 | "min": false,
74 | "rightSide": true,
75 | "show": true,
76 | "total": false,
77 | "values": true
78 | },
79 | "lines": true,
80 | "linewidth": 1,
81 | "links": [],
82 | "nullPointMode": "null",
83 | "percentage": false,
84 | "pointradius": 5,
85 | "points": false,
86 | "renderer": "flot",
87 | "seriesOverrides": [],
88 | "spaceLength": 10,
89 | "stack": false,
90 | "steppedLine": false,
91 | "targets": [
92 | {
93 | "expr": "sum(kafka_consumer_fetch_manager_metrics_records_lag{env=\"$env\", topic=\"$topic\"}) by(topic, client_id, instance)",
94 | "format": "time_series",
95 | "intervalFactor": 1,
96 | "legendFormat": "{{client_id}} ({{instance}})",
97 | "refId": "A"
98 | }
99 | ],
100 | "thresholds": [],
101 | "timeFrom": null,
102 | "timeShift": null,
103 | "title": "Consumer Client Lag",
104 | "tooltip": {
105 | "shared": true,
106 | "sort": 0,
107 | "value_type": "individual"
108 | },
109 | "type": "graph",
110 | "xaxis": {
111 | "buckets": null,
112 | "mode": "time",
113 | "name": null,
114 | "show": true,
115 | "values": []
116 | },
117 | "yaxes": [
118 | {
119 | "format": "short",
120 | "label": null,
121 | "logBase": 1,
122 | "max": null,
123 | "min": null,
124 | "show": true
125 | },
126 | {
127 | "format": "short",
128 | "label": null,
129 | "logBase": 1,
130 | "max": null,
131 | "min": null,
132 | "show": true
133 | }
134 | ],
135 | "yaxis": {
136 | "align": false,
137 | "alignLevel": null
138 | }
139 | },
140 | {
141 | "aliasColors": {},
142 | "bars": false,
143 | "dashLength": 10,
144 | "dashes": false,
145 | "datasource": "Prometheus",
146 | "description": "The average number of bytes fetched per request for a topic",
147 | "fill": 1,
148 | "gridPos": {
149 | "h": 6,
150 | "w": 12,
151 | "x": 12,
152 | "y": 0
153 | },
154 | "id": 10,
155 | "legend": {
156 | "alignAsTable": true,
157 | "avg": false,
158 | "current": true,
159 | "max": false,
160 | "min": false,
161 | "rightSide": true,
162 | "show": true,
163 | "total": false,
164 | "values": true
165 | },
166 | "lines": true,
167 | "linewidth": 1,
168 | "links": [],
169 | "nullPointMode": "null",
170 | "percentage": false,
171 | "pointradius": 5,
172 | "points": false,
173 | "renderer": "flot",
174 | "seriesOverrides": [],
175 | "spaceLength": 10,
176 | "stack": false,
177 | "steppedLine": false,
178 | "targets": [
179 | {
180 | "expr": "kafka_consumer_fetch_manager_metrics_fetch_size_avg{env=\"$env\", topic=\"$topic\"}",
181 | "format": "time_series",
182 | "intervalFactor": 1,
183 | "legendFormat": "{{client_id}} ({{instance}})",
184 | "refId": "A"
185 | }
186 | ],
187 | "thresholds": [],
188 | "timeFrom": null,
189 | "timeShift": null,
190 | "title": "Fetch Bytes Per Request",
191 | "tooltip": {
192 | "shared": true,
193 | "sort": 0,
194 | "value_type": "individual"
195 | },
196 | "type": "graph",
197 | "xaxis": {
198 | "buckets": null,
199 | "mode": "time",
200 | "name": null,
201 | "show": true,
202 | "values": []
203 | },
204 | "yaxes": [
205 | {
206 | "decimals": 0,
207 | "format": "decbytes",
208 | "label": "",
209 | "logBase": 1,
210 | "max": null,
211 | "min": "0",
212 | "show": true
213 | },
214 | {
215 | "format": "short",
216 | "label": null,
217 | "logBase": 1,
218 | "max": null,
219 | "min": null,
220 | "show": true
221 | }
222 | ],
223 | "yaxis": {
224 | "align": false,
225 | "alignLevel": null
226 | }
227 | },
228 | {
229 | "aliasColors": {},
230 | "bars": false,
231 | "dashLength": 10,
232 | "dashes": false,
233 | "datasource": "Prometheus",
234 | "description": "The average number of bytes consumed per second",
235 | "fill": 1,
236 | "gridPos": {
237 | "h": 6,
238 | "w": 12,
239 | "x": 0,
240 | "y": 6
241 | },
242 | "id": 3,
243 | "legend": {
244 | "alignAsTable": true,
245 | "avg": false,
246 | "current": true,
247 | "max": false,
248 | "min": false,
249 | "rightSide": true,
250 | "show": true,
251 | "total": false,
252 | "values": true
253 | },
254 | "lines": true,
255 | "linewidth": 1,
256 | "links": [],
257 | "nullPointMode": "null",
258 | "percentage": false,
259 | "pointradius": 5,
260 | "points": false,
261 | "renderer": "flot",
262 | "seriesOverrides": [],
263 | "spaceLength": 10,
264 | "stack": true,
265 | "steppedLine": false,
266 | "targets": [
267 | {
268 | "expr": "kafka_consumer_fetch_manager_metrics_bytes_consumed_rate{env=\"$env\", topic=\"$topic\"}",
269 | "format": "time_series",
270 | "hide": false,
271 | "instant": false,
272 | "intervalFactor": 1,
273 | "legendFormat": "{{client_id}} ({{instance}})",
274 | "refId": "A"
275 | }
276 | ],
277 | "thresholds": [],
278 | "timeFrom": null,
279 | "timeShift": null,
280 | "title": "Consumed Bytes Rate Per Second",
281 | "tooltip": {
282 | "shared": false,
283 | "sort": 0,
284 | "value_type": "individual"
285 | },
286 | "type": "graph",
287 | "xaxis": {
288 | "buckets": null,
289 | "mode": "time",
290 | "name": null,
291 | "show": true,
292 | "values": []
293 | },
294 | "yaxes": [
295 | {
296 | "decimals": 0,
297 | "format": "Bps",
298 | "label": null,
299 | "logBase": 1,
300 | "max": null,
301 | "min": null,
302 | "show": true
303 | },
304 | {
305 | "format": "short",
306 | "label": null,
307 | "logBase": 1,
308 | "max": null,
309 | "min": null,
310 | "show": true
311 | }
312 | ],
313 | "yaxis": {
314 | "align": false,
315 | "alignLevel": null
316 | }
317 | },
318 | {
319 | "aliasColors": {},
320 | "bars": false,
321 | "dashLength": 10,
322 | "dashes": false,
323 | "datasource": "Prometheus",
324 | "description": "The average number of records consumed per second",
325 | "fill": 1,
326 | "gridPos": {
327 | "h": 6,
328 | "w": 12,
329 | "x": 12,
330 | "y": 6
331 | },
332 | "id": 2,
333 | "legend": {
334 | "alignAsTable": true,
335 | "avg": false,
336 | "current": true,
337 | "max": false,
338 | "min": false,
339 | "rightSide": true,
340 | "show": true,
341 | "total": false,
342 | "values": true
343 | },
344 | "lines": true,
345 | "linewidth": 1,
346 | "links": [],
347 | "nullPointMode": "null",
348 | "percentage": false,
349 | "pointradius": 5,
350 | "points": false,
351 | "renderer": "flot",
352 | "seriesOverrides": [],
353 | "spaceLength": 10,
354 | "stack": true,
355 | "steppedLine": false,
356 | "targets": [
357 | {
358 | "expr": "kafka_consumer_fetch_manager_metrics_records_consumed_rate{env=\"$env\", topic=\"$topic\"}",
359 | "format": "time_series",
360 | "hide": false,
361 | "intervalFactor": 1,
362 | "legendFormat": "{{client_id}} ({{instance}})",
363 | "refId": "A"
364 | }
365 | ],
366 | "thresholds": [],
367 | "timeFrom": null,
368 | "timeShift": null,
369 | "title": "Consumed Records Rate Per Second",
370 | "tooltip": {
371 | "shared": false,
372 | "sort": 0,
373 | "value_type": "individual"
374 | },
375 | "type": "graph",
376 | "xaxis": {
377 | "buckets": null,
378 | "mode": "time",
379 | "name": null,
380 | "show": true,
381 | "values": []
382 | },
383 | "yaxes": [
384 | {
385 | "decimals": 0,
386 | "format": "rps",
387 | "label": null,
388 | "logBase": 1,
389 | "max": null,
390 | "min": null,
391 | "show": true
392 | },
393 | {
394 | "format": "short",
395 | "label": null,
396 | "logBase": 1,
397 | "max": null,
398 | "min": null,
399 | "show": true
400 | }
401 | ],
402 | "yaxis": {
403 | "align": false,
404 | "alignLevel": null
405 | }
406 | }
407 | ],
408 | "refresh": "5s",
409 | "schemaVersion": 16,
410 | "style": "dark",
411 | "tags": [
412 | "kafka",
413 | "clients"
414 | ],
415 | "templating": {
416 | "list": [
417 | {
418 | "allValue": null,
419 | "current": {},
420 | "datasource": "Prometheus",
421 | "hide": 0,
422 | "includeAll": false,
423 | "label": "env",
424 | "multi": false,
425 | "name": "env",
426 | "options": [],
427 | "query": "label_values(kafka_consumer_fetch_manager_metrics_records_consumed_rate, env)",
428 | "refresh": 1,
429 | "regex": "",
430 | "sort": 0,
431 | "tagValuesQuery": "",
432 | "tags": [],
433 | "tagsQuery": "",
434 | "type": "query",
435 | "useTags": false
436 | },
437 | {
438 | "allValue": null,
439 | "current": {},
440 | "datasource": "Prometheus",
441 | "hide": 0,
442 | "includeAll": false,
443 | "label": "topic",
444 | "multi": false,
445 | "name": "topic",
446 | "options": [],
447 | "query": "label_values(kafka_consumer_fetch_manager_metrics_records_consumed_rate, topic)",
448 | "refresh": 1,
449 | "regex": "",
450 | "sort": 0,
451 | "tagValuesQuery": "",
452 | "tags": [],
453 | "tagsQuery": "",
454 | "type": "query",
455 | "useTags": false
456 | }
457 | ]
458 | },
459 | "time": {
460 | "from": "now-1h",
461 | "to": "now"
462 | },
463 | "timepicker": {
464 | "refresh_intervals": [
465 | "5s",
466 | "10s",
467 | "30s",
468 | "1m",
469 | "5m",
470 | "15m",
471 | "30m",
472 | "1h",
473 | "2h",
474 | "1d"
475 | ],
476 | "time_options": [
477 | "5m",
478 | "15m",
479 | "1h",
480 | "6h",
481 | "12h",
482 | "24h",
483 | "2d",
484 | "7d",
485 | "30d"
486 | ]
487 | },
488 | "timezone": "browser",
489 | "title": "Kafka Consumers / Fetch Rate & Records Lag",
490 | "uid": "Ni95Y3dmz",
491 | "version": 20
492 | }
493 |
--------------------------------------------------------------------------------
/grafana/dashboards/grafana_dashboard_cluster_replication.json:
--------------------------------------------------------------------------------
1 | {
2 | "__inputs": [
3 | {
4 | "name": "Prometheus",
5 | "label": "Prometheus",
6 | "description": "",
7 | "type": "datasource",
8 | "pluginId": "prometheus",
9 | "pluginName": "Prometheus"
10 | }
11 | ],
12 | "__requires": [
13 | {
14 | "type": "grafana",
15 | "id": "grafana",
16 | "name": "Grafana",
17 | "version": "5.2.1"
18 | },
19 | {
20 | "type": "panel",
21 | "id": "graph",
22 | "name": "Graph",
23 | "version": "5.0.0"
24 | },
25 | {
26 | "type": "datasource",
27 | "id": "prometheus",
28 | "name": "Prometheus",
29 | "version": "5.0.0"
30 | }
31 | ],
32 | "annotations": {
33 | "list": [
34 | {
35 | "builtIn": 1,
36 | "datasource": "-- Grafana --",
37 | "enable": true,
38 | "hide": true,
39 | "iconColor": "rgba(0, 211, 255, 1)",
40 | "name": "Annotations & Alerts",
41 | "type": "dashboard"
42 | }
43 | ]
44 | },
45 | "editable": true,
46 | "gnetId": null,
47 | "graphTooltip": 0,
48 | "id": null,
49 | "iteration": 1540330422283,
50 | "links": [],
51 | "panels": [
52 | {
53 | "aliasColors": {},
54 | "bars": false,
55 | "dashLength": 10,
56 | "dashes": false,
57 | "datasource": "Prometheus",
58 | "editable": true,
59 | "error": false,
60 | "fill": 1,
61 | "grid": {},
62 | "gridPos": {
63 | "h": 7,
64 | "w": 8,
65 | "x": 0,
66 | "y": 0
67 | },
68 | "id": 1,
69 | "isNew": true,
70 | "legend": {
71 | "alignAsTable": false,
72 | "avg": false,
73 | "current": false,
74 | "hideEmpty": false,
75 | "hideZero": true,
76 | "max": false,
77 | "min": false,
78 | "rightSide": false,
79 | "show": true,
80 | "total": false,
81 | "values": false
82 | },
83 | "lines": true,
84 | "linewidth": 2,
85 | "links": [],
86 | "nullPointMode": "connected",
87 | "percentage": false,
88 | "pointradius": 5,
89 | "points": false,
90 | "renderer": "flot",
91 | "seriesOverrides": [],
92 | "spaceLength": 10,
93 | "stack": true,
94 | "steppedLine": false,
95 | "targets": [
96 | {
97 | "expr": "kafka_server_broker_topic_metrics_replicationbytesinpersec_rate{aggregate=\"OneMinute\",env=\"$env\"}",
98 | "format": "time_series",
99 | "intervalFactor": 2,
100 | "legendFormat": "{{instance}}",
101 | "refId": "A",
102 | "step": 60
103 | }
104 | ],
105 | "thresholds": [],
106 | "timeFrom": null,
107 | "timeShift": null,
108 | "title": "Replication BytesIn/Sec",
109 | "tooltip": {
110 | "msResolution": false,
111 | "shared": true,
112 | "sort": 1,
113 | "value_type": "cumulative"
114 | },
115 | "type": "graph",
116 | "xaxis": {
117 | "buckets": null,
118 | "mode": "time",
119 | "name": null,
120 | "show": true,
121 | "values": []
122 | },
123 | "yaxes": [
124 | {
125 | "format": "Bps",
126 | "label": null,
127 | "logBase": 1,
128 | "max": null,
129 | "min": 0,
130 | "show": true
131 | },
132 | {
133 | "format": "short",
134 | "label": null,
135 | "logBase": 1,
136 | "max": null,
137 | "min": 0,
138 | "show": true
139 | }
140 | ],
141 | "yaxis": {
142 | "align": false,
143 | "alignLevel": null
144 | }
145 | },
146 | {
147 | "aliasColors": {},
148 | "bars": false,
149 | "dashLength": 10,
150 | "dashes": false,
151 | "datasource": "Prometheus",
152 | "editable": true,
153 | "error": false,
154 | "fill": 1,
155 | "grid": {},
156 | "gridPos": {
157 | "h": 7,
158 | "w": 8,
159 | "x": 8,
160 | "y": 0
161 | },
162 | "id": 2,
163 | "isNew": true,
164 | "legend": {
165 | "avg": false,
166 | "current": false,
167 | "hideEmpty": false,
168 | "hideZero": true,
169 | "max": false,
170 | "min": false,
171 | "show": true,
172 | "total": false,
173 | "values": false
174 | },
175 | "lines": true,
176 | "linewidth": 2,
177 | "links": [],
178 | "nullPointMode": "connected",
179 | "percentage": false,
180 | "pointradius": 5,
181 | "points": false,
182 | "renderer": "flot",
183 | "seriesOverrides": [],
184 | "spaceLength": 10,
185 | "stack": true,
186 | "steppedLine": false,
187 | "targets": [
188 | {
189 | "expr": "kafka_server_broker_topic_metrics_replicationbytesoutpersec_rate{aggregate=\"OneMinute\",env=\"$env\"}",
190 | "format": "time_series",
191 | "intervalFactor": 2,
192 | "legendFormat": "{{instance}}",
193 | "refId": "A",
194 | "step": 60
195 | }
196 | ],
197 | "thresholds": [],
198 | "timeFrom": null,
199 | "timeShift": null,
200 | "title": "Replication BytesOut/Sec",
201 | "tooltip": {
202 | "msResolution": false,
203 | "shared": true,
204 | "sort": 1,
205 | "value_type": "cumulative"
206 | },
207 | "type": "graph",
208 | "xaxis": {
209 | "buckets": null,
210 | "mode": "time",
211 | "name": null,
212 | "show": true,
213 | "values": []
214 | },
215 | "yaxes": [
216 | {
217 | "format": "Bps",
218 | "label": null,
219 | "logBase": 1,
220 | "max": null,
221 | "min": 0,
222 | "show": true
223 | },
224 | {
225 | "format": "short",
226 | "label": null,
227 | "logBase": 1,
228 | "max": null,
229 | "min": 0,
230 | "show": true
231 | }
232 | ],
233 | "yaxis": {
234 | "align": false,
235 | "alignLevel": null
236 | }
237 | },
238 | {
239 | "aliasColors": {},
240 | "bars": false,
241 | "dashLength": 10,
242 | "dashes": false,
243 | "datasource": "Prometheus",
244 | "fill": 1,
245 | "gridPos": {
246 | "h": 7,
247 | "w": 8,
248 | "x": 16,
249 | "y": 0
250 | },
251 | "id": 6,
252 | "legend": {
253 | "avg": false,
254 | "current": false,
255 | "max": false,
256 | "min": false,
257 | "show": true,
258 | "total": false,
259 | "values": false
260 | },
261 | "lines": true,
262 | "linewidth": 1,
263 | "links": [],
264 | "nullPointMode": "null",
265 | "percentage": false,
266 | "pointradius": 5,
267 | "points": false,
268 | "renderer": "flot",
269 | "seriesOverrides": [],
270 | "spaceLength": 10,
271 | "stack": false,
272 | "steppedLine": false,
273 | "targets": [
274 | {
275 | "expr": "kafka_server_replica_fetcher_manager_maxlag_value{env=\"$env\"}",
276 | "format": "time_series",
277 | "intervalFactor": 1,
278 | "legendFormat": "{{instance}}",
279 | "refId": "A"
280 | }
281 | ],
282 | "thresholds": [],
283 | "timeFrom": null,
284 | "timeShift": null,
285 | "title": "Replication Max Lag Per Broker",
286 | "tooltip": {
287 | "shared": true,
288 | "sort": 0,
289 | "value_type": "individual"
290 | },
291 | "type": "graph",
292 | "xaxis": {
293 | "buckets": null,
294 | "mode": "time",
295 | "name": null,
296 | "show": true,
297 | "values": []
298 | },
299 | "yaxes": [
300 | {
301 | "format": "short",
302 | "label": null,
303 | "logBase": 1,
304 | "max": null,
305 | "min": "0",
306 | "show": true
307 | },
308 | {
309 | "format": "short",
310 | "label": null,
311 | "logBase": 1,
312 | "max": null,
313 | "min": null,
314 | "show": true
315 | }
316 | ],
317 | "yaxis": {
318 | "align": false,
319 | "alignLevel": null
320 | }
321 | },
322 | {
323 | "aliasColors": {},
324 | "bars": false,
325 | "dashLength": 10,
326 | "dashes": false,
327 | "datasource": "Prometheus",
328 | "decimals": null,
329 | "editable": true,
330 | "error": false,
331 | "fill": 1,
332 | "grid": {},
333 | "gridPos": {
334 | "h": 5,
335 | "w": 24,
336 | "x": 0,
337 | "y": 7
338 | },
339 | "id": 3,
340 | "isNew": true,
341 | "legend": {
342 | "alignAsTable": true,
343 | "avg": false,
344 | "current": false,
345 | "max": false,
346 | "min": false,
347 | "rightSide": true,
348 | "show": true,
349 | "total": false,
350 | "values": false
351 | },
352 | "lines": true,
353 | "linewidth": 2,
354 | "links": [],
355 | "nullPointMode": "connected",
356 | "percentage": false,
357 | "pointradius": 5,
358 | "points": false,
359 | "renderer": "flot",
360 | "seriesOverrides": [],
361 | "spaceLength": 10,
362 | "stack": false,
363 | "steppedLine": false,
364 | "targets": [
365 | {
366 | "expr": "kafka_server_fetcher_stats_bytespersec{env=\"$env\", aggregate=\"OneMinute\"}",
367 | "format": "time_series",
368 | "intervalFactor": 2,
369 | "legendFormat": "Host {{instance}}, client_id={{client_id}} / {{broker_host}}:{{broker_port}}",
370 | "refId": "A",
371 | "step": 60
372 | }
373 | ],
374 | "thresholds": [],
375 | "timeFrom": null,
376 | "timeShift": null,
377 | "title": "Fetch Bytes/sec Per Replication thread",
378 | "tooltip": {
379 | "msResolution": false,
380 | "shared": true,
381 | "sort": 0,
382 | "value_type": "cumulative"
383 | },
384 | "type": "graph",
385 | "xaxis": {
386 | "buckets": null,
387 | "mode": "time",
388 | "name": null,
389 | "show": true,
390 | "values": []
391 | },
392 | "yaxes": [
393 | {
394 | "format": "Bps",
395 | "label": null,
396 | "logBase": 1,
397 | "max": null,
398 | "min": null,
399 | "show": true
400 | },
401 | {
402 | "format": "short",
403 | "label": null,
404 | "logBase": 1,
405 | "max": null,
406 | "min": null,
407 | "show": true
408 | }
409 | ],
410 | "yaxis": {
411 | "align": false,
412 | "alignLevel": null
413 | }
414 | },
415 | {
416 | "aliasColors": {},
417 | "bars": false,
418 | "dashLength": 10,
419 | "dashes": false,
420 | "datasource": "Prometheus",
421 | "decimals": null,
422 | "editable": true,
423 | "error": false,
424 | "fill": 1,
425 | "grid": {},
426 | "gridPos": {
427 | "h": 5,
428 | "w": 24,
429 | "x": 0,
430 | "y": 12
431 | },
432 | "id": 4,
433 | "isNew": true,
434 | "legend": {
435 | "alignAsTable": true,
436 | "avg": false,
437 | "current": false,
438 | "max": false,
439 | "min": false,
440 | "rightSide": true,
441 | "show": true,
442 | "total": false,
443 | "values": false
444 | },
445 | "lines": true,
446 | "linewidth": 2,
447 | "links": [],
448 | "nullPointMode": "connected",
449 | "percentage": false,
450 | "pointradius": 5,
451 | "points": false,
452 | "renderer": "flot",
453 | "seriesOverrides": [],
454 | "spaceLength": 10,
455 | "stack": false,
456 | "steppedLine": false,
457 | "targets": [
458 | {
459 | "expr": "kafka_server_fetcher_stats_requestspersec{env=\"$env\", aggregate=\"OneMinute\"}",
460 | "format": "time_series",
461 | "intervalFactor": 2,
462 | "legendFormat": "Host {{instance}}, client_id={{client_id}} / {{broker_host}}:{{broker_port}}",
463 | "refId": "A",
464 | "step": 60
465 | }
466 | ],
467 | "thresholds": [],
468 | "timeFrom": null,
469 | "timeShift": null,
470 | "title": "Fetch Requests/sec Per Replication thread",
471 | "tooltip": {
472 | "msResolution": false,
473 | "shared": true,
474 | "sort": 0,
475 | "value_type": "cumulative"
476 | },
477 | "type": "graph",
478 | "xaxis": {
479 | "buckets": null,
480 | "mode": "time",
481 | "name": null,
482 | "show": true,
483 | "values": []
484 | },
485 | "yaxes": [
486 | {
487 | "format": "reqps",
488 | "label": null,
489 | "logBase": 1,
490 | "max": null,
491 | "min": null,
492 | "show": true
493 | },
494 | {
495 | "format": "short",
496 | "label": null,
497 | "logBase": 1,
498 | "max": null,
499 | "min": null,
500 | "show": true
501 | }
502 | ],
503 | "yaxis": {
504 | "align": false,
505 | "alignLevel": null
506 | }
507 | }
508 | ],
509 | "refresh": "5s",
510 | "schemaVersion": 16,
511 | "style": "dark",
512 | "tags": [
513 | "kafka",
514 | "broker",
515 | "replication"
516 | ],
517 | "templating": {
518 | "list": [
519 | {
520 | "allValue": null,
521 | "current": {},
522 | "datasource": "Prometheus",
523 | "hide": 0,
524 | "includeAll": false,
525 | "label": "Env",
526 | "multi": false,
527 | "name": "env",
528 | "options": [],
529 | "query": "label_values(kafka_server_brokerstate, env)",
530 | "refresh": 1,
531 | "regex": "",
532 | "sort": 0,
533 | "tagValuesQuery": "",
534 | "tags": [],
535 | "tagsQuery": "",
536 | "type": "query",
537 | "useTags": false
538 | }
539 | ]
540 | },
541 | "time": {
542 | "from": "now-1h",
543 | "to": "now"
544 | },
545 | "timepicker": {
546 | "refresh_intervals": [
547 | "5s",
548 | "10s",
549 | "30s",
550 | "1m",
551 | "5m",
552 | "15m",
553 | "30m",
554 | "1h",
555 | "2h",
556 | "1d"
557 | ],
558 | "time_options": [
559 | "5m",
560 | "15m",
561 | "1h",
562 | "6h",
563 | "12h",
564 | "24h",
565 | "2d",
566 | "7d",
567 | "30d"
568 | ]
569 | },
570 | "timezone": "browser",
571 | "title": "Kafka Cluster / Replication",
572 | "uid": "E9bXmq4ik",
573 | "version": 2
574 | }
--------------------------------------------------------------------------------
/grafana/provisioning/dashboards/kafka.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: 1
2 |
3 | providers:
4 | - name: 'default'
5 | orgId: 1
6 | folder: ''
7 | type: file
8 | disableDeletion: false
9 | editable: true
10 | updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards
11 | options:
12 | path: /var/lib/grafana/dashboards
13 |
--------------------------------------------------------------------------------
/grafana/provisioning/datasources/datasource.yaml:
--------------------------------------------------------------------------------
1 | # config file version
2 | apiVersion: 1
3 |
4 | # list of datasources that should be deleted from the database
5 | deleteDatasources:
6 | - name: Prometheus
7 | orgId: 1
8 |
9 | # list of datasources to insert/update depending
10 | # what's available in the database
11 | datasources:
12 | # name of the datasource. Required
13 | - name: Prometheus
14 | # datasource type. Required
15 | type: prometheus
16 | # access mode. proxy or direct (Server or Browser in the UI). Required
17 | access: direct
18 | # org id. will default to orgId 1 if not specified
19 | orgId: 1
20 | # url
21 | url: http://localhost:9090
22 | # database password, if used
23 | password:
24 | # database user, if used
25 | user:
26 | # database name, if used
27 | database:
28 | # enable/disable basic auth
29 | basicAuth:
30 | # basic auth username
31 | basicAuthUser:
32 | # basic auth password
33 | basicAuthPassword:
34 | # enable/disable with credentials headers
35 | withCredentials:
36 | # mark as default datasource. Max one per org
37 | isDefault: true
38 | # fields that will be converted to json and stored in json_data
39 | jsonData:
40 | graphiteVersion: "1.1"
41 | tlsAuth: true
42 | tlsAuthWithCACert: true
43 | # json object of data that will be encrypted.
44 | secureJsonData:
45 | tlsCACert: "..."
46 | tlsClientCert: "..."
47 | tlsClientKey: "..."
48 | version: 1
49 | # allow users to edit datasources from the UI.
50 | editable: true
51 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | KafkaPeek
6 |
11 |
12 |
29 |
30 |
31 |
36 |
37 |
38 | Keep a keen eye on your
39 |
42 |
43 | {' '}
44 | KafkaPeek helps engineering teams of all sizes quickly and easily stay up
45 | to date on the status of their deployments
46 |
47 |
48 |
"
129 | // ],
130 |
131 | // Allows you to use a custom runner instead of Jest's default test runner
132 | // runner: "jest-runner",
133 |
134 | // The paths to modules that run some code to configure or set up the testing environment before each test
135 | // setupFiles: [],
136 |
137 | // A list of paths to modules that run some code to configure or set up the testing framework before each test
138 | // setupFilesAfterEnv: [],
139 |
140 | // The number of seconds after which a test is considered as slow and reported as such in the results.
141 | // slowTestThreshold: 5,
142 |
143 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing
144 | // snapshotSerializers: [],
145 |
146 | // The test environment that will be used for testing
147 | // testEnvironment: "jest-environment-node",
148 |
149 | // Options that will be passed to the testEnvironment
150 | // testEnvironmentOptions: {},
151 |
152 | // Adds a location field to test results
153 | // testLocationInResults: false,
154 |
155 | // The glob patterns Jest uses to detect test files
156 | // testMatch: [
157 | // "**/__tests__/**/*.[jt]s?(x)",
158 | // "**/?(*.)+(spec|test).[tj]s?(x)"
159 | // ],
160 |
161 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
162 | // testPathIgnorePatterns: [
163 | // "/node_modules/"
164 | // ],
165 |
166 | // The regexp pattern or array of patterns that Jest uses to detect test files
167 | // testRegex: [],
168 |
169 | // This option allows the use of a custom results processor
170 | // testResultsProcessor: undefined,
171 |
172 | // This option allows use of a custom test runner
173 | // testRunner: "jest-circus/runner",
174 |
175 | // A map from regular expressions to paths to transformers
176 | transform: {
177 | "^.+\\.(js|jsx)$": "babel-jest"
178 | },
179 |
180 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
181 | // transformIgnorePatterns: [
182 | // "/node_modules/",
183 | // "\\.pnp\\.[^\\/]+$"
184 | // ],
185 |
186 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
187 | // unmockedModulePathPatterns: undefined,
188 |
189 | // Indicates whether each individual test should be reported during the run
190 | verbose: true,
191 |
192 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
193 | // watchPathIgnorePatterns: [],
194 |
195 | // Whether to use watchman for file crawling
196 | // watchman: true,
197 | };
198 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "KafkaPeek",
3 | "version": "1.0.0",
4 | "description": "Kafka observability and notification tool",
5 | "main": "index.js",
6 | "scripts": {
7 | "dev": "NODE_ENV=development webpack-dev-server --open & nodemon server/server.js",
8 | "build": "NODE_ENV=production webpack",
9 | "start": "NODE_ENV=production node server/server.js",
10 | "start:consumer": "node ./consumer/index.js",
11 | "start:producer": "node ./producer/index.js",
12 | "test": "jest --detectOpenHandles"
13 | },
14 | "repository": {
15 | "type": "git",
16 | "url": "git+https://github.com/oslabs-beta/KafkaPeek.git"
17 | },
18 | "keywords": [],
19 | "author": "",
20 | "license": "ISC",
21 | "bugs": {
22 | "url": "https://github.com/oslabs-beta/KafkaPeek/issues"
23 | },
24 | "homepage": "https://github.com/oslabs-beta/KafkaPeek#readme",
25 | "devDependencies": {
26 | "@babel/core": "^7.19.1",
27 | "@babel/preset-env": "^7.19.1",
28 | "@babel/preset-react": "^7.18.6",
29 | "@types/jest": "^29.1.2",
30 | "babel-loader": "^8.2.5",
31 | "css-loader": "^6.7.1",
32 | "eslint": "^8.23.1",
33 | "eslint-plugin-react": "^7.31.8",
34 | "fs": "^0.0.1-security",
35 | "html-webpack-plugin": "^5.5.0",
36 | "jest": "^29.1.2",
37 | "node-sass": "^7.0.3",
38 | "nodemon": "^2.0.20",
39 | "regenerator-runtime": "^0.13.9",
40 | "sass-loader": "^13.0.2",
41 | "style-loader": "^3.3.1",
42 | "supertest": "^6.3.0",
43 | "svg-inline-loader": "^0.8.2",
44 | "url-loader": "^4.1.1",
45 | "webpack": "^5.74.0",
46 | "webpack-cli": "^4.10.0",
47 | "webpack-dev-server": "^4.11.1"
48 | },
49 | "dependencies": {
50 | "apexcharts": "^3.35.5",
51 | "avsc": "^5.7.5",
52 | "aws-sdk": "^2.1219.0",
53 | "axios": "^0.27.2",
54 | "body-parser": "^1.20.0",
55 | "cookie-session": "^2.0.0",
56 | "cors": "^2.8.5",
57 | "dotenv": "^16.0.3",
58 | "express": "^4.18.1",
59 | "ip": "^2.0.0",
60 | "kafkajs": "^2.2.0",
61 | "mongoose": "^6.6.1",
62 | "node-fetch": "^3.2.10",
63 | "path": "^0.12.7",
64 | "pg": "^8.8.0",
65 | "react": "^18.2.0",
66 | "react-apexcharts": "^1.4.0",
67 | "react-dom": "^18.2.0",
68 | "react-router-dom": "^6.4.0",
69 | "react-select": "^5.4.0",
70 | "sass": "^1.54.9",
71 | "save": "^2.5.0",
72 | "socket.io": "^4.5.2",
73 | "socket.io-client": "^4.5.2"
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/producer/index.js:
--------------------------------------------------------------------------------
1 | const ip = require('ip')
2 |
3 | const { Kafka, CompressionTypes, logLevel } = require('kafkajs')
4 |
5 | const host = process.env.HOST_IP || ip.address()
6 |
7 | const kafka = new Kafka({
8 | logLevel: logLevel.DEBUG,
9 | brokers: [`localhost:9092`],
10 | clientId: 'example-producer',
11 | })
12 |
13 | const topic = 'topic-test'
14 | const producer = kafka.producer({
15 | allowAutoTopicCreation: false,
16 | });
17 |
18 | const getRandomNumber = () => Math.round(Math.random(10) * 1000)
19 | const createMessage = num => ({
20 | key: `key-${num}`,
21 | value: `value-${num}-${new Date().toISOString()}`,
22 | })
23 |
24 | const sendMessage = () => {
25 | return producer
26 | .send({
27 | topic,
28 | compression: CompressionTypes.GZIP,
29 | messages: Array(getRandomNumber())
30 | .fill()
31 | .map(_ => createMessage(getRandomNumber())),
32 | })
33 | .then(console.log)
34 | .catch(e => console.error(`[example/producer] ${e.message}`, e))
35 | }
36 |
37 | const run = async () => {
38 | await producer.connect()
39 | setInterval(sendMessage, 3000)
40 | }
41 |
42 | run().catch(e => console.error(`[example/producer] ${e.message}`, e))
43 |
44 | const errorTypes = ['unhandledRejection', 'uncaughtException']
45 | const signalTraps = ['SIGTERM', 'SIGINT', 'SIGUSR2']
46 |
47 | errorTypes.forEach(type => {
48 | process.on(type, async () => {
49 | try {
50 | console.log(`process.on ${type}`)
51 | await producer.disconnect()
52 | process.exit(0)
53 | } catch (_) {
54 | process.exit(1)
55 | }
56 | })
57 | })
58 |
59 | signalTraps.forEach(type => {
60 | process.once(type, async () => {
61 | try {
62 | await producer.disconnect()
63 | } finally {
64 | process.kill(process.pid, type)
65 | }
66 | })
67 | })
--------------------------------------------------------------------------------
/server/controllers/oauthController.js:
--------------------------------------------------------------------------------
1 | const express = require('express');
2 | const dotenv = require('dotenv');
3 | dotenv.config();
4 | const axios = require('axios');
5 |
6 | const client_id = process.env.GITHUB_CLIENT_ID;
7 | const client_secret = process.env.GITHUB_CLIENT_SECRET;
8 |
9 | module.exports = {
10 |
11 | async githubData(req, res, next) {
12 | try {
13 | const code = req.query.code;
14 | const token = await getAccessToken(code);
15 | const githubData = await getGithubUser(token);
16 |
17 | res.locals.github = githubData
18 | res.locals.token = token
19 |
20 | return next()
21 | } catch (error) {
22 | return next({ error, message: 'something went wrong getting GitHub data', log: 'middleware error in the githubData function' });
23 | }
24 | }
25 | }
26 |
27 | // -------- helper functions --------------------------------------------------------
28 | async function getAccessToken(code) {
29 | const res = await axios
30 | .post('https://github.com/login/oauth/access_token', {
31 | client_id,
32 | client_secret,
33 | code,
34 | scope: ['user:email'],
35 | })
36 | .then(function (response) {
37 | return response.data;
38 | })
39 | .catch(function (error) {
40 | console.log(error);
41 | });
42 |
43 | const params = new URLSearchParams(res);
44 |
45 | return params.get('access_token');
46 | }
47 |
48 | async function getGithubUser(access_token) {
49 | const res = await axios.get('https://api.github.com/user', {
50 | headers: {
51 | Authorization: `bearer ${access_token}`,
52 | },
53 | });
54 | return res.data;
55 | }
--------------------------------------------------------------------------------
/server/controllers/slackController.js:
--------------------------------------------------------------------------------
1 | const express = require('express');
2 | const axios = require('axios');
3 | const metricsObject = require('./utils.js')
4 |
5 |
6 | module.exports = {
7 |
8 | async initialNote(req, res, next) {
9 | try {
10 | const time = new Date().toUTCString().slice(5, -4)
11 | axios.post('https://hooks.slack.com/services/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', {
12 | "blocks": [
13 | {
14 | "type": "section",
15 | "text": {
16 | "type": "mrkdwn",
17 | "text": " :wave: *You have a new alert:*\n\n You will begin to receive notifictions for the following:"
18 | }
19 | },
20 | {
21 | "type": "section",
22 | "fields": [
23 | {
24 | "type": "mrkdwn",
25 | "text": `*Metric:*\n ${req.body.label}`
26 | },
27 | {
28 | "type": "mrkdwn",
29 | "text": `*Subscribed :*\nSubmitted ${time}`
30 | },
31 | {
32 | "type": "mrkdwn",
33 | "text": `*Threshold:*\n > ${req.body.threshold}`
34 | },
35 | {
36 | "type": "mrkdwn",
37 | "text": `*Assigned By:*\n - ${req.body.name}`
38 | }
39 | ]
40 | }
41 | ]
42 | })
43 | .then(() => {
44 | console.log('Form submitted successfully!');
45 | })
46 | .catch((error) => {
47 | console.log({error:error,message:'error inside axios'})
48 | });
49 | return next()
50 | } catch (error) {
51 | return next({ error, message: 'something went wrong sending slack message', log: 'middleware error in the initialNote function' });
52 | }
53 | },
54 |
55 | async checkingMetric(req, res, next) {
56 | metricsObject[req.body.value] = req.body.threshold
57 | return next()
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/server/controllers/utils.js:
--------------------------------------------------------------------------------
1 | // initializing notification object used as a reference
2 | const metricsObject = {
3 | bytesInPerSec: null,
4 | bytesOutPerSec: null,
5 | messagesInPerSec: null,
6 | offlinePartitions: null,
7 | underRepPartitions: null
8 | }
9 |
10 | module.exports = metricsObject;
11 |
12 | // health metrics references ------------------------------------------------------------------
13 | // bytesInPerSec: ['kafka_server_broker_topic_metrics_bytesinpersec_rate','']
14 | // bytesOutPerSec: ['kafka_server_broker_topic_metrics_bytesoutpersec_rate','']
15 | // messagesInPerSec: ['kafka_server_broker_topic_metrics_messagesinpersec_rate','']
16 | // jvmHeapUsage: ['kafka_jvm_heap_usage{env="cluster-demo", type="used"}','']
17 | // activeControllerCount: ['sum(kafka_controller_activecontrollercount)','']
18 | // underRepPartitions: ['kafka_server_replica_manager_underreplicatedpartitions','']
19 | // offlinePartitions: ['kafka_controller_offlinepartitionscount','']
20 | // brokersRunning: ['count(kafka_server_brokerstate)','']
21 |
22 | // performance metrics references ------------------------------------------------------------------
23 | // requestsPerSec: ["kafka_network_request_per_sec{aggregate=~"OneMinuteRate",request="Produce"}",""]
24 | // requestTotalTime :["kafka_network_request_metrics_time_ms{instance='jmx-kafka:5556', request='FetchConsumer',scope='Total',env='cluster-demo'}","[10m:10s]"]
25 | // responseQueueTime : ["kafka_network_request_metrics_time_ms{instance='jmx-kafka:5556', request='FetchConsumer',scope='ResponseQueue',env='cluster-demo', aggregate='99thPercentile'}","[10m:10s]"]
26 | // responseSendTime : ["kafka_network_request_metrics_time_ms{instance='jmx-kafka:5556', request='FetchConsumer',scope='ResponseSend',env='cluster-demo', aggregate='Mean'}","[10m:10s]"]
27 | // processorIdlePercent : ["kafka_network_processor_idle_percent","[10m:10s]"]
--------------------------------------------------------------------------------
/server/queries.js:
--------------------------------------------------------------------------------
1 | const axios = require('axios')
2 | const metricsObject = require('./controllers/utils.js')
3 |
4 | const slackPostFunc = (label, currentThreshold, currentValue) => {
5 | axios.post('https://hooks.slack.com/services/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', {
6 | "blocks": [
7 | {
8 | "type": "section",
9 | "text": {
10 | "type": "plain_text",
11 | "text": `:Warning: ${label} has gone over the threshold: ${currentThreshold}, please check your kafka cluster for more details.\nCurrent value: ${currentValue}`,
12 | "emoji": true
13 | }
14 | }
15 | ]
16 | })
17 | .then(() => {
18 | console.log('Alert Sent Succesfully!')
19 | })
20 | .catch((error) => {
21 | console.log(error)
22 | });
23 | return
24 | };
25 |
26 | const timeConvert = (arr, label) => {
27 | const newArr = []
28 | arr.forEach(data => {
29 | if (metricsObject[label]) {
30 | const thresholdNumber = Number(metricsObject[label])
31 | if (data[1] > thresholdNumber) {
32 | slackPostFunc(label, metricsObject[label], parseInt(data[1]))
33 | metricsObject[label] = null
34 | }
35 | }
36 |
37 | newArr.push([((data[0] - 14400) * 1000), data[1]]);
38 | });
39 | return newArr
40 | }
41 |
42 | const jvmConvert = (arr) => {
43 | const newArr = []
44 | arr.forEach(data => {
45 | newArr.push([((data[0] - 14400) * 1000), parseInt(data[1]) / 1000000]);
46 | });
47 | return newArr
48 | }
49 | const reqTTConvert = (arr) => {
50 | const newArr = [];
51 | arr.forEach(data => {
52 | newArr.push([((data[0] - 14400) * 1000), parseInt(data[1]) / 1000])
53 | })
54 | return newArr;
55 | }
56 |
57 | const multiGraphConvert = (arr) => {
58 | const newArr = [];
59 |
60 | arr.forEach(mainObj => {
61 | console.log('Metric Name', mainObj.metric.__name__)
62 | if (mainObj.metric.__name__ === 'kafka_network_request_metrics_time_ms') {
63 | if (mainObj.values) {
64 | newArr.push(reqTTConvert(mainObj.values));
65 | }
66 | if (mainObj.value) {
67 | newArr.push(reqTTConvert([mainObj.value]));
68 | }
69 | } else {
70 | if (mainObj.values) {
71 | newArr.push(timeConvert(mainObj.values));
72 | }
73 | if (mainObj.value) {
74 | newArr.push(timeConvert([mainObj.value]));
75 | }
76 | }
77 |
78 | })
79 | return newArr;
80 | }
81 |
82 | let counter = 0;
83 |
84 | const fetchQuery = async (query, timeFrame, label) => {
85 | // sending a fetch request to prometheus using axios
86 | if (counter < 4) {
87 | console.log(`sending PAST 10m of cluster query on params: ${query}, ${timeFrame}, ${label}`)
88 | try {
89 | const data = await axios.get(`http://localhost:9090/api/v1/query?query=${query}${timeFrame}`)
90 | counter++
91 | switch (query) {
92 | case ('kafka_jvm_heap_usage{env="cluster-demo", type="used"}'):
93 | let jvmPre = data.data.data.result[0].values
94 | return jvmConvert(jvmPre);
95 | case ('kafka_server_replica_manager_underreplicatedpartitions'):
96 | return data.data.data.result[0].value[1];
97 | case ('kafka_controller_offlinepartitionscount'):
98 | return data.data.data.result[0].value[1];
99 | case ('sum(kafka_controller_activecontrollercount)'):
100 | return data.data.data.result[0].value[1];
101 | case ('count(kafka_server_brokerstate)'):
102 | return data.data.data.result[0].value[1];
103 | case ('kafka_network_request_per_sec{aggregate=~"OneMinuteRate",request="Produce"}'):
104 | return data.data.data.result[0].value[1];
105 | case (`kafka_network_request_metrics_time_ms{instance='jmx-kafka:5556', request='FetchConsumer',scope='Total',env='cluster-demo'}`):
106 | let convertedVal = await multiGraphConvert(data.data.data.result)
107 | console.log('logging convertedVal 10min ->', convertedVal);
108 | return convertedVal;
109 | default:
110 | let preConvert = data.data.data.result[0].values
111 | let output = timeConvert(preConvert, label)
112 | return output
113 | }
114 | } catch (err) {
115 | console.log(`Error in ${query}, err: ${err}`)
116 | }
117 | } else {
118 | try {
119 | console.log(`sending CURRENT DATA of cluster query on params: ${query}, ${label}`)
120 | const data = await axios.get(`http://localhost:9090/api/v1/query?query=${query}`)
121 | switch (query) {
122 | case ('kafka_jvm_heap_usage{env="cluster-demo", type="used"}'):
123 | let jvmPre = [data.data.data.result[0].value]
124 | return jvmConvert(jvmPre);
125 | case ('kafka_server_replica_manager_underreplicatedpartitions'):
126 | return data.data.data.result[0].value[1];
127 | case ('kafka_controller_offlinepartitionscount'):
128 | return data.data.data.result[0].value[1];
129 | case ('sum(kafka_controller_activecontrollercount)'):
130 | return data.data.data.result[0].value[1];
131 | case ('count(kafka_server_brokerstate)'):
132 | return data.data.data.result[0].value[1];
133 | case ('kafka_network_request_per_sec{aggregate=~"OneMinuteRate",request="Produce"}'):
134 | return data.data.data.result[0].value[1];
135 | case ('kafka_network_processor_idle_percent'):
136 | case (`kafka_network_request_metrics_time_ms{instance='jmx-kafka:5556', request='FetchConsumer',scope='Total',env='cluster-demo'}`):
137 | let convertedVal = await multiGraphConvert(data.data.data.result)
138 | return convertedVal;
139 | default:
140 | let preConvert = [data.data.data.result[0].value]
141 | let output = timeConvert(preConvert, label)
142 | return output
143 | }
144 | } catch (err) {
145 | console.log(`Error in ${query}, err: ${err}`)
146 | }
147 | }
148 | }
149 |
150 | const resetCounter = () => {
151 | counter = 0;
152 | }
153 |
154 | module.exports = { fetchQuery, resetCounter }
--------------------------------------------------------------------------------
/server/router/routes.js:
--------------------------------------------------------------------------------
1 | const express = require('express');
2 | const router = express.Router();
3 | const dotenv = require('dotenv');
4 | const axios = require('axios');
5 | const cookieSession = require('cookie-session');
6 | const oauthController = require('../controllers/oauthController.js')
7 | const slackController = require('../controllers/slackController.js')
8 | dotenv.config();
9 |
10 | router.use(
11 | cookieSession({
12 | secret: 'mainSecret',
13 | })
14 | );
15 |
16 | // variables
17 | const client_id = process.env.GITHUB_CLIENT_ID;
18 | const client_secret = process.env.GITHUB_CLIENT_SECRET;
19 | const callback_url = 'http://localhost:4000/auth/github/callback';
20 |
21 | // oauth
22 | router.get('/github', (req, res, next) => {
23 | const url = `https://github.com/login/oauth/authorize?client_id=${client_id}&redirect_uri=${callback_url}`;
24 | return res.redirect(url);
25 | });
26 |
27 | router.get('/github/callback',
28 | oauthController.githubData,
29 | async (req, res) => {
30 | req.session.name = res.locals.github.name;
31 | req.session.id = res.locals.github.id;
32 | req.session.login = res.locals.github.login;
33 | req.session.email = res.locals.github.email;
34 | req.session.token = res.locals.token
35 | return res.redirect(`http://localhost:8080/?token=${req.session.token}`)
36 | });
37 |
38 | router.get('/data', (req, res) => {
39 | return res.status(200).send({
40 | name: req.session.name,
41 | id: req.session.id,
42 | login: req.session.login,
43 | email: req.session.email
44 | })
45 | })
46 |
47 | router.get('/logout', (req, res, next) => {
48 | req.session = null;
49 | return res.redirect('http://localhost:8080');
50 | });
51 |
52 | // slack notifications
53 | router.post('/form-submit',
54 | slackController.initialNote,
55 | slackController.checkingMetric,
56 | (req, res, next) => {
57 | return res.status(200).send(`KafkaPeek now tracking metric ${req.body.label}`)
58 | })
59 |
60 | module.exports = router;
61 |
--------------------------------------------------------------------------------
/server/server.js:
--------------------------------------------------------------------------------
1 | const express = require('express');
2 | const app = express();
3 | const path = require('path');
4 | const PORT = 4000;
5 | const authRouter = require('./router/routes.js');
6 |
7 | // socket.io
8 | const { createServer } = require('http');
9 | const { Server } = require('socket.io');
10 | const { response } = require('express');
11 | const cors = require('cors');
12 | const ioConfig = {
13 | cors: {
14 | origin: ['http://localhost:8080'],
15 | },
16 | };
17 |
18 | const httpServer = createServer(app);
19 | const io = new Server(httpServer, ioConfig);
20 | const { fetchQuery, resetCounter } = require('./queries');
21 | const { default: axios } = require('axios');
22 |
23 | // parsing data coming from frontend
24 | app.use(cors());
25 | app.use(express.json());
26 | app.use(express.urlencoded({ extended: true }));
27 | app.use(express.static(path.join(__dirname, '../client/assets')));
28 |
29 | // serving dynamic files
30 | app.get('/', (req, res, next) => {
31 | res.status(200).sendFile(path.join(__dirname, '/../client/index.html'));
32 | });
33 |
34 | // serving bundle to link in html file
35 | app.get('/dist/bundle.js', (req, res, next) => {
36 | res.status(200).sendFile(path.join(__dirname, '../dist/bundle.js'));
37 | });
38 |
39 |
40 | // initializing socket.io connection
41 | var fetchIntervalID;
42 | io.on('connection', (socket) => {
43 | console.log('a new websocket connection');
44 | socket.on('health', (args) => {
45 | fetchIntervalID = setInterval(async () => {
46 | const fetchObj = {};
47 | for (const [k, v] of Object.entries(args)) {
48 | fetchObj[k] = await fetchQuery(v[0], v[1], k);
49 | }
50 |
51 | socket.emit('health', fetchObj);
52 | }, 1000);
53 | console.log('Sending new metrics!');
54 | });
55 |
56 | socket.on('performance', (args) => {
57 | fetchIntervalID = setInterval(async () => {
58 | const fetchObj = {};
59 | for (const [k, v] of Object.entries(args)) {
60 | fetchObj[k] = await fetchQuery(v[0], v[1], k);
61 | }
62 |
63 | socket.emit('performance', fetchObj);
64 | }, 1000);
65 | });
66 |
67 | socket.on('stop', () => {
68 | clearInterval(fetchIntervalID);
69 | console.log('Metrics stopped!');
70 | });
71 |
72 | socket.on('disconnect', () => {
73 | clearInterval(fetchIntervalID);
74 | resetCounter();
75 | console.log('websocket to client was disconnected!');
76 | });
77 | });
78 |
79 | // routes
80 | app.use('/auth', authRouter);
81 |
82 | // catch all handler for unknown routes
83 | app.use((req, res) => {
84 | res.status(404).send('404');
85 | });
86 |
87 | // global error handler
88 | app.use((err, req, res, next) => {
89 | const defaultErr = {
90 | log: 'Express error handler caught unknown middleware error',
91 | status: 400,
92 | message: { err: 'An error occurred' },
93 | };
94 | const errorObj = Object.assign({}, defaultErr, err);
95 | return res.status(errorObj.status).json(errorObj.message);
96 | });
97 |
98 | httpServer.listen(PORT, () =>
99 | console.log(`Server listening on port: ${PORT}...`)
100 | );
101 |
102 | module.exports = { app, httpServer };
103 |
--------------------------------------------------------------------------------
/test-teardown-globals.js:
--------------------------------------------------------------------------------
1 | module.exports = () => {
2 | process.exit(0)
3 | };
--------------------------------------------------------------------------------
/webpack.config.js:
--------------------------------------------------------------------------------
1 | // const webpack = require("webpack");
2 | const path = require('path');
3 | const HtmlWebpackPlugin = require('html-webpack-plugin');
4 |
5 | const config = {
6 | mode: process.env.NODE_ENV,
7 | entry: './client/index.js',
8 | output: {
9 | path: path.resolve(__dirname, 'dist'),
10 | filename: 'bundle.js',
11 | publicPath: '/',
12 | },
13 | devServer: {
14 | host: 'localhost',
15 | port: 8080,
16 | static: {
17 | directory: path.join(__dirname, './build'),
18 | publicPath: '/',
19 | },
20 | hot: true,
21 | headers: { 'Access-Control-Allow-Origin': '*' },
22 | // port: 3000,
23 | proxy: {
24 | '/api': 'http://localhost:4000',
25 | },
26 | },
27 | module: {
28 | rules: [
29 | {
30 | test: /\.(js|jsx)$/,
31 | use: {
32 | loader: 'babel-loader',
33 | options: {
34 | presets: ['@babel/preset-env', '@babel/preset-react'],
35 | },
36 | },
37 | exclude: /node_modules/,
38 | },
39 | {
40 | test: /\.css$/,
41 | use: ['style-loader', 'css-loader'],
42 | },
43 | {
44 | test: /\.scss$/,
45 | use: ['style-loader', 'css-loader', 'sass-loader'],
46 | },
47 | {
48 | test: /\.svg$/,
49 | loader: 'svg-inline-loader',
50 | },
51 | {
52 | test: /\.png$/,
53 | use: [
54 | {
55 | loader: 'url-loader',
56 | options: {
57 | mimetype: 'image/png',
58 | },
59 | },
60 | ],
61 | },
62 | ],
63 | },
64 | resolve: { extensions: ['*', '.js', '.jsx'] },
65 | plugins: [new HtmlWebpackPlugin({ template: './client/index.html' })],
66 | };
67 |
68 | module.exports = config;
69 |
--------------------------------------------------------------------------------