└── README.adoc /README.adoc: -------------------------------------------------------------------------------- 1 | = Kafka Cheat Sheet 2 | :toc: 3 | :toc-placement!: 4 | :icons: 5 | 6 | Here you have some useful commands for kafka. 7 | 8 | Tested on Kafka 2.5 9 | 10 | Goals 11 | 12 | - [x] Add most useful commands for kafka 13 | - [x] Add kafkacat commands 14 | - [ ] Add commands output 15 | - [x] Add commands for Kafka on Kubernetes 16 | 17 | toc::[] 18 | 19 | == Pre-req 20 | 21 | First, set some kafka environment vars. 22 | 23 | [source,bash] 24 | ---- 25 | # For Kafka running on top of VMs/Bare Metal 26 | KAFKA_BIN=/opt/kafka/bin 27 | ZOOKEEPER_HOST=zookeeper-host:2181 28 | BROKER_HOST=broker-host:9092 29 | 30 | # For Kafka running on top of Kubernetes (Using strimzi) 31 | KAFKA_NAMESPACE=kafka-demo 32 | ZOOKEEPER_HOST=localhost:2181 33 | BROKER_HOST=localhost:9092 34 | ZOOKEEPER_POD=$(kubectl -n $KAFKA_NAMESPACE get pods -l app.kubernetes.io/name=zookeeper -o=jsonpath='{.items[0].metadata.name}') 35 | KAFKA_BROKER_POD=$(kubectl -n $KAFKA_NAMESPACE get pods -l app.kubernetes.io/name=kafka -o=jsonpath='{.items[0].metadata.name}') 36 | ---- 37 | 38 | == Zookeeper Operations 39 | 40 | You need to whitelist all the commands bellow. 41 | 42 | .zookeeper.properties 43 | ---- 44 | 4lw.commands.whitelist=stat,ruok,reqs,envi,dump,conf,cons,srvr,wchs,wchc,dirs,wchp,mntr,isro 45 | ---- 46 | 47 | * If using Zookeeper Auth (SASL) 48 | 49 | [source,bash] 50 | ---- 51 | # Zookeeper Auth 52 | export KAFKA_OPTS="-Djava.security.auth.login.config=/tmp/jaas.conf" 53 | ---- 54 | 55 | .jass.conf 56 | ---- 57 | Client { 58 | org.apache.zookeeper.server.auth.DigestLoginModule required 59 | username="test" 60 | password="test"; 61 | }; 62 | ---- 63 | 64 | * If using SSL/TLS on Zookeeper + SASL 65 | 66 | [source,bash] 67 | ---- 68 | export KAFKA_OPTS="-Djava.security.auth.login.config=/tmp/jaas.conf -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty -Dzookeeper.client.secure=true -Dzookeeper.ssl.trustStore.location=/tmp/kafka.server.truststore -Dzookeeper.ssl.trustStore.password=mypass -Dzookeeper.ssl.trustStore.type=PKCS12" 69 | ---- 70 | 71 | NOTE: Remember to change your zookeeper port on the `ZOOKEEPER_HOST` if necessary 72 | 73 | === Get runtime conf 74 | 75 | [source,bash] 76 | ---- 77 | # For VMs 78 | echo conf | curl telnet://$ZOOKEEPER_HOST 79 | 80 | # For Kubernetes 81 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo conf | curl telnet://localhost:2181" 82 | ---- 83 | 84 | === Get runtime environments 85 | 86 | [source,bash] 87 | ---- 88 | # For VMs 89 | echo envi | curl telnet://$ZOOKEEPER_HOST 90 | 91 | # For Kubernetes 92 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo envi | curl telnet://localhost:2181" 93 | ---- 94 | 95 | === Health Check 96 | 97 | [source,bash] 98 | ---- 99 | # For VMs 100 | echo stats | curl telnet://$ZOOKEEPER_HOST 101 | echo ruok | curl telnet://$ZOOKEEPER_HOST 102 | 103 | # For Kubernetes 104 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo stats | curl telnet://localhost:2181" 105 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo ruok | curl telnet://localhost:2181" 106 | ---- 107 | 108 | === Connections 109 | 110 | [source,bash] 111 | ---- 112 | # For VMs 113 | echo reqs | curl telnet://$ZOOKEEPER_HOST 114 | echo cons | curl telnet://$ZOOKEEPER_HOST 115 | 116 | # For Kubernetes 117 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo reqs | curl telnet://localhost:2181" 118 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo cons | curl telnet://localhost:2181" 119 | ---- 120 | 121 | === Details of the server 122 | 123 | [source,bash] 124 | ---- 125 | # For VMs 126 | echo srvr | curl telnet://$ZOOKEEPER_HOST 127 | 128 | # For Kubernetes 129 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo srvr | curl telnet://localhost:2181" 130 | ---- 131 | 132 | === Brief info about watches 133 | 134 | [source,bash] 135 | ---- 136 | # For VMs 137 | echo wchs | curl telnet://$ZOOKEEPER_HOST 138 | 139 | # For Kubernetes 140 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo wchs | curl telnet://localhost:2181" 141 | ---- 142 | 143 | === Details about watches 144 | 145 | [source,bash] 146 | ---- 147 | # For VMs 148 | echo wchc | curl telnet://$ZOOKEEPER_HOST 149 | 150 | # For Kubernetes 151 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo wchs | curl telnet://localhost:2181" 152 | ---- 153 | 154 | === Snapshots info 155 | 156 | [source,bash] 157 | ---- 158 | # For VMs 159 | echo dirs | curl telnet://$ZOOKEEPER_HOST 160 | 161 | # For Kubernetes 162 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo dirs | curl telnet://localhost:2181" 163 | ---- 164 | 165 | === Monitoring vars 166 | 167 | [source,bash] 168 | ---- 169 | # For VMs 170 | echo mntr | curl telnet://$ZOOKEEPER_HOST 171 | 172 | # For Kubernetes 173 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo mntr | curl telnet://localhost:2181" 174 | ---- 175 | 176 | === Get read-only or read-write mode 177 | 178 | [source,bash] 179 | ---- 180 | # For VMs 181 | echo isro | curl telnet://$ZOOKEEPER_HOST 182 | 183 | # For Kubernetes 184 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bash -c "echo isro | curl telnet://localhost:2181" 185 | ---- 186 | 187 | === Get Process 188 | 189 | [source,bash] 190 | ---- 191 | # For VMs 192 | jps | grep QuorumPeerMain 193 | 194 | # For kubernetes 195 | kubectl -n $KAFKA_NAMESPACE exec -it $ZOOKEEPER_POD -- bash -c "ps aux | grep QuorumPeerMain" 196 | ---- 197 | 198 | == Broker Operations 199 | 200 | === List active brokers 201 | 202 | [source,bash] 203 | ---- 204 | # For VMs 205 | $KAFKA_BIN/zookeeper-shell.sh $ZOOKEEPER_HOST ls /brokers/ids 206 | 207 | # For Kubernetes 208 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/zookeeper-shell.sh localhost:2181 ls /brokers/ids 209 | ---- 210 | 211 | [source,bash] 212 | ---- 213 | kafkacat -b $BROKER_HOST -L 214 | ---- 215 | 216 | === List Broker Controller 217 | 218 | [source,bash] 219 | ---- 220 | # For VMs 221 | $KAFKA_BIN/zookeeper-shell.sh $ZOOKEEPER_HOST get /controller 222 | 223 | # For Kubernetes 224 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/zookeeper-shell.sh localhost:2181 get /controller 225 | ---- 226 | 227 | === List broker details 228 | 229 | [source,bash] 230 | ---- 231 | # For VMs 232 | $KAFKA_BIN/zookeeper-shell.sh $ZOOKEEPER_HOST ls /brokers/ids/{id} 233 | 234 | # For Kubernetes 235 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/zookeeper-shell.sh localhost:2181 ls /brokers/ids/{id} 236 | ---- 237 | 238 | [source,bash] 239 | ---- 240 | kafkacat -b $BROKER_HOST -L 241 | ---- 242 | 243 | === List topics 244 | 245 | [source,bash] 246 | ---- 247 | # For VMs 248 | $KAFKA_BIN/zookeeper-shell.sh $ZOOKEEPER_HOST ls /brokers/topics 249 | 250 | # For Kubernetes 251 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/zookeeper-shell.sh localhost:2181 ls /brokers/topics 252 | ---- 253 | 254 | [source,bash] 255 | ---- 256 | kafkacat -b $BROKER_HOST -L -t 257 | ---- 258 | 259 | === Change Broker Config 260 | 261 | Change log cleaner threads. 262 | 263 | [source,bash] 264 | ---- 265 | # For VMs 266 | $KAFKA_BIN/kafka-configs.sh \ 267 | --bootstrap-server $BROKER_HOST \ 268 | --entity-type brokers \ 269 | --entity-name \ 270 | --alter \ 271 | --add-config log.cleaner.threads=2 272 | 273 | # For Kubernetes 274 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 275 | --bootstrap-server $BROKER_HOST \ 276 | --entity-type brokers \ 277 | --entity-name \ 278 | --alter \ 279 | --add-config log.cleaner.threads=2 280 | ---- 281 | 282 | === Describe broker dynamic config 283 | 284 | [source,bash] 285 | ---- 286 | # For VMs 287 | $KAFKA_BIN/kafka-configs.sh \ 288 | --bootstrap-server $BROKER_HOST \ 289 | --entity-type brokers \ 290 | --entity-name \ 291 | --describe 292 | 293 | # For Kubernetes 294 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 295 | --bootstrap-server $BROKER_HOST \ 296 | --entity-type brokers \ 297 | --entity-name \ 298 | --describe 299 | ---- 300 | 301 | === Delete broker config 302 | 303 | [source,bash] 304 | ---- 305 | # For VMs 306 | $KAFKA_BIN/kafka-configs.sh \ 307 | --bootstrap-server $BROKER_HOST \ 308 | --entity-type brokers \ 309 | --entity-name \ 310 | --alter \ 311 | --delete-config log.cleaner.threads 312 | 313 | # For Kubernetes 314 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 315 | --bootstrap-server $BROKER_HOST \ 316 | --entity-type brokers \ 317 | --entity-name \ 318 | --alter \ 319 | --delete-config log.cleaner.threads 320 | ---- 321 | 322 | === Change cluster-wide dynamic config 323 | 324 | [source,bash] 325 | ---- 326 | # For VMs 327 | $KAFKA_BIN/kafka-configs.sh \ 328 | --bootstrap-server $BROKER_HOST \ 329 | --entity-type brokers \ 330 | --entity-default \ 331 | --alter \ 332 | --add-config log.cleaner.threads=2 333 | 334 | # For Kubernetes 335 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 336 | --bootstrap-server $BROKER_HOST \ 337 | --entity-type brokers \ 338 | --entity-default \ 339 | --alter \ 340 | --add-config log.cleaner.threads=2 341 | ---- 342 | 343 | === Describe cluster-wide dynamic config 344 | 345 | [source,bash] 346 | ---- 347 | # For VMs 348 | $KAFKA_BIN/kafka-configs.sh \ 349 | --bootstrap-server $BROKER_HOST \ 350 | --entity-type brokers \ 351 | --entity-default \ 352 | --describe 353 | 354 | # For Kubernetes 355 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 356 | --bootstrap-server $BROKER_HOST \ 357 | --entity-type brokers \ 358 | --entity-default \ 359 | --describe 360 | ---- 361 | 362 | === Disable hostname verification 363 | 364 | [source,bash] 365 | ---- 366 | # For VMs 367 | $KAFKA_BIN/kafka-configs.sh \ 368 | --bootstrap-server $BROKER_HOST \ 369 | --entity-type brokers \ 370 | --entity-name \ 371 | --alter \ 372 | --add-config "listener.name.internal.ssl.endpoint.identification.algorithm=" 373 | 374 | # For Kubernetes 375 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 376 | --bootstrap-server $BROKER_HOST \ 377 | --entity-type brokers \ 378 | --entity-name \ 379 | --alter \ 380 | --add-config "listener.name.internal.ssl.endpoint.identification.algorithm=" 381 | ---- 382 | 383 | == Topic Operations 384 | 385 | === List topics using kafka-topics.sh 386 | 387 | [source,bash] 388 | ---- 389 | # For VMs 390 | $KAFKA_BIN/kafka-topics.sh \ 391 | --list \ 392 | --zookeeper $ZOOKEEPER_HOST 393 | 394 | # For Kubernetes 395 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 396 | --list \ 397 | --zookeeper $ZOOKEEPER_HOST 398 | ---- 399 | 400 | [source,bash] 401 | ---- 402 | # For VMs 403 | $KAFKA_BIN/kafka-topics.sh \ 404 | --bootstrap-server $BROKER_HOST \ 405 | --list 406 | 407 | # For Kubernetes 408 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 409 | --list \ 410 | --bootstrap-server $BROKER_HOST 411 | ---- 412 | 413 | === Describe topic 414 | 415 | [source,bash] 416 | ---- 417 | # For VMs 418 | $KAFKA_BIN/kafka-topics.sh \ 419 | --zookeeper $ZOOKEEPER_HOST \ 420 | --topic \ 421 | --describe 422 | 423 | # For Kubernetes 424 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 425 | --zookeeper $ZOOKEEPER_HOST \ 426 | --topic \ 427 | --describe 428 | ---- 429 | 430 | [source,bash] 431 | ---- 432 | kafkacat -b $BROKER_HOST -L -t 433 | ---- 434 | 435 | === Describe topic configs 436 | 437 | [source,bash] 438 | ---- 439 | # For VMs 440 | $KAFKA_BIN/kafka-configs.sh \ 441 | --zookeeper $ZOOKEEPER_HOST \ 442 | --entity-type topics \ 443 | --entity-name \ 444 | --describe 445 | 446 | # For Kubernetes 447 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 448 | --zookeeper $ZOOKEEPER_HOST \ 449 | --entity-type topics \ 450 | --entity-name \ 451 | --describe 452 | ---- 453 | 454 | === Delete topic config 455 | 456 | [source,bash] 457 | ---- 458 | # For VMs 459 | $KAFKA_BIN/kafka-configs.sh \ 460 | --zookeeper $ZOOKEEPER_HOST \ 461 | --entity-type topics \ 462 | --entity-name \ 463 | --delete-config 464 | 465 | # For Kubernetes 466 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 467 | --zookeeper $ZOOKEEPER_HOST \ 468 | --entity-type topics \ 469 | --entity-name \ 470 | --delete-config 471 | ---- 472 | 473 | === Copy topic to another topic in the same cluster 474 | 475 | [source,bash] 476 | ---- 477 | kafkacat -C -b $BROKER_HOST -t -e | kafkacat -P -b $BROKER_HOST -t 478 | ---- 479 | 480 | === Copy topic to another topic in another cluster 481 | 482 | [source,bash] 483 | ---- 484 | kafkacat -C -b $BROKER_HOST -t -e | kafkacat -P -b $BROKER_HOST2 -t 485 | ---- 486 | 487 | === Move topic to another broker 488 | 489 | ==== Create json necessary 490 | 491 | .topics-to-move.json 492 | [source,json] 493 | ---- 494 | {"topics": [{"topic": "topic1"}, 495 | {"topic": "topic2"}], 496 | "version":1 497 | } 498 | ---- 499 | 500 | ==== Generate plan to move to brokers 501 | 502 | .generate plan to move to broker 5 and 6 503 | [source,bash] 504 | ---- 505 | # For VMs 506 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 507 | --zookeeper $ZOOKEEPER_HOST \ 508 | --topics-to-move-json-file topics-to-move.json \ 509 | --broker-list "5,6" \ 510 | --generate 511 | 512 | # For Kubernetes 513 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 514 | --zookeeper $ZOOKEEPER_HOST \ 515 | --entity-type topics \ 516 | --entity-name \ 517 | --delete-config 518 | ---- 519 | 520 | NOTE: save the results from the command above to `cluster-reassignment.json` 521 | 522 | ==== Move to broker 5 and 6 523 | 524 | .move to broker 5 and 6 525 | [source,bash] 526 | ---- 527 | # For VMs 528 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 529 | --zookeeper $ZOOKEEPER_HOST \ 530 | --reassignment-json-file cluster-reassignment.json \ 531 | --execute 532 | 533 | # For Kubernetes 534 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 535 | --zookeeper $ZOOKEEPER_HOST \ 536 | --reassignment-json-file cluster-reassignment.json \ 537 | --execute 538 | ---- 539 | 540 | ==== Verify status 541 | 542 | .verify status 543 | [source,bash] 544 | ---- 545 | # For VMs 546 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 547 | --zookeeper $ZOOKEEPER_HOST \ 548 | --reassignment-json-file cluster-reassignment.json \ 549 | --verify 550 | 551 | # For Kubernetes 552 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 553 | --zookeeper $ZOOKEEPER_HOST \ 554 | --reassignment-json-file cluster-reassignment.json \ 555 | --verify 556 | ---- 557 | 558 | === Create topic 559 | 560 | [source,bash] 561 | ---- 562 | # For VMs 563 | $KAFKA_BIN/kafka-topics.sh \ 564 | --create \ 565 | --zookeeper $ZOOKEEPER_HOST \ 566 | --replication-factor 1 \ 567 | --partitions 1 \ 568 | --topic 569 | 570 | # For Kubernetes 571 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 572 | --create \ 573 | --zookeeper $ZOOKEEPER_HOST \ 574 | --replication-factor 1 \ 575 | --partitions 1 \ 576 | --topic 577 | ---- 578 | 579 | === Create topic with config 580 | 581 | [source,bash] 582 | ---- 583 | # For VMs 584 | $KAFKA_BIN/kafka-topics.sh \ 585 | --bootstrap-server $BROKER_HOST \ 586 | --create \ 587 | --topic \ 588 | --partitions 1 \ 589 | --replication-factor 1 \ 590 | --config max.message.bytes=64000 \ 591 | --config flush.messages=1 592 | 593 | # For Kubernetes 594 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 595 | --bootstrap-server $BROKER_HOST \ 596 | --create \ 597 | --topic \ 598 | --partitions 1 \ 599 | --replication-factor 1 \ 600 | --config max.message.bytes=64000 \ 601 | --config flush.messages=1 602 | ---- 603 | 604 | === Increase replication factor of __consumer_offsets 605 | 606 | ==== Create replication plan 607 | 608 | .reassignment.json 609 | [source,json] 610 | ---- 611 | {"version":1, 612 | "partitions":[ 613 | {"topic":"__consumer_offsets", "partition":0, "replicas":[106,101,102,105]}, 614 | {"topic":"__consumer_offsets", "partition":1, "replicas":[106,101,102,105]}, 615 | {"topic":"__consumer_offsets", "partition":2, "replicas":[106,101,102,105]}, 616 | {"topic":"__consumer_offsets", "partition":3, "replicas":[106,101,102,105]}, 617 | {"topic":"__consumer_offsets", "partition":4, "replicas":[106,101,102,105]}, 618 | {"topic":"__consumer_offsets", "partition":5, "replicas":[106,101,102,105]}, 619 | {"topic":"__consumer_offsets", "partition":6, "replicas":[106,101,102,105]}, 620 | {"topic":"__consumer_offsets", "partition":7, "replicas":[106,101,102,105]}, 621 | {"topic":"__consumer_offsets", "partition":8, "replicas":[106,101,102,105]}, 622 | {"topic":"__consumer_offsets", "partition":9, "replicas":[106,101,102,105]}, 623 | {"topic":"__consumer_offsets", "partition":10, "replicas":[101,102,103,105]}, 624 | {"topic":"__consumer_offsets", "partition":11, "replicas":[101,102,103,105]}, 625 | {"topic":"__consumer_offsets", "partition":12, "replicas":[101,102,103,105]}, 626 | {"topic":"__consumer_offsets", "partition":13, "replicas":[101,102,103,105]}, 627 | {"topic":"__consumer_offsets", "partition":14, "replicas":[101,102,103,105]}, 628 | {"topic":"__consumer_offsets", "partition":15, "replicas":[101,102,103,105]}, 629 | {"topic":"__consumer_offsets", "partition":16, "replicas":[101,102,103,105]}, 630 | {"topic":"__consumer_offsets", "partition":17, "replicas":[101,102,103,105]}, 631 | {"topic":"__consumer_offsets", "partition":18, "replicas":[101,102,103,105]}, 632 | {"topic":"__consumer_offsets", "partition":19, "replicas":[101,102,103,105]}, 633 | {"topic":"__consumer_offsets", "partition":20, "replicas":[102,103,104,105]}, 634 | {"topic":"__consumer_offsets", "partition":21, "replicas":[102,103,104,105]}, 635 | {"topic":"__consumer_offsets", "partition":22, "replicas":[102,103,104,105]}, 636 | {"topic":"__consumer_offsets", "partition":23, "replicas":[102,103,104,105]}, 637 | {"topic":"__consumer_offsets", "partition":24, "replicas":[102,103,104,105]}, 638 | {"topic":"__consumer_offsets", "partition":25, "replicas":[102,103,104,105]}, 639 | {"topic":"__consumer_offsets", "partition":26, "replicas":[102,103,104,105]}, 640 | {"topic":"__consumer_offsets", "partition":27, "replicas":[102,103,104,105]}, 641 | {"topic":"__consumer_offsets", "partition":28, "replicas":[102,103,104,105]}, 642 | {"topic":"__consumer_offsets", "partition":29, "replicas":[102,103,104,105]}, 643 | {"topic":"__consumer_offsets", "partition":30, "replicas":[103,104,106,105]}, 644 | {"topic":"__consumer_offsets", "partition":31, "replicas":[103,104,106,105]}, 645 | {"topic":"__consumer_offsets", "partition":32, "replicas":[103,104,106,105]}, 646 | {"topic":"__consumer_offsets", "partition":33, "replicas":[103,104,106,105]}, 647 | {"topic":"__consumer_offsets", "partition":34, "replicas":[103,104,106,105]}, 648 | {"topic":"__consumer_offsets", "partition":35, "replicas":[103,104,106,105]}, 649 | {"topic":"__consumer_offsets", "partition":36, "replicas":[103,104,106,105]}, 650 | {"topic":"__consumer_offsets", "partition":37, "replicas":[103,104,106,105]}, 651 | {"topic":"__consumer_offsets", "partition":38, "replicas":[103,104,106,105]}, 652 | {"topic":"__consumer_offsets", "partition":39, "replicas":[103,104,106,105]}, 653 | {"topic":"__consumer_offsets", "partition":40, "replicas":[104,106,101,105]}, 654 | {"topic":"__consumer_offsets", "partition":41, "replicas":[104,106,101,105]}, 655 | {"topic":"__consumer_offsets", "partition":42, "replicas":[104,106,101,105]}, 656 | {"topic":"__consumer_offsets", "partition":43, "replicas":[104,106,101,105]}, 657 | {"topic":"__consumer_offsets", "partition":44, "replicas":[104,106,101,105]}, 658 | {"topic":"__consumer_offsets", "partition":45, "replicas":[104,106,101,105]}, 659 | {"topic":"__consumer_offsets", "partition":46, "replicas":[104,106,101,105]}, 660 | {"topic":"__consumer_offsets", "partition":47, "replicas":[104,106,101,105]}, 661 | {"topic":"__consumer_offsets", "partition":48, "replicas":[104,106,101,105]}, 662 | {"topic":"__consumer_offsets", "partition":49, "replicas":[104,106,101,105]} 663 | ] 664 | } 665 | ---- 666 | 667 | ==== Increase partition 668 | 669 | [source,bash] 670 | ---- 671 | # For VMs 672 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 673 | --zookeeper $ZOOKEEPER_HOST \ 674 | --reassignment-json-file reassignment.json \ 675 | --execute 676 | 677 | # For Kubernetes 678 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 679 | --zookeeper $ZOOKEEPER_HOST \ 680 | --reassignment-json-file reassignment.json \ 681 | --execute 682 | ---- 683 | 684 | ==== Verify reassignment 685 | 686 | [source,bash] 687 | ---- 688 | # For VMs 689 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 690 | --zookeeper $ZOOKEEPER_HOST \ 691 | --reassignment-json-file reassignment.json \ 692 | --verify 693 | 694 | # For Kubernetes 695 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 696 | --zookeeper $ZOOKEEPER_HOST \ 697 | --reassignment-json-file reassignment.json \ 698 | --verify 699 | ---- 700 | 701 | === Alter topic 702 | 703 | ==== Alter retention time 704 | 705 | [source,bash] 706 | ---- 707 | # For VMs 708 | $KAFKA_BIN/kafka-topics.sh \ 709 | --zookeeper $ZOOKEEPER_HOST \ 710 | --alter \ 711 | --topic \ 712 | --config retention.ms=1000 713 | 714 | # For Kubernetes 715 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 716 | --zookeeper $ZOOKEEPER_HOST \ 717 | --alter \ 718 | --topic \ 719 | --config retention.ms=1000 720 | ---- 721 | 722 | ==== Alter min.insync.replicas 723 | 724 | [source,bash] 725 | ---- 726 | # For VMs 727 | $KAFKA_BIN/kafka-topics.sh \ 728 | --zookeeper $ZOOKEEPER_HOST \ 729 | --alter \ 730 | --topic \ 731 | --config min.insync.replicas=2 732 | 733 | # For Kubernetes 734 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 735 | --zookeeper $ZOOKEEPER_HOST \ 736 | --alter \ 737 | --topic \ 738 | --config min.insync.replicas=2 739 | ---- 740 | 741 | ==== Alter max.message.bytes 742 | 743 | [source,bash] 744 | ---- 745 | # For VMs 746 | $KAFKA_BIN/kafka-configs.sh \ 747 | --zookeeper $ZOOKEEPER_HOST \ 748 | --entity-type topics \ 749 | --entity-name \ 750 | --alter \ 751 | --add-config max.message.bytes=128000 752 | 753 | # For Kubernetes 754 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 755 | --zookeeper $ZOOKEEPER_HOST \ 756 | --entity-type topics \ 757 | --entity-name \ 758 | --alter \ 759 | --add-config max.message.bytes=128000 760 | ---- 761 | 762 | ==== Delete retention time 763 | 764 | [source,bash] 765 | ---- 766 | # For VMs 767 | $KAFKA_BIN/kafka-topics.sh \ 768 | --zookeeper $ZOOKEEPER_HOST \ 769 | --alter \ 770 | --topic \ 771 | --delete-config retention.ms 772 | 773 | # For Kubernetes 774 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 775 | --zookeeper $ZOOKEEPER_HOST \ 776 | --alter \ 777 | --topic \ 778 | --delete-config retention.ms 779 | ---- 780 | 781 | [source,bash] 782 | ---- 783 | # For VMs 784 | $KAFKA_BIN/kafka-configs.sh \ 785 | --zookeeper $ZOOKEEPER_HOST \ 786 | --entity-type topics \ 787 | --entity-name \ 788 | --alter \ 789 | --delete-config retention.ms 790 | 791 | # For Kubernetes 792 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 793 | --zookeeper $ZOOKEEPER_HOST \ 794 | --entity-type topics \ 795 | --entity-name \ 796 | --alter \ 797 | --delete-config retention.ms 798 | ---- 799 | 800 | === List topics under-replicated 801 | 802 | [source,bash] 803 | ---- 804 | # For VMs 805 | $KAFKA_BIN/kafka-topics.sh \ 806 | --zookeeper $ZOOKEEPER_HOST \ 807 | --describe \ 808 | --under-replicated-partitions 809 | 810 | # For Kubernetes 811 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 812 | --zookeeper $ZOOKEEPER_HOST \ 813 | --describe \ 814 | --under-replicated-partitions 815 | ---- 816 | 817 | === Delete topic 818 | 819 | [source,bash] 820 | ---- 821 | # For VMs 822 | $KAFKA_BIN/kafka-topics.sh \ 823 | --delete \ 824 | --zookeeper $ZOOKEEPER_HOST \ 825 | --topic 826 | 827 | # For Kubernetes 828 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 829 | --delete \ 830 | --zookeeper $ZOOKEEPER_HOST \ 831 | --topic 832 | ---- 833 | 834 | [source,bash] 835 | ---- 836 | # For VMs 837 | $KAFKA_BIN/kafka-topics.sh \ 838 | --bootstrap-server $BROKER_HOST \ 839 | --delete \ 840 | --topic 841 | 842 | # For Kubernetes 843 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 844 | --bootstrap-server $BROKER_HOST \ 845 | --delete \ 846 | --topic 847 | ---- 848 | 849 | === Get earliest offset 850 | 851 | [source,bash] 852 | ---- 853 | # For VMs 854 | $KAFKA_BIN/kafka-run-class.sh \ 855 | kafka.tools.GetOffsetShell \ 856 | --broker-list $BROKER_HOST \ 857 | --topic \ 858 | --time -2 859 | 860 | # For Kubernetes 861 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-run-class.sh \ 862 | kafka.tools.GetOffsetShell \ 863 | --broker-list $BROKER_HOST \ 864 | --topic \ 865 | --time -2 866 | ---- 867 | 868 | === Get latest offset 869 | 870 | [source,bash] 871 | ---- 872 | # For VMs 873 | $KAFKA_BIN/kafka-run-class.sh \ 874 | kafka.tools.GetOffsetShell \ 875 | --broker-list $BROKER_HOST \ 876 | --topic \ 877 | --time -1 878 | 879 | # For Kubernetes 880 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-run-class.sh \ 881 | kafka.tools.GetOffsetShell \ 882 | --broker-list $BROKER_HOST \ 883 | --topic \ 884 | --time -1 885 | ---- 886 | 887 | == Partition Operations 888 | 889 | === Increase partition number 890 | 891 | [source,bash] 892 | ---- 893 | # For VMs 894 | $KAFKA_BIN/kafka-topics.sh \ 895 | --alter \ 896 | --topic \ 897 | --partitions 8 898 | 899 | # For Kubernetes 900 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 901 | --alter \ 902 | --topic \ 903 | --partitions 8 904 | ---- 905 | 906 | === Increase replication factor 907 | 908 | .topics.json 909 | [source,json] 910 | ---- 911 | { 912 | "topics": [ 913 | { 914 | "topic": "test" 915 | } 916 | ], 917 | "version": 1 918 | } 919 | ---- 920 | 921 | [source,bash] 922 | ---- 923 | # For VMs 924 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 925 | --zookeeper $ZOOKEEPER_HOST \ 926 | --generate \ 927 | --broker-list "401,402,601" \ 928 | --topics-to-move-json-file topics.json 929 | 930 | # For Kubernetes 931 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 932 | --zookeeper $ZOOKEEPER_HOST \ 933 | --generate \ 934 | --broker-list "401,402,601" \ 935 | --topics-to-move-json-file topics.json 936 | ---- 937 | 938 | .new-replication-factor.json 939 | [source,json] 940 | ---- 941 | {"version":1,"partitions":[{"topic":"topic1","partition":0,"replicas":[5,6,7]}]} 942 | ---- 943 | 944 | .execute new replication factor 945 | [source,bash] 946 | ---- 947 | # For VMs 948 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 949 | --zookeeper $ZOOKEEPER_HOST \ 950 | --reassignment-json-file new-replication-factor.json \ 951 | --execute 952 | 953 | # For Kubernetes 954 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 955 | --zookeeper $ZOOKEEPER_HOST \ 956 | --reassignment-json-file new-replication-factor.json \ 957 | --execute 958 | ---- 959 | 960 | .verify status of partition reassignment 961 | [source,bash] 962 | ---- 963 | # For VMs 964 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 965 | --zookeeper $ZOOKEEPER_HOST \ 966 | --reassignment-json-file new-replication-factor.json \ 967 | --verify 968 | 969 | $KAFKA_BIN/kafka-topics.sh \ 970 | --bootstrap-server $ZOOKEEPER_HOST \ 971 | --topic \ 972 | --describe 973 | 974 | # For Kubernetes 975 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 976 | --zookeeper $ZOOKEEPER_HOST \ 977 | --reassignment-json-file new-replication-factor.json \ 978 | --verify 979 | 980 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 981 | --bootstrap-server $ZOOKEEPER_HOST \ 982 | --topic \ 983 | --describe 984 | ---- 985 | 986 | === Reassign partitions 987 | 988 | Create plan 989 | 990 | .topics.json 991 | [source,json] 992 | ---- 993 | { 994 | "topics": [ 995 | { 996 | "topic": "test" 997 | } 998 | ], 999 | "version": 1 1000 | } 1001 | ---- 1002 | 1003 | [source,bash] 1004 | ---- 1005 | # For VMs 1006 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 1007 | --zookeeper $ZOOKEEPER_HOST \ 1008 | --generate \ 1009 | --broker-list "401,402,601" \ 1010 | --topics-to-move-json-file topics.json 1011 | 1012 | # For Kubernetes 1013 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 1014 | --zookeeper $ZOOKEEPER_HOST \ 1015 | --generate \ 1016 | --broker-list "401,402,601" \ 1017 | --topics-to-move-json-file topics.json 1018 | ---- 1019 | 1020 | Save the result of the above command to a file named replicas.json 1021 | 1022 | [source,bash] 1023 | ---- 1024 | # For VMs 1025 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 1026 | --zookeeper $ZOOKEEPER_HOST \ 1027 | --reassignment-json-file replicas.json \ 1028 | --execute 1029 | 1030 | $KAFKA_BIN/kafka-reassign-partitions.sh \ 1031 | --zookeeper $ZOOKEEPER_HOST \ 1032 | --reassignment-json-file replicas.json \ 1033 | --verify 1034 | 1035 | # For Kubernetes 1036 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 1037 | --zookeeper $ZOOKEEPER_HOST \ 1038 | --reassignment-json-file replicas.json \ 1039 | --execute 1040 | 1041 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-reassign-partitions.sh \ 1042 | --zookeeper $ZOOKEEPER_HOST \ 1043 | --reassignment-json-file replicas.json \ 1044 | --verify 1045 | ---- 1046 | 1047 | === List unavailable partitions 1048 | 1049 | [source,bash] 1050 | ---- 1051 | # For VMs 1052 | $KAFKA_BIN/kafka-topics.sh \ 1053 | --zookeeper $ZOOKEEPER_HOST \ 1054 | --describe \ 1055 | --unavailable-partitions 1056 | 1057 | # For Kubernetes 1058 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-topics.sh \ 1059 | --zookeeper $ZOOKEEPER_HOST \ 1060 | --describe \ 1061 | --unavailable-partitions 1062 | ---- 1063 | 1064 | === Force election on all partitions 1065 | 1066 | [source,bash] 1067 | ---- 1068 | # For VMs 1069 | $KAFKA_BIN/kafka-leader-election.sh \ 1070 | --election-type preferred \ 1071 | --bootstrap-server $BROKER_HOST \ 1072 | --all-topic-partitions 1073 | 1074 | # For Kubernetes 1075 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-leader-election.sh \ 1076 | --election-type preferred \ 1077 | --bootstrap-server $BROKER_HOST \ 1078 | --all-topic-partitions 1079 | ---- 1080 | 1081 | === Force election on specific topic and partition 1082 | 1083 | [source,bash] 1084 | ---- 1085 | # For VMs 1086 | $KAFKA_BIN/kafka-leader-election.sh \ 1087 | --election-type preferred \ 1088 | --bootstrap-server $BROKER_HOST \ 1089 | --topic \ 1090 | --partition 1091 | 1092 | # For Kubernetes 1093 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-leader-election.sh \ 1094 | --election-type preferred \ 1095 | --bootstrap-server $BROKER_HOST \ 1096 | --topic \ 1097 | --partition 1098 | ---- 1099 | 1100 | == Consumer 1101 | 1102 | === List consumer groups 1103 | 1104 | [source,bash] 1105 | ---- 1106 | # For VMs 1107 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1108 | --list \ 1109 | --bootstrap-server $BROKER_HOST 1110 | 1111 | # For Kubernetes 1112 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1113 | --list \ 1114 | --bootstrap-server $BROKER_HOST 1115 | ---- 1116 | 1117 | === Describe consumer groups 1118 | 1119 | [source,bash] 1120 | ---- 1121 | # For VMs 1122 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1123 | --describe \ 1124 | --group \ 1125 | --bootstrap-server $BROKER_HOST 1126 | 1127 | # For Kubernetes 1128 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1129 | --describe \ 1130 | --group \ 1131 | --bootstrap-server $BROKER_HOST 1132 | ---- 1133 | 1134 | === Describe all consumer groups 1135 | 1136 | [source,bash] 1137 | ---- 1138 | # For VMs 1139 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1140 | --describe \ 1141 | --bootstrap-server $BROKER_HOST \ 1142 | --all-groups 1143 | 1144 | # For Kubernetes 1145 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1146 | --describe \ 1147 | --bootstrap-server $BROKER_HOST \ 1148 | --all-groups 1149 | ---- 1150 | 1151 | === Delete consumer group 1152 | 1153 | [source,bash] 1154 | ---- 1155 | # For VMs 1156 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1157 | --bootstrap-server $BROKER_HOST \ 1158 | --delete \ 1159 | --group \ 1160 | --group 1161 | 1162 | # For Kubernetes 1163 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1164 | --bootstrap-server $BROKER_HOST \ 1165 | --delete \ 1166 | --group \ 1167 | --group 1168 | ---- 1169 | 1170 | === Active member in a consumer group 1171 | 1172 | [source,bash] 1173 | ---- 1174 | # For VMs 1175 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1176 | --bootstrap-server $BROKER_HOST \ 1177 | --describe \ 1178 | --group \ 1179 | --members 1180 | 1181 | # For Kubernetes 1182 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1183 | --bootstrap-server $BROKER_HOST \ 1184 | --describe \ 1185 | --group \ 1186 | --members 1187 | ---- 1188 | 1189 | === Partition Assigned to each member 1190 | 1191 | [source,bash] 1192 | ---- 1193 | # For VMs 1194 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1195 | --bootstrap-server $BROKER_HOST \ 1196 | --describe \ 1197 | --group \ 1198 | --members \ 1199 | --verbose 1200 | 1201 | # For Kubernetes 1202 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1203 | --bootstrap-server $BROKER_HOST \ 1204 | --describe \ 1205 | --group \ 1206 | --members \ 1207 | --verbose 1208 | ---- 1209 | 1210 | === Consumer Group State 1211 | 1212 | [source,bash] 1213 | ---- 1214 | # For VMs 1215 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1216 | --bootstrap-server $BROKER_HOST \ 1217 | --describe \ 1218 | --group \ 1219 | --state 1220 | 1221 | # For Kubernetes 1222 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1223 | --bootstrap-server $BROKER_HOST \ 1224 | --describe \ 1225 | --group \ 1226 | --state 1227 | ---- 1228 | 1229 | === Consuming message 1230 | 1231 | [source,bash] 1232 | ---- 1233 | # For VMs 1234 | $KAFKA_BIN/kafka-console-consumer.sh \ 1235 | --bootstrap-server $BROKER_HOST \ 1236 | --topic 1237 | 1238 | # For Kubernetes 1239 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1240 | --bootstrap-server $BROKER_HOST \ 1241 | --topic 1242 | ---- 1243 | 1244 | [source,bash] 1245 | ---- 1246 | kafkacat -C -b $BROKER_HOST -t 1247 | ---- 1248 | 1249 | === Consuming message and formatting output 1250 | 1251 | [source,bash] 1252 | ---- 1253 | kafkacat -C -b $BROKER_HOST -t -q -f 'Topic %t using partition %p at offset %o has key = %k and value = %S' 1254 | ---- 1255 | 1256 | === Consuming message from the beginning 1257 | 1258 | [source,bash] 1259 | ---- 1260 | # For VMs 1261 | $KAFKA_BIN/kafka-console-consumer.sh \ 1262 | --bootstrap-server $BROKER_HOST \ 1263 | --topic \ 1264 | --from-beginning 1265 | 1266 | # For Kubernetes 1267 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1268 | --bootstrap-server $BROKER_HOST \ 1269 | --topic \ 1270 | --from-beginning 1271 | ---- 1272 | 1273 | === Consuming message from the end 1274 | 1275 | [source,bash] 1276 | ---- 1277 | # For VMs 1278 | $KAFKA_BIN/kafka-console-consumer.sh \ 1279 | --bootstrap-server $BROKER_HOST \ 1280 | --topic 1281 | 1282 | # For Kubernetes 1283 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1284 | --bootstrap-server $BROKER_HOST \ 1285 | --topic 1286 | ---- 1287 | 1288 | === Consuming message and show output in JSON 1289 | 1290 | [source,bash] 1291 | ---- 1292 | kafkacat -b $BROKER_HOST -t -J 1293 | ---- 1294 | 1295 | === Consuming and showing message key 1296 | 1297 | [source,bash] 1298 | ---- 1299 | # For VMs 1300 | $KAFKA_BIN/kafka-console-consumer.sh \ 1301 | --bootstrap-server $BROKER_HOST \ 1302 | --topic \ 1303 | --property print.key=true \ 1304 | --property key.separator=, 1305 | 1306 | # For Kubernetes 1307 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1308 | --bootstrap-server $BROKER_HOST \ 1309 | --topic \ 1310 | --property print.key=true \ 1311 | --property key.separator=, 1312 | ---- 1313 | 1314 | === Read one message 1315 | 1316 | [source,bash] 1317 | ---- 1318 | # For VMs 1319 | $KAFKA_BIN/kafka-console-consumer.sh \ 1320 | --bootstrap-server $BROKER_HOST \ 1321 | --topic \ 1322 | --max-messages 1 1323 | 1324 | # For Kubernetes 1325 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1326 | --bootstrap-server $BROKER_HOST \ 1327 | --topic \ 1328 | --max-messages 1 1329 | ---- 1330 | 1331 | === Read the last 2 messages from topic and then exit 1332 | 1333 | [source,bash] 1334 | ---- 1335 | kafkacat -C -b $BROKER_HOST -t -o -2 -e 1336 | ---- 1337 | 1338 | === Read the last 2 messages from partition 0 1339 | 1340 | [source,bash] 1341 | ---- 1342 | kafkacat -C -b $BROKER_HOST -t -o -2 -e -p 0 1343 | ---- 1344 | 1345 | === Read from __consumer_offsets 1346 | 1347 | [source,bash] 1348 | ---- 1349 | # For VMs 1350 | $KAFKA_BIN/kafka-console-consumer.sh \ 1351 | --bootstrap-server $BROKER_HOST \ 1352 | --topic __consumer_offsets \ 1353 | --formatter 'kafka.coordinator.group.GroupMetadataManager$OffsetsMessageFormatter' \ 1354 | --max-messages 1 1355 | 1356 | # For Kubernetes 1357 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1358 | --bootstrap-server $BROKER_HOST \ 1359 | --topic __consumer_offsets \ 1360 | --formatter 'kafka.coordinator.group.GroupMetadataManager$OffsetsMessageFormatter' \ 1361 | --max-messages 1 1362 | ---- 1363 | 1364 | === Describe __consumer_offsets 1365 | 1366 | [source,bash] 1367 | ---- 1368 | # For VMs 1369 | $KAFKA_BIN/kafka-run-class.sh kafka.admin.ConsumerGroupCommand \ 1370 | --bootstrap-server $BROKER_HOST \ 1371 | --group \ 1372 | --new-consumer \ 1373 | --describe 1374 | 1375 | # For Kubernetes 1376 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-run-class.sh kafka.admin.ConsumerGroupCommand \ 1377 | --bootstrap-server $BROKER_HOST \ 1378 | --group \ 1379 | --new-consumer \ 1380 | --describe 1381 | ---- 1382 | 1383 | === Read from __transaction_state 1384 | 1385 | [source,bash] 1386 | ---- 1387 | # For VMs 1388 | $KAFKA_BIN/kafka-console-consumer.sh \ 1389 | --bootstrap-server $BROKER_HOST \ 1390 | --formatter "kafka.coordinator.transaction.TransactionLog\$TransactionLogMessageFormatter" \ 1391 | --topic __transaction_state \ 1392 | --from-beginning 1393 | 1394 | # For Kubernetes 1395 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1396 | --bootstrap-server $BROKER_HOST \ 1397 | --topic __transaction_state \ 1398 | --from-beginning \ 1399 | --formatter "kafka.coordinator.transaction.TransactionLog\$TransactionLogMessageFormatter" 1400 | ---- 1401 | 1402 | === Consume using consumer group 1403 | 1404 | [source,bash] 1405 | ---- 1406 | # For VMs 1407 | $KAFKA_BIN/kafka-console-consumer.sh \ 1408 | --topic \ 1409 | --bootstrap-server $BROKER_HOST \ 1410 | --group 1411 | 1412 | # For Kubernetes 1413 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-consumer.sh \ 1414 | --topic \ 1415 | --bootstrap-server $BROKER_HOST \ 1416 | --group 1417 | ---- 1418 | 1419 | === Topics to which group is subscribed 1420 | 1421 | [source,bash] 1422 | ---- 1423 | # For VMs 1424 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1425 | --bootstrap-server $BROKER_HOST \ 1426 | --group \ 1427 | --describe 1428 | 1429 | # For Kubernetes 1430 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1431 | --bootstrap-server $BROKER_HOST \ 1432 | --group \ 1433 | --describe 1434 | ---- 1435 | 1436 | === Reset offset 1437 | 1438 | ==== Reset to the latest offset 1439 | 1440 | [source,bash] 1441 | ---- 1442 | # For VMs 1443 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1444 | --bootstrap-server $BROKER_HOST \ 1445 | --reset-offsets \ 1446 | --group \ 1447 | --topic topic1 \ 1448 | --to-latest 1449 | 1450 | # For Kubernetes 1451 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1452 | --bootstrap-server $BROKER_HOST \ 1453 | --reset-offsets \ 1454 | --group \ 1455 | --topic topic1 \ 1456 | --to-latest 1457 | ---- 1458 | 1459 | ==== Reset offset for a consumer group in a topic 1460 | 1461 | [source,bash] 1462 | ---- 1463 | # For VMs 1464 | # There are many other resetting options 1465 | # --shift-by / --to-current / --to-latest / --to-offset 1466 | # --to-datetime --by-duration 1467 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1468 | --bootstrap-server $BROKER_HOST \ 1469 | --group \ 1470 | --topic \ 1471 | --reset-offsets \ 1472 | --to-earliest \ 1473 | --execute 1474 | 1475 | # For Kubernetes 1476 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1477 | --bootstrap-server $BROKER_HOST \ 1478 | --group \ 1479 | --topic \ 1480 | --reset-offsets \ 1481 | --to-earliest \ 1482 | --execute 1483 | ---- 1484 | 1485 | ==== Reset offset from all consumer groups 1486 | 1487 | [source,bash] 1488 | ---- 1489 | # For VMs 1490 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1491 | --bootstrap-server $BROKER_HOST \ 1492 | --all-groups \ 1493 | --reset-offsets \ 1494 | --topic \ 1495 | --to-earliest 1496 | 1497 | # For Kubernetes 1498 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1499 | --bootstrap-server $BROKER_HOST \ 1500 | --all-groups \ 1501 | --reset-offsets \ 1502 | --topic \ 1503 | --to-earliest 1504 | ---- 1505 | 1506 | ==== Forward by 2 for example 1507 | 1508 | [source,bash] 1509 | ---- 1510 | # For VMs 1511 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1512 | --bootstrap-server $BROKER_HOST \ 1513 | --group \ 1514 | --reset-offsets \ 1515 | --shift-by 2 \ 1516 | --execute \ 1517 | --topic 1518 | 1519 | # For Kubernetes 1520 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1521 | --bootstrap-server $BROKER_HOST \ 1522 | --group \ 1523 | --reset-offsets \ 1524 | --shift-by 2 \ 1525 | --execute \ 1526 | --topic 1527 | ---- 1528 | 1529 | ==== Backward by 2 for example 1530 | 1531 | [source,bash] 1532 | ---- 1533 | # For VMs 1534 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1535 | --bootstrap-server $BROKER_HOST \ 1536 | --group \ 1537 | --reset-offsets \ 1538 | --shift-by -2 \ 1539 | --execute \ 1540 | --topic 1541 | 1542 | # For Kubernetes 1543 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1544 | --bootstrap-server $BROKER_HOST \ 1545 | --group \ 1546 | --reset-offsets \ 1547 | --shift-by -2 \ 1548 | --execute \ 1549 | --topic 1550 | ---- 1551 | 1552 | === Describe consumer group 1553 | 1554 | [source,bash] 1555 | ---- 1556 | # For VMs 1557 | $KAFKA_BIN/kafka-consumer-groups.sh \ 1558 | --bootstrap-server $BROKER_HOST \ 1559 | --describe \ 1560 | --group 1561 | 1562 | # For Kubernetes 1563 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-groups.sh \ 1564 | --bootstrap-server $BROKER_HOST \ 1565 | --describe \ 1566 | --group 1567 | ---- 1568 | 1569 | === Check offset for consumer group 1570 | 1571 | [source,bash] 1572 | ---- 1573 | # For VMs 1574 | $KAFKA_BIN/kafka-consumer-offset-checker.sh \ 1575 | --zookeeper $ZOOKEEPER_HOST \ 1576 | --group \ 1577 | --topic 1578 | 1579 | # For Kubernetes 1580 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-offset-checker.sh \ 1581 | --zookeeper $ZOOKEEPER_HOST \ 1582 | --group \ 1583 | --topic 1584 | ---- 1585 | 1586 | == Producer 1587 | 1588 | === Send message using file 1589 | 1590 | [source,bash] 1591 | ---- 1592 | # For VMs 1593 | $KAFKA_BIN/kafka-console-producer.sh \ 1594 | --broker-list $BROKER_HOST \ 1595 | --topic < messages.txt 1596 | 1597 | # For Kubernetes 1598 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-producer.sh \ 1599 | --broker-list $BROKER_HOST \ 1600 | --topic < messages.txt 1601 | ---- 1602 | 1603 | [source,bash] 1604 | ---- 1605 | kafkacat -P -l -b $BROKER_HOST -t messages.txt 1606 | ---- 1607 | 1608 | === Send message using standard input 1609 | 1610 | [source,bash] 1611 | ---- 1612 | # For VMs 1613 | $KAFKA_BIN/kafka-console-producer.sh \ 1614 | --broker-list $BROKER_HOST \ 1615 | --topic 1616 | 1617 | # For Kubernetes 1618 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-producer.sh \ 1619 | --broker-list $BROKER_HOST \ 1620 | --topic 1621 | ---- 1622 | 1623 | [source,bash] 1624 | ---- 1625 | kafkacat -P -b $BROKER_HOST -t 1626 | ---- 1627 | 1628 | === Send message using snappy compression 1629 | 1630 | [source,bash] 1631 | ---- 1632 | kafkacat -P -b $BROKER_HOST -t -z snappy 1633 | ---- 1634 | 1635 | === Send 200 messages to a topic 1636 | 1637 | [source,bash] 1638 | ---- 1639 | seq 200 | kafkacat -P -b $BROKER_HOST -t 1640 | ---- 1641 | 1642 | === Send message using string 1643 | 1644 | [source,bash] 1645 | ---- 1646 | # For VMs 1647 | echo "My Message" | $KAFKA_BIN/kafka-console-producer.sh \ 1648 | --broker-list $BROKER_HOST \ 1649 | --topic 1650 | 1651 | # For Kubernetes 1652 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-producer.sh \ 1653 | --broker-list $BROKER_HOST \ 1654 | --topic 1655 | ---- 1656 | 1657 | [source,bash] 1658 | ---- 1659 | echo "My Message" | kafkacat -b $BROKER_HOST -t 1660 | ---- 1661 | 1662 | === Send message using headers 1663 | 1664 | [source,bash] 1665 | ---- 1666 | echo "My Message" | kafkacat -b $BROKER_HOST -t 1667 | echo "My Message" | kafkacat -b $BROKER_HOST -H "header1=value1" -H "header2=value2" 1668 | ---- 1669 | 1670 | === Send message using ack=all 1671 | 1672 | [source,bash] 1673 | ---- 1674 | # For VMs 1675 | $KAFKA_BIN/kafka-console-producer.sh \ 1676 | --broker-list $BROKER_HOST \ 1677 | --topic \ 1678 | --producer-property acks=all 1679 | 1680 | # For Kubernetes 1681 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-producer.sh \ 1682 | --broker-list $BROKER_HOST \ 1683 | --topic \ 1684 | --producer-property acks=all 1685 | ---- 1686 | 1687 | === Send message with key 1688 | 1689 | [source,bash] 1690 | ---- 1691 | # For VMs 1692 | $KAFKA_BIN/kafka-console-producer.sh \ 1693 | --broker-list $BROKER_HOST \ 1694 | --topic \ 1695 | --property parse.key=true \ 1696 | --property key.separator=, 1697 | 1698 | # For Kubernetes 1699 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-console-producer.sh \ 1700 | --broker-list $BROKER_HOST \ 1701 | --topic \ 1702 | --property parse.key=true \ 1703 | --property key.separator=, 1704 | ---- 1705 | 1706 | NOTE: Your message should be: ,. For example: Gus,1000. 1707 | 1708 | == Quotas 1709 | 1710 | === Add quota for user and client-id 1711 | 1712 | [source,bash] 1713 | ---- 1714 | # For VMs 1715 | $KAFKA_BIN/kafka-configs.sh \ 1716 | --zookeeper $ZOOKEEPER_HOST \ 1717 | --alter \ 1718 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1719 | --entity-type users \ 1720 | --entity-name \ 1721 | --entity-type clients \ 1722 | --entity-name 1723 | 1724 | # For Kubernetes 1725 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1726 | --zookeeper $ZOOKEEPER_HOST \ 1727 | --alter \ 1728 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1729 | --entity-type users \ 1730 | --entity-name \ 1731 | --entity-type clients \ 1732 | --entity-name 1733 | ---- 1734 | 1735 | === Add quota for user 1736 | 1737 | [source,bash] 1738 | ---- 1739 | # For VMs 1740 | $KAFKA_BIN/kafka-configs.sh \ 1741 | --zookeeper $ZOOKEEPER_HOST \ 1742 | --alter \ 1743 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1744 | --entity-type users \ 1745 | --entity-name 1746 | 1747 | # For Kubernetes 1748 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1749 | --zookeeper $ZOOKEEPER_HOST \ 1750 | --alter \ 1751 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1752 | --entity-type users \ 1753 | --entity-name 1754 | ---- 1755 | 1756 | === Add quota for client-id 1757 | 1758 | [source,bash] 1759 | ---- 1760 | # For VMs 1761 | $KAFKA_BIN/kafka-configs.sh \ 1762 | --zookeeper $ZOOKEEPER_HOST \ 1763 | --alter \ 1764 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1765 | --entity-type clients \ 1766 | --entity-name 1767 | 1768 | # For Kubernetes 1769 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1770 | --zookeeper $ZOOKEEPER_HOST \ 1771 | --alter \ 1772 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1773 | --entity-type clients \ 1774 | --entity-name 1775 | ---- 1776 | 1777 | === Add default client-id quota for user 1778 | 1779 | [source,bash] 1780 | ---- 1781 | # For VMs 1782 | $KAFKA_BIN/kafka-configs.sh \ 1783 | --zookeeper $ZOOKEEPER_HOST \ 1784 | --alter \ 1785 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1786 | --entity-type users \ 1787 | --entity-name \ 1788 | --entity-type clients \ 1789 | --entity-default 1790 | 1791 | # For Kubernetes 1792 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1793 | --zookeeper $ZOOKEEPER_HOST \ 1794 | --alter \ 1795 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1796 | --entity-type users \ 1797 | --entity-name \ 1798 | --entity-type clients \ 1799 | --entity-default 1800 | ---- 1801 | 1802 | === Add default quota for user 1803 | 1804 | [source,bash] 1805 | ---- 1806 | # For VMs 1807 | $KAFKA_BIN/kafka-configs.sh \ 1808 | --zookeeper $ZOOKEEPER_HOST \ 1809 | --alter \ 1810 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1811 | --entity-type users \ 1812 | --entity-default 1813 | 1814 | # For Kubernetes 1815 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1816 | --zookeeper $ZOOKEEPER_HOST \ 1817 | --alter \ 1818 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1819 | --entity-type users \ 1820 | --entity-default 1821 | ---- 1822 | 1823 | === Add default quota for client-id 1824 | 1825 | [source,bash] 1826 | ---- 1827 | # For VMs 1828 | $KAFKA_BIN/kafka-configs.sh \ 1829 | --zookeeper $ZOOKEEPER_HOST \ 1830 | --alter \ 1831 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1832 | --entity-type clients \ 1833 | --entity-default 1834 | 1835 | # For Kubernetes 1836 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1837 | --zookeeper $ZOOKEEPER_HOST \ 1838 | --alter \ 1839 | --add-config 'producer_byte_rate=1024,consumer_byte_rate=2048,request_percentage=200' \ 1840 | --entity-type clients \ 1841 | --entity-default 1842 | ---- 1843 | 1844 | === Describe quota for user and client-id 1845 | 1846 | [source,bash] 1847 | ---- 1848 | # For VMs 1849 | $KAFKA_BIN/kafka-configs.sh \ 1850 | --zookeeper $ZOOKEEPER_HOST \ 1851 | --describe \ 1852 | --entity-type users \ 1853 | --entity-name \ 1854 | --entity-type clients \ 1855 | --entity-name 1856 | 1857 | # For Kubernetes 1858 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1859 | --zookeeper $ZOOKEEPER_HOST \ 1860 | --describe \ 1861 | --entity-type users \ 1862 | --entity-name \ 1863 | --entity-type clients \ 1864 | --entity-name 1865 | ---- 1866 | 1867 | === Describe quota for a user 1868 | 1869 | [source,bash] 1870 | ---- 1871 | # For VMs 1872 | $KAFKA_BIN/kafka-configs.sh \ 1873 | --zookeeper $ZOOKEEPER_HOST \ 1874 | --describe \ 1875 | --entity-type users \ 1876 | --entity-name 1877 | 1878 | # For Kubernetes 1879 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1880 | --zookeeper $ZOOKEEPER_HOST \ 1881 | --describe \ 1882 | --entity-type users \ 1883 | --entity-name 1884 | ---- 1885 | 1886 | === Describe quota for a client 1887 | 1888 | [source,bash] 1889 | ---- 1890 | # For VMs 1891 | $KAFKA_BIN/kafka-configs.sh \ 1892 | --zookeeper $ZOOKEEPER_HOST \ 1893 | --describe \ 1894 | --entity-type clients \ 1895 | --entity-name 1896 | 1897 | # For Kubernetes 1898 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-configs.sh \ 1899 | --zookeeper $ZOOKEEPER_HOST \ 1900 | --describe \ 1901 | --entity-type clients \ 1902 | --entity-name 1903 | ---- 1904 | 1905 | == ACLs 1906 | 1907 | === Allow and to read and write 1908 | 1909 | [source,bash] 1910 | ---- 1911 | # For VMs 1912 | $KAFKA_BIN/kafka-acls.sh \ 1913 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1914 | --add \ 1915 | --allow-principal User: \ 1916 | --allow-principal User: \ 1917 | --allow-host \ 1918 | --allow-host \ 1919 | --operation Read \ 1920 | --operation Write \ 1921 | --topic 1922 | 1923 | # For Kubernetes 1924 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-acls.sh \ 1925 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1926 | --add \ 1927 | --allow-principal User: \ 1928 | --allow-principal User: \ 1929 | --allow-host \ 1930 | --allow-host \ 1931 | --operation Read \ 1932 | --operation Write \ 1933 | --topic 1934 | ---- 1935 | 1936 | === Allow all read from topic but 1937 | 1938 | [source,bash] 1939 | ---- 1940 | # For VMs 1941 | $KAFKA_BIN/kafka-acls.sh \ 1942 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1943 | --add \ 1944 | --allow-principal User:* \ 1945 | --allow-host * \ 1946 | --deny-principal User: \ 1947 | --deny-host \ 1948 | --operation Read \ 1949 | --topic 1950 | 1951 | # For Kubernetes 1952 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-acls.sh \ 1953 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1954 | --add \ 1955 | --allow-principal User:* \ 1956 | --allow-host * \ 1957 | --deny-principal User: \ 1958 | --deny-host \ 1959 | --operation Read \ 1960 | --topic 1961 | ---- 1962 | 1963 | === Allow to produce on all topics 1964 | 1965 | [source,bash] 1966 | ---- 1967 | # For VMs 1968 | $KAFKA_BIN/kafka-acls.sh \ 1969 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1970 | --add \ 1971 | --allow-principal User: \ 1972 | --allow-host \ 1973 | --producer --topic * 1974 | 1975 | # For Kubernetes 1976 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-acls.sh \ 1977 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1978 | --add \ 1979 | --allow-principal User: \ 1980 | --allow-host \ 1981 | --producer --topic * 1982 | ---- 1983 | 1984 | === Allow to consume on all topics 1985 | 1986 | [source,bash] 1987 | ---- 1988 | # For VMs 1989 | $KAFKA_BIN/kafka-acls.sh \ 1990 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1991 | --add \ 1992 | --allow-principal User: \ 1993 | --allow-host \ 1994 | --consume --topic * 1995 | 1996 | # For Kubernetes 1997 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-acls.sh \ 1998 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 1999 | --add \ 2000 | --allow-principal User: \ 2001 | --allow-host \ 2002 | --consume --topic * 2003 | ---- 2004 | 2005 | === Remove ACL for and to read and write 2006 | 2007 | [source,bash] 2008 | ---- 2009 | # For VMs 2010 | $KAFKA_BIN/kafka-acls.sh \ 2011 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 2012 | --remove \ 2013 | --allow-principal User: \ 2014 | --allow-principal User: \ 2015 | --allow-host \ 2016 | --allow-host \ 2017 | --operation Read \ 2018 | --operation Write \ 2019 | --topic 2020 | 2021 | # For Kubernetes 2022 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-acls.sh \ 2023 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 2024 | --remove \ 2025 | --allow-principal User: \ 2026 | --allow-principal User: \ 2027 | --allow-host \ 2028 | --allow-host \ 2029 | --operation Read \ 2030 | --operation Write \ 2031 | --topic 2032 | ---- 2033 | 2034 | === List ACLs on specific topic 2035 | 2036 | [source,bash] 2037 | ---- 2038 | # For VMs 2039 | $KAFKA_BIN/kafka-acls.sh \ 2040 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 2041 | --list \ 2042 | --topic 2043 | 2044 | # For Kubernetes 2045 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-acls.sh \ 2046 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 2047 | --list \ 2048 | --topic 2049 | ---- 2050 | 2051 | === List ACLs on all topics 2052 | 2053 | [source,bash] 2054 | ---- 2055 | # For VMs 2056 | $KAFKA_BIN/kafka-acls.sh \ 2057 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 2058 | --list \ 2059 | --topic * 2060 | 2061 | # For Kubernetes 2062 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-acls.sh \ 2063 | --authorizer-properties zookeeper.connect=$ZOOKEEPER_HOST \ 2064 | --list \ 2065 | --topic * 2066 | ---- 2067 | 2068 | == Mirror Maker 2069 | 2070 | === Mirror topic 2071 | 2072 | [source,bash] 2073 | ---- 2074 | # For VMs 2075 | $KAFKA_BIN/kafka-mirror-maker.sh \ 2076 | --consumer.config consumer.properties \ 2077 | --producer.config producer.properties \ 2078 | --whitelist 2079 | 2080 | # For Kubernetes 2081 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-mirror-maker.sh \ 2082 | --consumer.config consumer.properties \ 2083 | --producer.config producer.properties \ 2084 | --whitelist 2085 | ---- 2086 | 2087 | == Delegation Token 2088 | 2089 | === Create token 2090 | 2091 | [source,bash] 2092 | ---- 2093 | # For VMs 2094 | $KAFKA_BIN/kafka-delegation-tokens.sh \ 2095 | --bootstrap-server $BROKER_HOST \ 2096 | --create \ 2097 | --max-life-time-period -1 \ 2098 | --command-config client.properties \ 2099 | --renewer-principal User: 2100 | 2101 | # For Kubernetes 2102 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-delegation-tokens.sh \ 2103 | --bootstrap-server $BROKER_HOST \ 2104 | --create \ 2105 | --max-life-time-period -1 \ 2106 | --command-config client.properties \ 2107 | --renewer-principal User: 2108 | ---- 2109 | 2110 | === Renew token 2111 | 2112 | [source,bash] 2113 | ---- 2114 | # For VMs 2115 | $KAFKA_BIN/kafka-delegation-tokens.sh \ 2116 | --bootstrap-server $BROKER_HOST \ 2117 | --renew \ 2118 | --renew-time-period -1 \ 2119 | --command-config client.properties \ 2120 | --hmac ABCDEFGHIJK 2121 | 2122 | # For Kubernetes 2123 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-delegation-tokens.sh \ 2124 | --bootstrap-server $BROKER_HOST \ 2125 | --renew \ 2126 | --renew-time-period -1 \ 2127 | --command-config client.properties \ 2128 | --hmac ABCDEFGHIJK 2129 | ---- 2130 | 2131 | === Expire token 2132 | 2133 | [source,bash] 2134 | ---- 2135 | # For VMs 2136 | $KAFKA_BIN/kafka-delegation-tokens.sh \ 2137 | --bootstrap-server $BROKER_HOST \ 2138 | --expire \ 2139 | --expiry-time-period -1 \ 2140 | --command-config client.properties \ 2141 | --hmac ABCDEFGHIJK 2142 | 2143 | # For Kubernetes 2144 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-delegation-tokens.sh \ 2145 | --bootstrap-server $BROKER_HOST \ 2146 | --expire \ 2147 | --expiry-time-period -1 \ 2148 | --command-config client.properties \ 2149 | --hmac ABCDEFGHIJK 2150 | ---- 2151 | 2152 | === Describe token 2153 | 2154 | [source,bash] 2155 | ---- 2156 | # For VMs 2157 | $KAFKA_BIN/kafka-delegation-tokens.sh \ 2158 | --bootstrap-server $BROKER_HOST \ 2159 | --describe \ 2160 | --command-config client.properties \ 2161 | --owner-principal User: 2162 | 2163 | # For Kubernetes 2164 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-delegation-tokens.sh \ 2165 | --bootstrap-server $BROKER_HOST \ 2166 | --describe \ 2167 | --command-config client.properties \ 2168 | --owner-principal User: 2169 | ---- 2170 | 2171 | == Performance Test 2172 | 2173 | === Producer 2174 | 2175 | [source,bash] 2176 | ---- 2177 | # For VMs 2178 | $KAFKA_BIN/kafka-producer-perf-test.sh \ 2179 | --topic teste \ 2180 | --num-records 50000000 \ 2181 | --record-size 100 \ 2182 | --throughput -1 \ 2183 | --producer-props acks=all bootstrap.servers=$BROKER_HOST buffer.memory=67108864 batch.size=8196 2184 | 2185 | # For Kubernetes 2186 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-producer-perf-test.sh \ 2187 | --topic teste \ 2188 | --num-records 50000000 \ 2189 | --record-size 100 \ 2190 | --throughput -1 \ 2191 | --producer-props acks=all bootstrap.servers=$BROKER_HOST buffer.memory=67108864 batch.size=8196 2192 | ---- 2193 | 2194 | === Consumer 2195 | 2196 | [source,bash] 2197 | ---- 2198 | # For VMs 2199 | $KAFKA_BIN/kafka-consumer-perf-test.sh \ 2200 | --group grupo \ 2201 | --print-metrics \ 2202 | --show-detailed-stats \ 2203 | --topic teste \ 2204 | --messages 600000 \ 2205 | --broker-list $BROKER_HOST \ 2206 | --timeout 1000000 2207 | 2208 | # For Kubernetes 2209 | kubectl -n $KAFKA_NAMESPACE exec -it $KAFKA_BROKER_POD -c kafka -- bin/kafka-consumer-perf-test.sh \ 2210 | --group grupo \ 2211 | --print-metrics \ 2212 | --show-detailed-stats \ 2213 | --topic teste \ 2214 | --messages 600000 \ 2215 | --broker-list $BROKER_HOST \ 2216 | --timeout 1000000 2217 | ---- 2218 | 2219 | --------------------------------------------------------------------------------