├── .gitignore ├── README.md ├── ansible.cfg ├── coding-with-kafka.code-workspace ├── commands.md ├── compose.yaml ├── config ├── install-kafka.sh ├── install-zookeeper.sh ├── kafka.service ├── my-server.properties ├── my-zookeeper.properties └── zookeeper.service ├── devops ├── .terraform.lock.hcl ├── main.tf ├── templates │ ├── ansible-inventory.tftpl │ ├── cluster.env.tftpl │ └── hostsfile.tftpl └── variables.tf ├── guides ├── python-setup.md └── terraform.md ├── main.yaml ├── package-lock.json ├── package.json ├── playbooks ├── kafka.yaml ├── templates │ ├── kafka.properties.j2 │ ├── kafka.systemd.j2 │ ├── zookeeper.properties.j2 │ └── zookeeper.systemd.j2 └── zookeeper.yaml ├── src ├── aconsume.py ├── aproduce.py ├── consume.py ├── js_consume.js ├── js_produce.js └── produce.py ├── vm └── Dockerfile ├── webapp_django ├── cfehome │ ├── __init__.py │ ├── asgi.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── manage.py ├── mykafka │ ├── __init__.py │ └── client.py ├── orders │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ └── listen.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_order_is_shipped.py │ │ └── __init__.py │ ├── models.py │ ├── tests.py │ ├── utils.py │ └── views.py └── requirements.txt └── webapp_fastapi ├── main.py ├── requirements.txt └── run.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_STORE 2 | host-config.txt 3 | inventory.ini 4 | *.env 5 | # Python gitignore 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/#use-with-ide 116 | .pdm.toml 117 | 118 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 119 | __pypackages__/ 120 | 121 | # Celery stuff 122 | celerybeat-schedule 123 | celerybeat.pid 124 | 125 | # SageMath parsed files 126 | *.sage.py 127 | 128 | # Environments 129 | .env 130 | .venv 131 | env/ 132 | venv/ 133 | ENV/ 134 | env.bak/ 135 | venv.bak/ 136 | 137 | # Spyder project settings 138 | .spyderproject 139 | .spyproject 140 | 141 | # Rope project settings 142 | .ropeproject 143 | 144 | # mkdocs documentation 145 | /site 146 | 147 | # mypy 148 | .mypy_cache/ 149 | .dmypy.json 150 | dmypy.json 151 | 152 | # Pyre type checker 153 | .pyre/ 154 | 155 | # pytype static type analyzer 156 | .pytype/ 157 | 158 | # Cython debug symbols 159 | cython_debug/ 160 | 161 | # PyCharm 162 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 163 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 164 | # and can be added to the global gitignore or merged into this file. For a more nuclear 165 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 166 | #.idea/ 167 | 168 | # Node.js gitignore 169 | # Logs 170 | logs 171 | *.log 172 | npm-debug.log* 173 | yarn-debug.log* 174 | yarn-error.log* 175 | lerna-debug.log* 176 | .pnpm-debug.log* 177 | 178 | # Diagnostic reports (https://nodejs.org/api/report.html) 179 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 180 | 181 | # Runtime data 182 | pids 183 | *.pid 184 | *.seed 185 | *.pid.lock 186 | 187 | # Directory for instrumented libs generated by jscoverage/JSCover 188 | lib-cov 189 | 190 | # Coverage directory used by tools like istanbul 191 | coverage 192 | *.lcov 193 | 194 | # nyc test coverage 195 | .nyc_output 196 | 197 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 198 | .grunt 199 | 200 | # Bower dependency directory (https://bower.io/) 201 | bower_components 202 | 203 | # node-waf configuration 204 | .lock-wscript 205 | 206 | # Compiled binary addons (https://nodejs.org/api/addons.html) 207 | build/Release 208 | 209 | # Dependency directories 210 | node_modules/ 211 | jspm_packages/ 212 | 213 | # Snowpack dependency directory (https://snowpack.dev/) 214 | web_modules/ 215 | 216 | # TypeScript cache 217 | *.tsbuildinfo 218 | 219 | # Optional npm cache directory 220 | .npm 221 | 222 | # Optional eslint cache 223 | .eslintcache 224 | 225 | # Optional stylelint cache 226 | .stylelintcache 227 | 228 | # Microbundle cache 229 | .rpt2_cache/ 230 | .rts2_cache_cjs/ 231 | .rts2_cache_es/ 232 | .rts2_cache_umd/ 233 | 234 | # Optional REPL history 235 | .node_repl_history 236 | 237 | # Output of 'npm pack' 238 | *.tgz 239 | 240 | # Yarn Integrity file 241 | .yarn-integrity 242 | 243 | # dotenv environment variable files 244 | .env 245 | .env.development.local 246 | .env.test.local 247 | .env.production.local 248 | .env.local 249 | 250 | # parcel-bundler cache (https://parceljs.org/) 251 | .cache 252 | .parcel-cache 253 | 254 | # Next.js build output 255 | .next 256 | out 257 | 258 | # Nuxt.js build / generate output 259 | .nuxt 260 | dist 261 | 262 | # Gatsby files 263 | .cache/ 264 | # Comment in the public line in if your project uses Gatsby and not Next.js 265 | # https://nextjs.org/blog/next-9-1#public-directory-support 266 | # public 267 | 268 | # vuepress build output 269 | .vuepress/dist 270 | 271 | # vuepress v2.x temp and cache directory 272 | .temp 273 | .cache 274 | 275 | # Docusaurus cache and generated files 276 | .docusaurus 277 | 278 | # Serverless directories 279 | .serverless/ 280 | 281 | # FuseBox cache 282 | .fusebox/ 283 | 284 | # DynamoDB Local files 285 | .dynamodb/ 286 | 287 | # TernJS port file 288 | .tern-port 289 | 290 | # Stores VSCode versions used for testing VSCode extensions 291 | .vscode-test 292 | 293 | # yarn v2 294 | .yarn/cache 295 | .yarn/unplugged 296 | .yarn/build-state.yml 297 | .yarn/install-state.gz 298 | .pnp.* 299 | 300 | 301 | # Teraform gitignore 302 | # Local .terraform directories 303 | **/.terraform/* 304 | 305 | # .tfstate files 306 | *.tfstate 307 | *.tfstate.* 308 | 309 | # Crash log files 310 | crash.log 311 | crash.*.log 312 | 313 | # Exclude all .tfvars files, which are likely to contain sensitive data, such as 314 | # password, private keys, and other secrets. These should not be part of version 315 | # control as they are data points which are potentially sensitive and subject 316 | # to change depending on the environment. 317 | *.tfvars 318 | *.tfvars.json 319 | 320 | # Ignore override files as they are usually used to override resources locally and so 321 | # are not checked in 322 | override.tf 323 | override.tf.json 324 | *_override.tf 325 | *_override.tf.json 326 | 327 | # Include override files you do wish to add to version control using negated pattern 328 | # !example_override.tf 329 | 330 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 331 | # example: *tfplan* 332 | 333 | # Ignore CLI configuration files 334 | .terraformrc 335 | terraform.rc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Coding-with-Kafka 2 | 3 | ## Getting Started 4 | 5 | If you're going through the course, do the following to start where we do: 6 | ```bash 7 | mkdir -p ~/dev/coding-with-kakfa 8 | cd ~/dev/coding-with-kakfa 9 | git clone https://github.com/codingforentrepreneurs/Coding-with-Kafka . 10 | git checkout start 11 | rm -rf .git 12 | git init 13 | git commit -m "Course started!" 14 | ``` 15 | 16 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | ansible_python_interpreter='/usr/bin/python3' 3 | deprecation_warnings=False 4 | inventory=./inventory.ini 5 | remote_user="root" 6 | retries=2 7 | host_key_checking = False 8 | remote_tmp="/tmp/ansible" 9 | roles_path = ./roles -------------------------------------------------------------------------------- /coding-with-kafka.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | } 6 | ], 7 | "settings": {} 8 | } -------------------------------------------------------------------------------- /commands.md: -------------------------------------------------------------------------------- 1 | # Reference Commands 2 | 3 | 4 | ## Zookeeper in Docker Compose 5 | 6 | __Enter Container__ 7 | ```bash 8 | docker compose exec -it zookeeper /bin/bash 9 | ``` 10 | > Assumes that `zookeeper` is the service/hostname in `compose.yaml` 11 | 12 | __Zookeeper Shell__ 13 | ``` 14 | /bin/zookeeper-shell localhost:2181 15 | ``` 16 | 17 | __Zookeeper Shell via Docker Compose__ 18 | ```bash 19 | docker compose exec -it zookeeper /bin/zookeeper-shell localhost:2181 20 | ``` 21 | 22 | __Zookeeper Instance, are you okay? (`ruok`)__ 23 | 24 | ```bash 25 | echo "ruok" | nc localhost 2181 26 | ``` 27 | or 28 | ```bash 29 | echo "ruok" | nc zookeeper 2181 30 | ``` 31 | 32 | __Zookeeper stats (`stat`)__ 33 | ```bash 34 | echo "stat" | nc localhost 2181 35 | ``` 36 | 37 | __Zookeeper configuration (`conf`)__ 38 | ```bash 39 | echo "conf" | nc localhost 2181 40 | ``` 41 | 42 | 43 | ## Kafka in Docker Compose 44 | 45 | __Enter Container__ 46 | ```bash 47 | docker compose exec -it kafka-1 /bin/bash 48 | ``` 49 | 50 | __View Available Kafka CLIs__ 51 | ``` 52 | ls /bin | grep "kafka" 53 | ``` 54 | 55 | Key ones: 56 | - `kafka-topics`: to create/delete topics 57 | - `kafka-console-producer`: to create messages for a topic 58 | - `kafka-console-consumer`: to view messages for a topic 59 | 60 | 61 | __Create a topic__ 62 | ```bash 63 | /bin/kafka-topics --bootstrap-server localhost:9092 --create --topic hello-world --partitions 4 --replication-factor 3 64 | ``` 65 | 66 | ```python 67 | # Understanding Partitions and Replication Factor 68 | a = 'some_message' 69 | b = 'some_other message' 70 | c = 'yet another one' 71 | d = 'new one' 72 | e = 'newer one' 73 | f = 'newest one' 74 | 75 | my_topic = [a, b, c] 76 | 77 | my_topic = [ 78 | [a, e], # kafka-1 79 | [b, d], # kafka-2 80 | [c, f] # kafka-3 81 | ] 82 | 83 | my_topic_copy = [ 84 | [a, e], # kafka-2 85 | [b, d], # kafka-1 86 | [c, f] # kafka-3 87 | ] 88 | 89 | my_topic = [a, b,c,f,e,d] 90 | ``` 91 | 92 | 93 | __Interactive shell for creating messages for an existing topic__ 94 | 95 | ```bash 96 | /bin/kafka-console-producer --bootstrap-server localhost:9092 --topic hello-world 97 | ``` 98 | 99 | __Listen for a topic's messages__ 100 | - Listen to messages for topic 101 | ```bash 102 | /bin/kafka-console-consumer --bootstrap-server localhost:9092 --topic hello-world --from-beginning 103 | ``` 104 | 105 | 106 | __Create a topic and related messages__ 107 | ```bash 108 | /bin/kafka-topics --bootstrap-server=localhost:9092 --delete --if-exists --topic my-topic 109 | ``` 110 | 111 | 112 | ## Zookeeper on Virtual Machine 113 | 114 | Assuming that kafka is avialable on `/opt/kafka`, you can use the following: 115 | 116 | 117 | __Start with the default config__ 118 | ```bash 119 | /opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties 120 | ``` 121 | 122 | __Start with the custom config__ 123 | 124 | ```bash 125 | /opt/kafka/bin/zookeeper-server-start.sh /data/my-config/zookeeper.properties 126 | ``` 127 | `/data/my-config/zookeeper.properties` is essentially the same as the default config with some minor changes (including the Zookeeper servers in the quorom). 128 | 129 | __Stop with the default config__ 130 | ```bash 131 | /opt/kafka/bin/zookeeper-server-stop.sh 132 | ``` 133 | 134 | __Zookeeper Shell when Zookeeper is running/available__ 135 | ```bash 136 | /opt/kafka/bin/zookeeper-shell.sh localhost:2181 137 | ``` 138 | 139 | 140 | ## Kakfa on Virtual Machine 141 | 142 | Assuming that kafka is avialable on `/opt/kafka` 143 | 144 | 145 | __Start the default config__ 146 | ```bash 147 | /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties 148 | ``` 149 | 150 | __Start the custom config__ 151 | ```bash 152 | /opt/kafka/bin/kafka-server-start.sh /data/my-config/server.properties 153 | ``` 154 | 155 | __Create a topic__ 156 | ```bash 157 | /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka1:9092 --create --topic hello-world --partitions 1 --replication-factor 1 158 | ``` 159 | 160 | __Interactive shell for creating messages for an existing topic__ 161 | 162 | ```bash 163 | /opt/kafka/bin/kafka-console-producer.sh --bootstrap-server kafka1:9092 --topic hello-world 164 | ``` 165 | 166 | __Listen for a topic's messages__ 167 | - Listen to messages for topic 168 | ```bash 169 | /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka1:9092 --topic hello-world --from-beginning 170 | ``` 171 | 172 | 173 | __Delete a topic__ 174 | ```bash 175 | /opt/kafka/bin/kafka-topics.sh --bootstrap-server=kafka1:9092 --delete --if-exists --topic hello-world 176 | ``` 177 | 178 | __List topics__ 179 | ```bash 180 | /opt/kafka/bin/kafka-topics.sh --bootstrap-server=kafka1:9092 --list 181 | ``` 182 | 183 | 184 | ## Systemd Commands 185 | 186 | __Create a new service__ 187 | ```bash 188 | nano /etc/systemd/system/my-service.service 189 | ``` 190 | 191 | Use the format 192 | ``` 193 | [Unit] 194 | Description=My service server 195 | After=network.target 196 | 197 | [Service] 198 | Type=simple 199 | User=tars 200 | ExecStart=/path/to/executable /path/to/config 201 | WorkingDirectory=/path/to/working/dir 202 | Restart=on-failure 203 | RestartSec=10s 204 | StandardOutput=file:/var/log/my-service/my-service.out 205 | StandardError=file:/var/log/my-service/my-service.err 206 | LimitNOFILE=800000 207 | Environment=PATH=/usr/bin:/bin:/usr/local/bin 208 | 209 | [Install] 210 | WantedBy=multi-user.target 211 | ``` 212 | 213 | 214 | 215 | __Reload the daemon__ 216 | ```bash 217 | sudo systemctl daemon-reload 218 | ``` 219 | 220 | __Start the service__ 221 | ```bash 222 | sudo systemctl start zookeeper 223 | ``` 224 | In place of `start`: 225 | - Use `stop` to stop the service 226 | - Use `restart` to restart the service 227 | - Use `status` to get the systemd status of the service 228 | 229 | 230 | __Journal logs about the service__ 231 | ```bash 232 | journalctl -u zookeeper.service 233 | ``` 234 | -------------------------------------------------------------------------------- /compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | zookeeper: # hostname -> zookeeper:2181 3 | image: confluentinc/cp-zookeeper:latest 4 | environment: 5 | # clientPort 6 | ZOOKEEPER_CLIENT_PORT: 2181 # default 7 | # server id 8 | ZOOKEEPER_SERVER_ID: 1 # max 3 9 | # zookeepers servers 10 | ZOOKEEPER_SERVERS: zookeeper:2181:2888;zookeeper2:2182:2889;zookeeper3:2183:2890 11 | # ruok stat 12 | KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=*" 13 | ports: 14 | - "2181:2181" 15 | - "2888:2888" 16 | zookeeper2: # hostname -> zookeeper2:2181 17 | image: confluentinc/cp-zookeeper:latest 18 | environment: 19 | # clientPort 20 | ZOOKEEPER_CLIENT_PORT: 2181 # default 21 | # server id 22 | ZOOKEEPER_SERVER_ID: 2 # max 3 23 | # zookeepers servers 24 | ZOOKEEPER_SERVERS: zookeeper:2181:2888;zookeeper2:2182:2889;zookeeper3:2183:2890 25 | # ruok stat 26 | KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=*" 27 | ports: 28 | - "2182:2181" 29 | - "2889:2888" 30 | zookeeper3: # hostname -> zookeeper3:2181 31 | image: confluentinc/cp-zookeeper:latest 32 | environment: 33 | # clientPort 34 | ZOOKEEPER_CLIENT_PORT: 2181 # default 35 | # server id 36 | ZOOKEEPER_SERVER_ID: 3 # max 3 37 | # zookeepers servers 38 | ZOOKEEPER_SERVERS: zookeeper:2181:2888;zookeeper2:2182:2889;zookeeper3:2183:2890 39 | # ruok stat 40 | KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=*" 41 | ports: 42 | - "2183:2181" 43 | - "2890:2888" 44 | kafka-1: # hostname -> kafka-1:9092 45 | image: confluentinc/cp-kafka:latest 46 | restart: on-failure # always 47 | environment: 48 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:9092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:19092,DOCKER://host.docker.internal:29092 49 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT 50 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL 51 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181,zookeeper2:2181,zookeeper3:2181" 52 | KAFKA_BROKER_ID: 1 53 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 54 | ports: 55 | - "19092:19092" # 127.0.0.1:19092 56 | - "29092:29092" 57 | depends_on: 58 | - zookeeper 59 | - zookeeper2 60 | - zookeeper3 61 | kafka-2: # hostname -> kafka-2:9092 62 | image: confluentinc/cp-kafka:latest 63 | restart: on-failure # always 64 | environment: 65 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:9092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:19093,DOCKER://host.docker.internal:29093 66 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT 67 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL 68 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181,zookeeper2:2181,zookeeper3:2181" 69 | KAFKA_BROKER_ID: 2 70 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 71 | ports: 72 | - "19093:19093" # 127.0.0.1:19092 73 | - "29093:29093" 74 | depends_on: 75 | - zookeeper 76 | - zookeeper2 77 | - zookeeper3 78 | kafka-3: # hostname -> kafka-3:9092 79 | image: confluentinc/cp-kafka:latest 80 | restart: on-failure # always 81 | environment: 82 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:9092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:19094,DOCKER://host.docker.internal:29094 83 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT 84 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL 85 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181,zookeeper2:2181,zookeeper3:2181" 86 | KAFKA_BROKER_ID: 3 87 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 88 | ports: 89 | - "19094:19094" # 127.0.0.1:19092 90 | - "29094:29094" 91 | depends_on: 92 | - zookeeper 93 | - zookeeper2 94 | - zookeeper3 95 | vm: 96 | build: 97 | context: ./vm 98 | dockerfile: Dockerfile 99 | stdin_open: true 100 | tty: true -------------------------------------------------------------------------------- /config/install-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Create user "tars" 4 | sudo useradd -r -s /sbin/nologin tars 5 | sudo usermod -aG sudo tars 6 | 7 | echo "tars ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tars 8 | 9 | 10 | # Define all required directories 11 | directories=( 12 | /data/my-config 13 | /var/log/zookeeper 14 | /var/log/kafka 15 | /opt/kafka 16 | /tmp/zookeeper 17 | /data/zookeeper 18 | /data/kafka 19 | ) 20 | 21 | # Loop through each directory 22 | for dir in "${directories[@]}"; do 23 | # Create the directory with sudo, avoiding errors if it already exists 24 | sudo mkdir -p "$dir" 25 | 26 | # Change the ownership to 'tars' user and group, recursively 27 | sudo chown -R tars:tars "$dir" 28 | done 29 | 30 | # Install Java and Required packages 31 | sudo apt-get update && sudo apt-get -y install wget ca-certificates zip net-tools vim nano tar netcat openjdk-8-jdk 32 | # Verifying versions 33 | java -version 34 | 35 | # Add file limits configs - allow to open 100,000 file descriptors 36 | echo "* hard nofile 100000 37 | * soft nofile 100000" | sudo tee --append /etc/security/limits.conf 38 | 39 | # update memory swap 40 | sudo sysctl vm.swappiness=1 41 | echo 'vm.swappiness=1' | sudo tee --append /etc/sysctl.conf 42 | 43 | # Copy contents of the terraform hostsfile and append to /etc/hosts 44 | # nano /etc/hosts 45 | cat /etc/hosts 46 | 47 | 48 | # Download Kafka (including Zookeeper) from 49 | # https://kafka.apache.org/downloads 50 | curl https://dlcdn.apache.org/kafka/3.7.0/kafka_2.13-3.7.0.tgz -o kafka.tgz 51 | tar -xvzf kafka.tgz 52 | mv kafka_*/* /opt/kafka/ 53 | rm kafka.tgz 54 | 55 | ls /opt/kafka/bin 56 | 57 | 58 | # verify: 59 | cat /opt/kafka/config/server.properties 60 | 61 | # copy 62 | cp /opt/kafka/config/server.properties /data/my-config/ 63 | # Run 64 | # /opt/kafka/bin/kafka-server-start.sh /data/my-config/server.properties -------------------------------------------------------------------------------- /config/install-zookeeper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Create user "tars" 4 | sudo useradd -r -s /sbin/nologin tars 5 | sudo usermod -aG sudo tars 6 | 7 | echo "tars ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tars 8 | 9 | 10 | # Define all required directories 11 | directories=( 12 | /data/my-config 13 | /var/log/zookeeper 14 | /var/log/kafka 15 | /opt/kafka 16 | /tmp/zookeeper 17 | /data/zookeeper 18 | /data/kafka 19 | ) 20 | 21 | # Loop through each directory 22 | for dir in "${directories[@]}"; do 23 | # Create the directory with sudo, avoiding errors if it already exists 24 | sudo mkdir -p "$dir" 25 | 26 | # Change the ownership to 'tars' user and group, recursively 27 | sudo chown -R tars:tars "$dir" 28 | done 29 | 30 | 31 | # Set the Zookeeper Instance ID 32 | # uncomment during manual use 33 | # sudo nano /data/zookeeper/myid 34 | 35 | # Extract the Zookeeper Instance ID from 36 | # the hostname 37 | hostname=$(hostname) 38 | host_id=$(echo $hostname | grep -o '[0-9]*' | sed 's/^0*//') 39 | echo "$host_id" | sudo tee /data/zookeeper/myid 40 | cat /data/zookeeper/myid 41 | 42 | 43 | # Install Java and Required packages 44 | sudo apt-get update && sudo apt-get -y install wget ca-certificates zip net-tools vim nano tar netcat openjdk-8-jdk 45 | # Verifying versions 46 | java -version 47 | 48 | # update memory swap 49 | # to leverage as little SSD as possible 50 | sudo sysctl vm.swappiness=1 51 | echo 'vm.swappiness=1' | sudo tee --append /etc/sysctl.conf 52 | 53 | # Copy contents of the terraform hostsfile and append to /etc/hosts 54 | # nano /etc/hosts 55 | cat /etc/hosts 56 | 57 | 58 | # Download Kafka (including Zookeeper) from 59 | # https://kafka.apache.org/downloads 60 | curl https://dlcdn.apache.org/kafka/3.7.0/kafka_2.13-3.7.0.tgz -o kafka.tgz 61 | tar -xvzf kafka.tgz 62 | mv kafka_*/* /opt/kafka/ 63 | rm kafka.tgz 64 | 65 | ls /opt/kafka/bin | grep "zookeeper" 66 | 67 | 68 | 69 | # verify: 70 | cat /opt/kafka/config/zookeeper.properties 71 | 72 | # Run 73 | # /opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties 74 | # /opt/kafka/bin/zookeeper-server-start.sh /data/my-config/zookeeper.properties 75 | 76 | # In new terminal, enter the shell 77 | # /opt/kafka/bin/zookeeper-shell.sh localhost:2181 -------------------------------------------------------------------------------- /config/kafka.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=kafka server 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | User=tars 8 | ExecStart=/opt/kafka/bin/kafka-server-start.sh /data/my-config/server.properties 9 | WorkingDirectory=/opt/kafka 10 | Restart=on-failure 11 | RestartSec=10s 12 | StandardOutput=file:/var/log/kafka/kafka.out 13 | StandardError=file:/var/log/kafka/kafka.err 14 | LimitNOFILE=800000 15 | Environment=PATH=/usr/bin:/bin:/usr/local/bin 16 | 17 | [Install] 18 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /config/my-server.properties: -------------------------------------------------------------------------------- 1 | # The id of the broker. This must be set to a unique integer for each broker. 2 | broker.id=1 3 | 4 | ############################# Socket Server Settings ############################# 5 | # The address the socket server listens on. If not configured, the host name will be equal to the value of 6 | # java.net.InetAddress.getCanonicalHostName(), with PLAINTEXT listener name, and port 9092. 7 | # FORMAT: 8 | # listeners = listener_name://host_name:port 9 | # EXAMPLE: 10 | # listeners = PLAINTEXT://your.host.name:9092 11 | listeners=INTERNAL://kafka1:9092,EXTERNAL://172.234.228.67:19092 12 | 13 | # Listener name, hostname and port the broker will advertise to clients. 14 | # If not set, it uses the value for "listeners". 15 | advertised.listeners=INTERNAL://kafka1:9092,EXTERNAL://172.234.228.67:19092 16 | 17 | # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details 18 | listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT 19 | inter.broker.listener.name=INTERNAL 20 | 21 | # A comma separated list of directories under which to store log files 22 | log.dirs=/data/kafka 23 | 24 | # The default number of log partitions per topic. More partitions allow greater 25 | # parallelism for consumption, but this will also result in more files across 26 | # the brokers. 27 | num.partitions=1 28 | 29 | ############################# Internal Topic Settings ############################# 30 | # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" 31 | # For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. 32 | offsets.topic.replication.factor=1 33 | transaction.state.log.replication.factor=1 34 | transaction.state.log.min.isr=1 35 | 36 | # Zookeeper connection string (see zookeeper docs for details). 37 | # This is a comma separated host:port pairs, each corresponding to a zk 38 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 39 | # You can also append an optional chroot string to the urls to specify the 40 | # root directory for all kafka znodes. 41 | zookeeper.connect=zookeeper1:2181 42 | -------------------------------------------------------------------------------- /config/my-zookeeper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # the directory where the snapshot is stored. 16 | dataDir=/tmp/zookeeper 17 | # the port at which the clients will connect 18 | clientPort=2181 19 | # disable the per-ip limit on the number of connections since this is a non-production config 20 | maxClientCnxns=0 21 | # Disable the adminserver by default to avoid port conflicts. 22 | # Set the port to something non-conflicting if choosing to enable this 23 | admin.enableServer=false 24 | # admin.serverPort=8080 25 | 26 | # 4lw 27 | 4lw.commands.whitelist=stat,ruok,conf,isro,dump 28 | 29 | # the servers in the ensemble 30 | server.0=zookeeper1:2888:3888 31 | # server.1=zookeeper2:2888:3888 32 | # server.2=zookeeper3:2888:3888 -------------------------------------------------------------------------------- /config/zookeeper.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Zookeeper server 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | User=tars 8 | ExecStart=/opt/kafka/bin/zookeeper-server-start.sh /data/my-config/zookeeper.properties 9 | WorkingDirectory=/opt/kafka 10 | Restart=on-failure 11 | RestartSec=10s 12 | StandardOutput=file:/var/log/zookeeper/zookeeper.out 13 | StandardError=file:/var/log/zookeeper/zookeeper.err 14 | LimitNOFILE=800000 15 | Environment=PATH=/usr/bin:/bin:/usr/local/bin 16 | 17 | [Install] 18 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /devops/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/local" { 5 | version = "2.5.1" 6 | hashes = [ 7 | "h1:/GAVA/xheGQcbOZEq0qxANOg+KVLCA7Wv8qluxhTjhU=", 8 | "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561", 9 | "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561", 10 | "zh:196bfb49086f22fd4db46033e01655b0e5e036a5582d250412cc690fa7995de5", 11 | "zh:37c92ec084d059d37d6cffdb683ccf68e3a5f8d2eb69dd73c8e43ad003ef8d24", 12 | "zh:4269f01a98513651ad66763c16b268f4c2da76cc892ccfd54b401fff6cc11667", 13 | "zh:51904350b9c728f963eef0c28f1d43e73d010333133eb7f30999a8fb6a0cc3d8", 14 | "zh:73a66611359b83d0c3fcba2984610273f7954002febb8a57242bbb86d967b635", 15 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 16 | "zh:7ae387993a92bcc379063229b3cce8af7eaf082dd9306598fcd42352994d2de0", 17 | "zh:9e0f365f807b088646db6e4a8d4b188129d9ebdbcf2568c8ab33bddd1b82c867", 18 | "zh:b5263acbd8ae51c9cbffa79743fbcadcb7908057c87eb22fd9048268056efbc4", 19 | "zh:dfcd88ac5f13c0d04e24be00b686d069b4879cc4add1b7b1a8ae545783d97520", 20 | ] 21 | } 22 | 23 | provider "registry.terraform.io/linode/linode" { 24 | version = "2.17.0" 25 | hashes = [ 26 | "h1:eUQyU+FcZC4Ev3AZP5h16SQTEA9JieRL3Zn0ArCLdyI=", 27 | "zh:15e4a37e3e1e4bea8137a5ea2131c39b7e92f6ce4ebf5abfaacf8065bc2974d6", 28 | "zh:181de2fd5086e516d60916f37ba243fc1f6f01ab7bde4339a98bf6f5b2363459", 29 | "zh:1b207ef911419b36d7ccbcf5bc7ef64e06f3b22257031a5da5f99619122523a9", 30 | "zh:494d88961c315d74837e603eb5f49fb4dd94830be6933a5ca3e88020154d1069", 31 | "zh:67ce3c880d8278c8303ebcc6c0781aa5ca11d7cee1ae573a0768a78f59f0d44e", 32 | "zh:817cc942ab2dd49c8ae6ce973d1d2bdaff9693782959290eb029fd70d952f3f4", 33 | "zh:ad0da4874b8106ea3ad08ea623bd46d5af2f098d0040135c239db1362cd243c9", 34 | "zh:b21278a41f9c3103b0b8fd9224f860391ea484d70f757fa8fd16f3dd819de318", 35 | "zh:dd8c0d2f1413316c5e95afc6f9d803dc2ae576d3819f895138f3769effac4316", 36 | "zh:e5d49de8f6723afe601450adde26591ed4a5d24aedef2d87788042cb612cfc43", 37 | "zh:eeb8a08069069da6b5dc9d1d43a10231881c3f05ec95d957952f5217513464c9", 38 | "zh:efa7f3a0ec2befc5623d7b9588f11faead8a7e647cefe70af0139e8dd67a7537", 39 | "zh:f3fe82efa9170c6683ebcd6880751492326ffa023c0a2725e72330aaaf1a8584", 40 | "zh:fd3b9d1407d30cff73e0169374d65cd544146dd3bb83ee86bcfef2ab8b5929f7", 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /devops/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | linode = { 4 | source = "linode/linode" 5 | } 6 | } 7 | } 8 | 9 | provider "linode" { 10 | token = var.linode_api_token 11 | } 12 | 13 | resource "linode_instance" "zookeeper-instance" { 14 | label = "zookeeper${count.index + 1}" # unique in linode 15 | count = var.zookeeper_server_count 16 | image = "linode/ubuntu22.04" 17 | region = "us-sea" 18 | type = var.server_type 19 | authorized_keys = [trimspace(file(var.ssh_public_key_path))] 20 | private_ip = true 21 | 22 | provisioner "remote-exec" { 23 | connection { 24 | host = "${self.ip_address}" 25 | type = "ssh" 26 | user = "root" 27 | private_key = "${file(var.ssh_private_key_path)}" 28 | } 29 | inline = [ 30 | "hostnamectl set-hostname zookeeper${count.index + 1}", 31 | ] 32 | } 33 | 34 | } 35 | 36 | resource "linode_instance" "kafka-instance" { 37 | label = "kafka${count.index + 1}" 38 | count = var.kafka_server_count 39 | image = "linode/ubuntu22.04" 40 | region = "us-sea" 41 | type = var.server_type 42 | authorized_keys = [trimspace(file(var.ssh_public_key_path))] 43 | private_ip = true 44 | 45 | provisioner "remote-exec" { 46 | connection { 47 | host = "${self.ip_address}" 48 | type = "ssh" 49 | user = "root" 50 | private_key = "${file(var.ssh_private_key_path)}" 51 | } 52 | inline = [ 53 | "hostnamectl set-hostname kafka${count.index + 1}", 54 | ] 55 | } 56 | } 57 | 58 | 59 | locals { 60 | project_root_dir = "${dirname(abspath(path.root))}" 61 | root_dir = "${abspath(path.root)}" 62 | templates_dir = "${local.root_dir}/templates" 63 | kafka_instances = [ for host in linode_instance.kafka-instance.* : { 64 | ip_address: host.ip_address 65 | label: host.label 66 | hostname: host.label 67 | private_ip: host.private_ip_address 68 | }] 69 | zookeeper_instances = [ for host in linode_instance.zookeeper-instance.* : { 70 | ip_address: host.ip_address 71 | label: host.label 72 | hostname: host.label 73 | private_ip: host.private_ip_address 74 | }] 75 | } 76 | 77 | 78 | resource "local_file" "zookeeper_kafka_hostsfile" { 79 | content = templatefile("${local.templates_dir}/hostsfile.tftpl", { 80 | kafka_instances=local.kafka_instances, 81 | zookeeper_instances=local.zookeeper_instances 82 | }) 83 | filename = "${local.project_root_dir}/host-config.txt" 84 | } 85 | 86 | 87 | resource "local_file" "ansible_inventory" { 88 | content = templatefile("${local.templates_dir}/ansible-inventory.tftpl", { 89 | kafka_instances=local.kafka_instances, 90 | zookeeper_instances=local.zookeeper_instances 91 | }) 92 | filename = "${local.project_root_dir}/inventory.ini" 93 | } 94 | 95 | resource "local_file" "cluster_brokers_env_file" { 96 | content = templatefile("${local.templates_dir}/cluster.env.tftpl", { 97 | kafka_instances=local.kafka_instances, 98 | zookeeper_instances=local.zookeeper_instances 99 | }) 100 | filename = "${local.project_root_dir}/cluster.env" 101 | } -------------------------------------------------------------------------------- /devops/templates/ansible-inventory.tftpl: -------------------------------------------------------------------------------- 1 | [zookeeper_servers] 2 | %{ for index, instance in zookeeper_instances ~} 3 | ${instance.hostname} ansible_host=${instance.ip_address} private_ip=${instance.private_ip} node_id=${index+1} 4 | %{ endfor ~} 5 | 6 | 7 | [kafka_servers] 8 | %{ for index, instance in kafka_instances ~} 9 | ${instance.hostname} ansible_host=${instance.ip_address} private_ip=${instance.private_ip} node_id=${index+1} 10 | %{ endfor ~} -------------------------------------------------------------------------------- /devops/templates/cluster.env.tftpl: -------------------------------------------------------------------------------- 1 | %{ for index, instance in kafka_instances ~} 2 | KAFKA_BROKER_${index+1}=${instance.ip_address}:19092 3 | %{ endfor ~} -------------------------------------------------------------------------------- /devops/templates/hostsfile.tftpl: -------------------------------------------------------------------------------- 1 | %{ for index, instance in zookeeper_instances ~} 2 | ${instance.private_ip} ${instance.hostname} 3 | %{ endfor ~} 4 | %{ for index, instance in kafka_instances ~} 5 | ${instance.private_ip} ${instance.hostname} 6 | %{ endfor ~} -------------------------------------------------------------------------------- /devops/variables.tf: -------------------------------------------------------------------------------- 1 | variable "linode_api_token" { 2 | sensitive = true 3 | } 4 | 5 | variable "server_type" { 6 | default = "g6-standard-2" 7 | type = string 8 | } 9 | 10 | variable "zookeeper_server_count" { 11 | default = 3 12 | type = number 13 | } 14 | 15 | variable "kafka_server_count" { 16 | default = 5 17 | type = number 18 | } 19 | 20 | variable "ssh_public_key_path" { 21 | description = "SSH public key path to use for Linode instances" 22 | default = "~/.ssh/id_rsa.pub" 23 | type = string 24 | sensitive = false 25 | } 26 | 27 | variable "ssh_private_key_path" { 28 | description = "Path to the SSH public key" 29 | default = "~/.ssh/id_rsa" 30 | sensitive = true 31 | } -------------------------------------------------------------------------------- /guides/python-setup.md: -------------------------------------------------------------------------------- 1 | # Python Setup 2 | This is a simple guide to setup Python on your machine for this course. 3 | 4 | Using VSCode can make using Python much easier as it supports automatically activating virtual environments. 5 | 6 | ## Download Python 7 | Go to [python.org](https://python.org) and download the latest version for your machine. 8 | 9 | 10 | ## Virtual Environments 11 | 12 | ### Windows 13 | In-depth guide [here](https://www.codingforentrepreneurs.com/guides/install-python-on-windows/) 14 | 15 | ```powershell 16 | cd path\to\your\project 17 | C:\Python310\python.exe -m venv venv 18 | .\venv\Scripts\activate 19 | ``` 20 | 21 | Once activated you can run: 22 | 23 | ```powershell 24 | python -m pip install pip --upgrade 25 | ``` 26 | or 27 | ```powershell 28 | pip install pip --upgrade 29 | ``` 30 | 31 | 32 | ### macOS 33 | 34 | In-depth guide [here](https://www.codingforentrepreneurs.com/guides/install-python-on-macos/) 35 | 36 | 37 | ```bash 38 | cd path/to/your/project 39 | python3 -m venv venv 40 | source venv/bin/activate 41 | ``` 42 | 43 | Once activated you can run: 44 | 45 | ```bash 46 | python -m pip install pip --upgrade 47 | ``` 48 | or 49 | ```bash 50 | pip install pip --upgrade 51 | ``` -------------------------------------------------------------------------------- /guides/terraform.md: -------------------------------------------------------------------------------- 1 | # Terraform Setup & Common Commands 2 | 3 | 4 | ## Install Terraform 5 | 6 | Download and install from [the docs](https://developer.hashicorp.com/terraform/install) for your machine. 7 | 8 | Open terminal/powershell and verify with: 9 | 10 | ```bash 11 | terraform --version 12 | ``` 13 | 14 | ## Files to always ignore in Git 15 | 16 | - `*.tfvars` 17 | - `*.tfstate` 18 | 19 | Others listed in the [Github Terraform Gitignore](https://raw.githubusercontent.com/github/gitignore/main/Terraform.gitignore) 20 | 21 | 22 | ## Commands for Terraform 23 | 24 | ### `terraform init` 25 | 26 | After you declare `main.tf`, run the following next to `main.tf` 27 | ``` 28 | terraform init 29 | ``` 30 | > Tip, if you are one directory up and the Terraform module is in `devops/main.tf`, you can run `terraform -chdir=devops init` 31 | 32 | You will run `terraform init` anytime you add a new provider. 33 | 34 | ### `terraform plan` 35 | Anytime you want to review changes you have made with terraform, run: 36 | 37 | ```bash 38 | terraform plan 39 | ``` 40 | > Once again, if you are one directory up and the Terraform module is in `devops/main.tf`, you can run `terraform -chdir=devops plan` 41 | 42 | 43 | ### `terraform apply` 44 | Once you are ready to apply changes you can run: 45 | 46 | ```bash 47 | terraform apply 48 | ``` 49 | 50 | A few other items about this: 51 | - `terraform apply` will require your input on if you are ready to make changes 52 | - `terraform apply -auto-approve` will automatically approve changes and do them 53 | - `terraform apply -auto-approve -destroy` will automatically destroy all changes 54 | 55 | 56 | ### `terraform destroy` 57 | This is how you can take down everything you provisioned with Terraform: 58 | 59 | ```bash 60 | terraform destroy 61 | ``` 62 | This command is actually a shortcut to `terraform apply -destroy` and requires using input. 63 | 64 | 65 | ### `terraform console` 66 | Sometimes you can use terraform console to better understand what is going on in your infrastructure. 67 | 68 | ``` 69 | terraform console 70 | ``` 71 | When you're done, you can exit with `Ctrl+C` or typing `exit` and hitting enter. -------------------------------------------------------------------------------- /main.yaml: -------------------------------------------------------------------------------- 1 | - name: Configure Zookeeper 2 | import_playbook: playbooks/zookeeper.yaml 3 | 4 | - name: Configure Kafka 5 | import_playbook: playbooks/kafka.yaml 6 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "coding-with-kafka", 3 | "lockfileVersion": 3, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "dependencies": { 8 | "kafkajs": "^2.2.4" 9 | }, 10 | "devDependencies": { 11 | "tsx": "^4.7.1" 12 | } 13 | }, 14 | "node_modules/@esbuild/aix-ppc64": { 15 | "version": "0.19.12", 16 | "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.19.12.tgz", 17 | "integrity": "sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==", 18 | "cpu": [ 19 | "ppc64" 20 | ], 21 | "dev": true, 22 | "optional": true, 23 | "os": [ 24 | "aix" 25 | ], 26 | "engines": { 27 | "node": ">=12" 28 | } 29 | }, 30 | "node_modules/@esbuild/android-arm": { 31 | "version": "0.19.12", 32 | "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.19.12.tgz", 33 | "integrity": "sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w==", 34 | "cpu": [ 35 | "arm" 36 | ], 37 | "dev": true, 38 | "optional": true, 39 | "os": [ 40 | "android" 41 | ], 42 | "engines": { 43 | "node": ">=12" 44 | } 45 | }, 46 | "node_modules/@esbuild/android-arm64": { 47 | "version": "0.19.12", 48 | "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.19.12.tgz", 49 | "integrity": "sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA==", 50 | "cpu": [ 51 | "arm64" 52 | ], 53 | "dev": true, 54 | "optional": true, 55 | "os": [ 56 | "android" 57 | ], 58 | "engines": { 59 | "node": ">=12" 60 | } 61 | }, 62 | "node_modules/@esbuild/android-x64": { 63 | "version": "0.19.12", 64 | "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.19.12.tgz", 65 | "integrity": "sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew==", 66 | "cpu": [ 67 | "x64" 68 | ], 69 | "dev": true, 70 | "optional": true, 71 | "os": [ 72 | "android" 73 | ], 74 | "engines": { 75 | "node": ">=12" 76 | } 77 | }, 78 | "node_modules/@esbuild/darwin-arm64": { 79 | "version": "0.19.12", 80 | "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.19.12.tgz", 81 | "integrity": "sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g==", 82 | "cpu": [ 83 | "arm64" 84 | ], 85 | "dev": true, 86 | "optional": true, 87 | "os": [ 88 | "darwin" 89 | ], 90 | "engines": { 91 | "node": ">=12" 92 | } 93 | }, 94 | "node_modules/@esbuild/darwin-x64": { 95 | "version": "0.19.12", 96 | "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.19.12.tgz", 97 | "integrity": "sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A==", 98 | "cpu": [ 99 | "x64" 100 | ], 101 | "dev": true, 102 | "optional": true, 103 | "os": [ 104 | "darwin" 105 | ], 106 | "engines": { 107 | "node": ">=12" 108 | } 109 | }, 110 | "node_modules/@esbuild/freebsd-arm64": { 111 | "version": "0.19.12", 112 | "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.12.tgz", 113 | "integrity": "sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA==", 114 | "cpu": [ 115 | "arm64" 116 | ], 117 | "dev": true, 118 | "optional": true, 119 | "os": [ 120 | "freebsd" 121 | ], 122 | "engines": { 123 | "node": ">=12" 124 | } 125 | }, 126 | "node_modules/@esbuild/freebsd-x64": { 127 | "version": "0.19.12", 128 | "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.19.12.tgz", 129 | "integrity": "sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg==", 130 | "cpu": [ 131 | "x64" 132 | ], 133 | "dev": true, 134 | "optional": true, 135 | "os": [ 136 | "freebsd" 137 | ], 138 | "engines": { 139 | "node": ">=12" 140 | } 141 | }, 142 | "node_modules/@esbuild/linux-arm": { 143 | "version": "0.19.12", 144 | "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.19.12.tgz", 145 | "integrity": "sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w==", 146 | "cpu": [ 147 | "arm" 148 | ], 149 | "dev": true, 150 | "optional": true, 151 | "os": [ 152 | "linux" 153 | ], 154 | "engines": { 155 | "node": ">=12" 156 | } 157 | }, 158 | "node_modules/@esbuild/linux-arm64": { 159 | "version": "0.19.12", 160 | "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.19.12.tgz", 161 | "integrity": "sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA==", 162 | "cpu": [ 163 | "arm64" 164 | ], 165 | "dev": true, 166 | "optional": true, 167 | "os": [ 168 | "linux" 169 | ], 170 | "engines": { 171 | "node": ">=12" 172 | } 173 | }, 174 | "node_modules/@esbuild/linux-ia32": { 175 | "version": "0.19.12", 176 | "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.19.12.tgz", 177 | "integrity": "sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA==", 178 | "cpu": [ 179 | "ia32" 180 | ], 181 | "dev": true, 182 | "optional": true, 183 | "os": [ 184 | "linux" 185 | ], 186 | "engines": { 187 | "node": ">=12" 188 | } 189 | }, 190 | "node_modules/@esbuild/linux-loong64": { 191 | "version": "0.19.12", 192 | "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.19.12.tgz", 193 | "integrity": "sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA==", 194 | "cpu": [ 195 | "loong64" 196 | ], 197 | "dev": true, 198 | "optional": true, 199 | "os": [ 200 | "linux" 201 | ], 202 | "engines": { 203 | "node": ">=12" 204 | } 205 | }, 206 | "node_modules/@esbuild/linux-mips64el": { 207 | "version": "0.19.12", 208 | "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.19.12.tgz", 209 | "integrity": "sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w==", 210 | "cpu": [ 211 | "mips64el" 212 | ], 213 | "dev": true, 214 | "optional": true, 215 | "os": [ 216 | "linux" 217 | ], 218 | "engines": { 219 | "node": ">=12" 220 | } 221 | }, 222 | "node_modules/@esbuild/linux-ppc64": { 223 | "version": "0.19.12", 224 | "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.19.12.tgz", 225 | "integrity": "sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg==", 226 | "cpu": [ 227 | "ppc64" 228 | ], 229 | "dev": true, 230 | "optional": true, 231 | "os": [ 232 | "linux" 233 | ], 234 | "engines": { 235 | "node": ">=12" 236 | } 237 | }, 238 | "node_modules/@esbuild/linux-riscv64": { 239 | "version": "0.19.12", 240 | "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.19.12.tgz", 241 | "integrity": "sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg==", 242 | "cpu": [ 243 | "riscv64" 244 | ], 245 | "dev": true, 246 | "optional": true, 247 | "os": [ 248 | "linux" 249 | ], 250 | "engines": { 251 | "node": ">=12" 252 | } 253 | }, 254 | "node_modules/@esbuild/linux-s390x": { 255 | "version": "0.19.12", 256 | "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.19.12.tgz", 257 | "integrity": "sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg==", 258 | "cpu": [ 259 | "s390x" 260 | ], 261 | "dev": true, 262 | "optional": true, 263 | "os": [ 264 | "linux" 265 | ], 266 | "engines": { 267 | "node": ">=12" 268 | } 269 | }, 270 | "node_modules/@esbuild/linux-x64": { 271 | "version": "0.19.12", 272 | "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.12.tgz", 273 | "integrity": "sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg==", 274 | "cpu": [ 275 | "x64" 276 | ], 277 | "dev": true, 278 | "optional": true, 279 | "os": [ 280 | "linux" 281 | ], 282 | "engines": { 283 | "node": ">=12" 284 | } 285 | }, 286 | "node_modules/@esbuild/netbsd-x64": { 287 | "version": "0.19.12", 288 | "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.19.12.tgz", 289 | "integrity": "sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA==", 290 | "cpu": [ 291 | "x64" 292 | ], 293 | "dev": true, 294 | "optional": true, 295 | "os": [ 296 | "netbsd" 297 | ], 298 | "engines": { 299 | "node": ">=12" 300 | } 301 | }, 302 | "node_modules/@esbuild/openbsd-x64": { 303 | "version": "0.19.12", 304 | "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.19.12.tgz", 305 | "integrity": "sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw==", 306 | "cpu": [ 307 | "x64" 308 | ], 309 | "dev": true, 310 | "optional": true, 311 | "os": [ 312 | "openbsd" 313 | ], 314 | "engines": { 315 | "node": ">=12" 316 | } 317 | }, 318 | "node_modules/@esbuild/sunos-x64": { 319 | "version": "0.19.12", 320 | "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.19.12.tgz", 321 | "integrity": "sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA==", 322 | "cpu": [ 323 | "x64" 324 | ], 325 | "dev": true, 326 | "optional": true, 327 | "os": [ 328 | "sunos" 329 | ], 330 | "engines": { 331 | "node": ">=12" 332 | } 333 | }, 334 | "node_modules/@esbuild/win32-arm64": { 335 | "version": "0.19.12", 336 | "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.19.12.tgz", 337 | "integrity": "sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A==", 338 | "cpu": [ 339 | "arm64" 340 | ], 341 | "dev": true, 342 | "optional": true, 343 | "os": [ 344 | "win32" 345 | ], 346 | "engines": { 347 | "node": ">=12" 348 | } 349 | }, 350 | "node_modules/@esbuild/win32-ia32": { 351 | "version": "0.19.12", 352 | "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.19.12.tgz", 353 | "integrity": "sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ==", 354 | "cpu": [ 355 | "ia32" 356 | ], 357 | "dev": true, 358 | "optional": true, 359 | "os": [ 360 | "win32" 361 | ], 362 | "engines": { 363 | "node": ">=12" 364 | } 365 | }, 366 | "node_modules/@esbuild/win32-x64": { 367 | "version": "0.19.12", 368 | "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.19.12.tgz", 369 | "integrity": "sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA==", 370 | "cpu": [ 371 | "x64" 372 | ], 373 | "dev": true, 374 | "optional": true, 375 | "os": [ 376 | "win32" 377 | ], 378 | "engines": { 379 | "node": ">=12" 380 | } 381 | }, 382 | "node_modules/esbuild": { 383 | "version": "0.19.12", 384 | "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.19.12.tgz", 385 | "integrity": "sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg==", 386 | "dev": true, 387 | "hasInstallScript": true, 388 | "bin": { 389 | "esbuild": "bin/esbuild" 390 | }, 391 | "engines": { 392 | "node": ">=12" 393 | }, 394 | "optionalDependencies": { 395 | "@esbuild/aix-ppc64": "0.19.12", 396 | "@esbuild/android-arm": "0.19.12", 397 | "@esbuild/android-arm64": "0.19.12", 398 | "@esbuild/android-x64": "0.19.12", 399 | "@esbuild/darwin-arm64": "0.19.12", 400 | "@esbuild/darwin-x64": "0.19.12", 401 | "@esbuild/freebsd-arm64": "0.19.12", 402 | "@esbuild/freebsd-x64": "0.19.12", 403 | "@esbuild/linux-arm": "0.19.12", 404 | "@esbuild/linux-arm64": "0.19.12", 405 | "@esbuild/linux-ia32": "0.19.12", 406 | "@esbuild/linux-loong64": "0.19.12", 407 | "@esbuild/linux-mips64el": "0.19.12", 408 | "@esbuild/linux-ppc64": "0.19.12", 409 | "@esbuild/linux-riscv64": "0.19.12", 410 | "@esbuild/linux-s390x": "0.19.12", 411 | "@esbuild/linux-x64": "0.19.12", 412 | "@esbuild/netbsd-x64": "0.19.12", 413 | "@esbuild/openbsd-x64": "0.19.12", 414 | "@esbuild/sunos-x64": "0.19.12", 415 | "@esbuild/win32-arm64": "0.19.12", 416 | "@esbuild/win32-ia32": "0.19.12", 417 | "@esbuild/win32-x64": "0.19.12" 418 | } 419 | }, 420 | "node_modules/fsevents": { 421 | "version": "2.3.3", 422 | "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", 423 | "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", 424 | "dev": true, 425 | "hasInstallScript": true, 426 | "optional": true, 427 | "os": [ 428 | "darwin" 429 | ], 430 | "engines": { 431 | "node": "^8.16.0 || ^10.6.0 || >=11.0.0" 432 | } 433 | }, 434 | "node_modules/get-tsconfig": { 435 | "version": "4.7.3", 436 | "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.3.tgz", 437 | "integrity": "sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg==", 438 | "dev": true, 439 | "dependencies": { 440 | "resolve-pkg-maps": "^1.0.0" 441 | }, 442 | "funding": { 443 | "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" 444 | } 445 | }, 446 | "node_modules/kafkajs": { 447 | "version": "2.2.4", 448 | "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", 449 | "integrity": "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==", 450 | "engines": { 451 | "node": ">=14.0.0" 452 | } 453 | }, 454 | "node_modules/resolve-pkg-maps": { 455 | "version": "1.0.0", 456 | "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", 457 | "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", 458 | "dev": true, 459 | "funding": { 460 | "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" 461 | } 462 | }, 463 | "node_modules/tsx": { 464 | "version": "4.7.1", 465 | "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.7.1.tgz", 466 | "integrity": "sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==", 467 | "dev": true, 468 | "dependencies": { 469 | "esbuild": "~0.19.10", 470 | "get-tsconfig": "^4.7.2" 471 | }, 472 | "bin": { 473 | "tsx": "dist/cli.mjs" 474 | }, 475 | "engines": { 476 | "node": ">=18.0.0" 477 | }, 478 | "optionalDependencies": { 479 | "fsevents": "~2.3.3" 480 | } 481 | } 482 | } 483 | } 484 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "kafkajs": "^2.2.4" 4 | }, 5 | "scripts": { 6 | "consume": "tsx src/js_consume.js", 7 | "produce": "tsx src/js_produce.js" 8 | }, 9 | "devDependencies": { 10 | "tsx": "^4.7.1" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /playbooks/kafka.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare Cluster for Kafka 3 | hosts: kafka_servers 4 | become: true # Required to edit /etc/hosts 5 | vars: 6 | my_config_dir: '/data/my-config' 7 | my_kafka_props: '{{ my_config_dir }}/kafka.properties' 8 | tasks: 9 | - name: Set hostname 10 | ansible.builtin.hostname: 11 | name: "{{ inventory_hostname }}" 12 | 13 | - name: Replace /etc/hosts with Terraform-managed host-config.txt 14 | ansible.builtin.copy: 15 | src: "{{ inventory_dir }}/host-config.txt" 16 | dest: /etc/hosts 17 | backup: yes 18 | 19 | - name: Ensure A few Directories Exist My Config Dir 20 | ansible.builtin.file: 21 | path: '{{ my_config_dir }}' 22 | state: directory 23 | mode: '0755' 24 | 25 | - name: Copy Kafka Properties 26 | ansible.builtin.template: 27 | src: ./templates/kafka.properties.j2 28 | dest: '{{ my_kafka_props }}' 29 | backup: yes 30 | register: kafka_properties 31 | 32 | - name: Deploy Kafka systemd service file 33 | ansible.builtin.template: 34 | src: ./templates/kafka.systemd.j2 35 | dest: /etc/systemd/system/kafka.service 36 | owner: root 37 | group: root 38 | mode: '0644' 39 | register: kafka_service 40 | notify: Reload systemd 41 | 42 | - name: Flush handlers to ensure systemd is reloaded immediately 43 | meta: flush_handlers 44 | 45 | - name: Copy Kafka Bootstrap Script 46 | ansible.builtin.copy: 47 | src: '{{ inventory_dir }}/config/install-kafka.sh' 48 | dest: /tmp/bootstrap-kafka.sh 49 | mode: '0755' 50 | 51 | - name: Run Bootstrap Script 52 | ansible.builtin.shell: /tmp/bootstrap-kafka.sh 53 | 54 | - name: Check Kafka status 55 | ansible.builtin.systemd: 56 | name: kafka 57 | state: started 58 | register: kafka_service 59 | # failed_when: zookeeper_service.status.ActiveState != 'active' 60 | ignore_errors: true 61 | 62 | - name: Start kafka if not running 63 | ansible.builtin.systemd: 64 | name: kafka 65 | state: started 66 | enabled: yes 67 | when: kafka_service.status.ActiveState != 'active' 68 | 69 | - name: Restart kafka if properties changed 70 | ansible.builtin.systemd: 71 | name: kafka 72 | state: restarted 73 | when: kafka_properties.changed and kafka_service.status.ActiveState == 'active' 74 | 75 | - name: Restart kafka if systemd conf changed 76 | ansible.builtin.systemd: 77 | name: kafka 78 | state: restarted 79 | when: kafka_service.changed 80 | 81 | handlers: 82 | - name: Reload systemd 83 | ansible.builtin.systemd: 84 | daemon_reload: yes 85 | -------------------------------------------------------------------------------- /playbooks/templates/kafka.properties.j2: -------------------------------------------------------------------------------- 1 | # https://kafka.apache.org/31/generated/kafka_config.html 2 | # The essential configurations are the following: 3 | # broker.id 4 | # log.dirs 5 | # zookeeper.connect 6 | 7 | # based on instance hostname 8 | broker.id={{ node_id }} 9 | 10 | listeners=INTERNAL://{{ inventory_hostname }}:9092,EXTERNAL://{{ ansible_host }}:19092 11 | advertised.listeners=INTERNAL://{{ inventory_hostname }}:9092,EXTERNAL://{{ ansible_host }}:19092 12 | listener.security.protocol.map=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT 13 | inter.broker.listener.name=INTERNAL 14 | 15 | # Topic create/delete 16 | delete.topic.enable=true 17 | auto.create.topics.enable=true 18 | 19 | # topic partitions and replications 20 | num.partitions=4 21 | default.replication.factor=3 22 | min.insync.replicas=2 23 | 24 | # data logging 25 | log.dirs=/data/kafka 26 | log.retention.hours=168 27 | log.segment.bytes=1073741824 28 | log.retention.check.interval.ms=300000 29 | 30 | # zookeeper connection 31 | zookeeper.connect=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181 32 | 33 | zookeeper.connect={% for host in groups['zookeeper_servers'] %}zookeeper{{ hostvars[host]['node_id'] }}:2181{% if not loop.last %},{% endif %}{% endfor %} 34 | 35 | zookeeper.connection.timeout.ms=6000 36 | 37 | 38 | -------------------------------------------------------------------------------- /playbooks/templates/kafka.systemd.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=kafka server 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | User=tars 8 | ExecStart=/opt/kafka/bin/kafka-server-start.sh {{ my_kafka_props }} 9 | WorkingDirectory=/opt/kafka 10 | Restart=on-failure 11 | RestartSec=10s 12 | StandardOutput=file:/var/log/kafka/kafka.out 13 | StandardError=file:/var/log/kafka/kafka.err 14 | LimitNOFILE=800000 15 | Environment=PATH=/usr/bin:/bin:/usr/local/bin 16 | 17 | [Install] 18 | WantedBy=multi-user.target 19 | -------------------------------------------------------------------------------- /playbooks/templates/zookeeper.properties.j2: -------------------------------------------------------------------------------- 1 | # custom zookeeper settings 2 | # all found at https://zookeeper.apache.org/doc/r3.3.3/zookeeperAdmin.html#sc_configuration 3 | 4 | # the directory where the snapshot is stored. 5 | dataDir=/data/zookeeper 6 | # the port at which the clients will connect 7 | clientPort=2181 8 | # disable the per-ip limit on the number of connections since this is a non-production config 9 | maxClientCnxns=0 10 | 11 | # tickTime the basic time unit in milliseconds, regulating heartbeats and timeouts, with a minimum session timeout of two ticks. 12 | tickTime=2000 13 | # the number of ticks that the initial synchronization phase can take 14 | initLimit=10 15 | # the number of ticks that can pass between sending a request and getting an acknowledgement 16 | syncLimit=5 17 | 18 | 4lw.commands.whitelist=stat, ruok, conf, isro, dump 19 | 20 | # the servers in the ensemble 21 | {% for host in groups['zookeeper_servers'] %} 22 | server.{{ loop.index }}=zookeeper{{ hostvars[host]['node_id'] }}:2888:3888 23 | {% endfor %} 24 | -------------------------------------------------------------------------------- /playbooks/templates/zookeeper.systemd.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Zookeeper server 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | User=tars 8 | ExecStart=/opt/kafka/bin/zookeeper-server-start.sh {{ my_zookeeper_props }} 9 | WorkingDirectory=/opt/kafka 10 | Restart=on-failure 11 | RestartSec=10s 12 | StandardOutput=file:/var/log/zookeeper/zookeeper.out 13 | StandardError=file:/var/log/zookeeper/zookeeper.err 14 | LimitNOFILE=800000 15 | Environment=PATH=/usr/bin:/bin:/usr/local/bin 16 | 17 | [Install] 18 | WantedBy=multi-user.target 19 | -------------------------------------------------------------------------------- /playbooks/zookeeper.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare Cluster for Zookeeper 3 | hosts: zookeeper_servers 4 | become: true # Required to edit /etc/hosts 5 | vars: 6 | my_config_dir: '/data/my-config' 7 | my_zookeeper_props: '{{ my_config_dir }}/zookeeper.properties' 8 | tasks: 9 | - name: Set hostname 10 | ansible.builtin.hostname: 11 | name: "{{ inventory_hostname }}" 12 | 13 | - name: Replace /etc/hosts with Terraform-managed host-config.txt 14 | ansible.builtin.copy: 15 | src: "{{ inventory_dir }}/host-config.txt" 16 | dest: /etc/hosts 17 | backup: yes 18 | 19 | - name: Ensure A few Directories Exist My Config Dir 20 | ansible.builtin.file: 21 | path: '{{ my_config_dir }}' 22 | state: directory 23 | mode: '0755' 24 | 25 | - name: Copy Zookeeper Properties 26 | ansible.builtin.template: 27 | src: ./templates/zookeeper.properties.j2 28 | dest: '{{ my_zookeeper_props }}' 29 | backup: yes 30 | register: zookeeper_properties 31 | 32 | - name: Deploy Zookeeper systemd service file 33 | ansible.builtin.template: 34 | src: ./templates/zookeeper.systemd.j2 35 | dest: /etc/systemd/system/zookeeper.service 36 | owner: root 37 | group: root 38 | mode: '0644' 39 | notify: Reload systemd 40 | 41 | - name: Flush handlers to ensure systemd is reloaded immediately 42 | meta: flush_handlers 43 | 44 | - name: Copy Zookeeper Bootstrap Script 45 | ansible.builtin.copy: 46 | src: '{{ inventory_dir }}/config/install-zookeeper.sh' 47 | dest: /tmp/bootstrap-zookeeper.sh 48 | mode: '0755' 49 | 50 | - name: Run Bootstrap Script 51 | ansible.builtin.shell: /tmp/bootstrap-zookeeper.sh 52 | 53 | - name: Check Zookeeper status 54 | ansible.builtin.systemd: 55 | name: zookeeper 56 | state: started 57 | register: zookeeper_service 58 | # failed_when: zookeeper_service.status.ActiveState != 'active' 59 | ignore_errors: true 60 | 61 | - name: Start Zookeeper if not running 62 | ansible.builtin.systemd: 63 | name: zookeeper 64 | state: started 65 | enabled: yes 66 | when: zookeeper_service.status.ActiveState != 'active' 67 | 68 | - name: Restart Zookeeper if properties changed 69 | ansible.builtin.systemd: 70 | name: zookeeper 71 | state: restarted 72 | when: zookeeper_properties.changed and zookeeper_service.status.ActiveState == 'active' 73 | 74 | - name: Check if Zookeeper is responding to 'ruok' 75 | ansible.builtin.shell: | 76 | echo "ruok" | nc localhost 2181 ; echo 77 | register: zookeeper_ruok 78 | ignore_errors: true 79 | 80 | - name: Display Zookeeper 'ruok' response 81 | ansible.builtin.debug: 82 | msg: "{{ zookeeper_ruok.stdout_lines }}" 83 | 84 | handlers: 85 | - name: Reload systemd 86 | ansible.builtin.systemd: 87 | daemon_reload: yes 88 | -------------------------------------------------------------------------------- /src/aconsume.py: -------------------------------------------------------------------------------- 1 | from aiokafka import AIOKafkaConsumer 2 | import asyncio 3 | 4 | KAFKA_BROKER_URL="127.0.0.1:19092" 5 | KAFKA_TOPIC="hello-world" 6 | 7 | 8 | async def consume(): 9 | consumer = AIOKafkaConsumer( 10 | KAFKA_TOPIC, 11 | bootstrap_servers=KAFKA_BROKER_URL) 12 | await consumer.start() 13 | try: 14 | # Consume messages 15 | async for msg in consumer: 16 | print("consumed: ", msg.topic, msg.partition, msg.offset, 17 | msg.key, msg.value, msg.timestamp) 18 | finally: 19 | # Will leave consumer group; perform autocommit if enabled. 20 | await consumer.stop() 21 | 22 | asyncio.run(consume()) -------------------------------------------------------------------------------- /src/aproduce.py: -------------------------------------------------------------------------------- 1 | import json 2 | from aiokafka import AIOKafkaProducer 3 | import asyncio 4 | 5 | KAFKA_BROKER_URL="127.0.0.1:19092" 6 | KAFKA_TOPIC="hello-world" 7 | 8 | 9 | async def send_one(): 10 | producer = AIOKafkaProducer( 11 | bootstrap_servers=KAFKA_BROKER_URL) 12 | # Get cluster layout and initial topic/partition leadership information 13 | await producer.start() 14 | data = {'hello': 'aworld'} 15 | data_json = json.dumps(data) 16 | data_ready = data_json.encode('utf-8') 17 | try: 18 | # Produce message 19 | await producer.send_and_wait(KAFKA_TOPIC, data_ready) 20 | finally: 21 | # Wait for all pending messages to be delivered or expire. 22 | await producer.stop() 23 | 24 | asyncio.run(send_one()) -------------------------------------------------------------------------------- /src/consume.py: -------------------------------------------------------------------------------- 1 | import json 2 | from kafka import KafkaConsumer, TopicPartition 3 | import pathlib 4 | from decouple import Config, RepositoryEnv 5 | # KAFKA_BROKER_URL="127.0.0.1:19092" 6 | # KAFKA_BROKER_URL2="127.0.0.1:19093" 7 | # KAFKA_BROKER_URL3="127.0.0.1:19094" 8 | # bootstrap_servers=[KAFKA_BROKER_URL, KAFKA_BROKER_URL2, KAFKA_BROKER_URL3] 9 | 10 | # KAFKA_BROKER_URL="172.234.228.67:19092" 11 | # bootstrap_servers=[KAFKA_BROKER_URL] 12 | 13 | BASE_DIR = pathlib.Path(__file__).resolve().parent.parent 14 | CLUSTER_ENV_PATH = BASE_DIR / "cluster.env" 15 | config = Config(RepositoryEnv(str(CLUSTER_ENV_PATH))) 16 | 17 | KAFKA_BROKER_1=config('KAFKA_BROKER_1', default=None) 18 | KAFKA_BROKER_2 = config("KAFKA_BROKER_2", default=None) 19 | KAFKA_BROKER_3 = config("KAFKA_BROKER_3", default=None) 20 | KAFKA_BROKER_4 = config("KAFKA_BROKER_4", default=None) 21 | KAFKA_BROKER_5 = config("KAFKA_BROKER_5", default=None) 22 | bootstrap_servers = [KAFKA_BROKER_1, KAFKA_BROKER_2, KAFKA_BROKER_3, KAFKA_BROKER_4, KAFKA_BROKER_5] 23 | 24 | 25 | 26 | KAFKA_TOPIC="some_topic" 27 | 28 | consumer = KafkaConsumer( 29 | KAFKA_TOPIC, "order_update", 'webapp_page_view', 30 | bootstrap_servers=bootstrap_servers, 31 | # auto_offset_reset='smallest' # from_beginning: true 32 | ) 33 | # consumer.assign([TopicPartition(KAFKA_TOPIC, 1)]) 34 | 35 | for msg in consumer: 36 | raw_value = msg.value 37 | value_str = raw_value.decode("utf-8") 38 | try: 39 | data = json.loads(value_str) 40 | except json.decoder.JSONDecodeError: 41 | data = None 42 | print("invalid json") 43 | print(data, type(data), type(value_str)) -------------------------------------------------------------------------------- /src/js_consume.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs") 2 | 3 | const KAFKA_BROKER_URL="127.0.0.1:19092" 4 | const KAFKA_TOPIC="hello-world" 5 | 6 | const kafka = new Kafka({ 7 | clientId: 'my-node-app', 8 | brokers: [KAFKA_BROKER_URL] 9 | }) 10 | 11 | async function handleMessage({topic, partition, message}) { 12 | const messageValue = message.value.toString() 13 | console.log(messageValue) 14 | } 15 | 16 | async function main() { 17 | const consumer = kafka.consumer({groupId: "nodejs"}) 18 | // await 19 | await consumer.connect() 20 | await consumer.subscribe({topic: KAFKA_TOPIC, fromBeginning: true}) 21 | await consumer.run({ 22 | eachMessage: handleMessage 23 | }) 24 | 25 | } 26 | 27 | 28 | main().then(x=>console.log(x)).catch(err=>console.log(err)) -------------------------------------------------------------------------------- /src/js_produce.js: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs") 2 | 3 | const KAFKA_BROKER_URL="127.0.0.1:19092" 4 | const KAFKA_TOPIC="hello-world" 5 | 6 | const kafka = new Kafka({ 7 | clientId: 'my-node-app', 8 | brokers: [KAFKA_BROKER_URL] 9 | }) 10 | 11 | 12 | async function main() { 13 | const producer = kafka.producer() 14 | // await 15 | await producer.connect() 16 | const data = {hello: "js_world"} 17 | const dataJson = JSON.stringify(data) 18 | await producer.send({ 19 | topic: KAFKA_TOPIC, 20 | messages: [ 21 | {value: dataJson} 22 | ] 23 | }) 24 | await producer.disconnect() 25 | 26 | } 27 | 28 | 29 | main().then(x=>console.log(x)).catch(err=>console.log(err)) -------------------------------------------------------------------------------- /src/produce.py: -------------------------------------------------------------------------------- 1 | import json 2 | from kafka import KafkaProducer 3 | import random 4 | 5 | # KAFKA_BROKER_URL="127.0.0.1:19092" 6 | # KAFKA_BROKER_URL2="127.0.0.1:19093" 7 | # KAFKA_BROKER_URL3="127.0.0.1:19094" 8 | # bootstrap_servers=[KAFKA_BROKER_URL, KAFKA_BROKER_URL2, KAFKA_BROKER_URL3] 9 | 10 | KAFKA_BROKER_URL="172.234.228.67:19092" 11 | bootstrap_servers=[KAFKA_BROKER_URL] 12 | 13 | KAFKA_TOPIC="my-new-topic" 14 | 15 | producer = KafkaProducer(bootstrap_servers=bootstrap_servers) 16 | 17 | data = { 18 | "hello": "world" 19 | } 20 | data_json_str = json.dumps(data) 21 | msg = data_json_str.encode('utf-8') 22 | 23 | # send message 24 | # print(msg, "not sent") 25 | 26 | # future = producer.send(KAFKA_TOPIC, msg) # async 27 | # result = future.get(timeout=60) 28 | # print(result) 29 | 30 | # producer.flush() 31 | 32 | for _ in range(2): 33 | data = {'hello': f'world-{random.randint(10, 10_000)}'} 34 | data_json = json.dumps(data) 35 | data_ready = data_json.encode('utf-8') 36 | # data_ready = "hello world".encode('utf-8') 37 | producer.send(KAFKA_TOPIC, data_ready) # partition=1) 38 | 39 | producer.flush() -------------------------------------------------------------------------------- /vm/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use Debian Bullseye Slim as the base image 2 | FROM debian:bullseye-slim 3 | 4 | # Update package lists and install nano 5 | RUN apt-get update && \ 6 | apt-get install -y nano netcat-traditional nmap bash && \ 7 | rm -rf /var/lib/apt/lists/* 8 | 9 | # Set bash as the default shell 10 | SHELL ["/bin/bash", "-c"] 11 | 12 | # Set an entry point that runs indefinitely 13 | ENTRYPOINT ["tail", "-f", "/dev/null"] -------------------------------------------------------------------------------- /webapp_django/cfehome/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codingforentrepreneurs/Coding-with-Kafka/f00b5c292cc68c9ce1c5269c330b4e2a04f49915/webapp_django/cfehome/__init__.py -------------------------------------------------------------------------------- /webapp_django/cfehome/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for cfehome project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cfehome.settings") 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /webapp_django/cfehome/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for cfehome project. 3 | 4 | Generated by 'django-admin startproject' using Django 5.0.1. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/5.0/ref/settings/ 11 | """ 12 | 13 | from pathlib import Path 14 | 15 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 16 | BASE_DIR = Path(__file__).resolve().parent.parent 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = "django-insecure-w9$j--x58k8f6qfrh#+f@+wd$2)++er7z0rcdf=f&m^738p4j7" 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = ["*"] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | "django.contrib.admin", 35 | "django.contrib.auth", 36 | "django.contrib.contenttypes", 37 | "django.contrib.sessions", 38 | "django.contrib.messages", 39 | "django.contrib.staticfiles", 40 | "orders", 41 | ] 42 | 43 | MIDDLEWARE = [ 44 | "django.middleware.security.SecurityMiddleware", 45 | "django.contrib.sessions.middleware.SessionMiddleware", 46 | "django.middleware.common.CommonMiddleware", 47 | "django.middleware.csrf.CsrfViewMiddleware", 48 | "django.contrib.auth.middleware.AuthenticationMiddleware", 49 | "django.contrib.messages.middleware.MessageMiddleware", 50 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 51 | ] 52 | 53 | ROOT_URLCONF = "cfehome.urls" 54 | 55 | TEMPLATES = [ 56 | { 57 | "BACKEND": "django.template.backends.django.DjangoTemplates", 58 | "DIRS": [], 59 | "APP_DIRS": True, 60 | "OPTIONS": { 61 | "context_processors": [ 62 | "django.template.context_processors.debug", 63 | "django.template.context_processors.request", 64 | "django.contrib.auth.context_processors.auth", 65 | "django.contrib.messages.context_processors.messages", 66 | ], 67 | }, 68 | }, 69 | ] 70 | 71 | WSGI_APPLICATION = "cfehome.wsgi.application" 72 | 73 | 74 | # Database 75 | # https://docs.djangoproject.com/en/5.0/ref/settings/#databases 76 | 77 | DATABASES = { 78 | "default": { 79 | "ENGINE": "django.db.backends.sqlite3", 80 | "NAME": BASE_DIR / "db.sqlite3", 81 | } 82 | } 83 | 84 | 85 | # Password validation 86 | # https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators 87 | 88 | AUTH_PASSWORD_VALIDATORS = [ 89 | { 90 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", 91 | }, 92 | { 93 | "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", 94 | }, 95 | { 96 | "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", 97 | }, 98 | { 99 | "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", 100 | }, 101 | ] 102 | 103 | 104 | # Internationalization 105 | # https://docs.djangoproject.com/en/5.0/topics/i18n/ 106 | 107 | LANGUAGE_CODE = "en-us" 108 | 109 | TIME_ZONE = "UTC" 110 | 111 | USE_I18N = True 112 | 113 | USE_TZ = True 114 | 115 | 116 | # Static files (CSS, JavaScript, Images) 117 | # https://docs.djangoproject.com/en/5.0/howto/static-files/ 118 | 119 | STATIC_URL = "static/" 120 | 121 | # Default primary key field type 122 | # https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field 123 | 124 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 125 | -------------------------------------------------------------------------------- /webapp_django/cfehome/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | URL configuration for cfehome project. 3 | 4 | The `urlpatterns` list routes URLs to views. For more information please see: 5 | https://docs.djangoproject.com/en/5.0/topics/http/urls/ 6 | Examples: 7 | Function views 8 | 1. Add an import: from my_app import views 9 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 10 | Class-based views 11 | 1. Add an import: from other_app.views import Home 12 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 13 | Including another URLconf 14 | 1. Import the include() function: from django.urls import include, path 15 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 16 | """ 17 | from django.contrib import admin 18 | from django.urls import path 19 | 20 | urlpatterns = [ 21 | path("admin/", admin.site.urls), 22 | ] 23 | -------------------------------------------------------------------------------- /webapp_django/cfehome/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for cfehome project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cfehome.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /webapp_django/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cfehome.settings") 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == "__main__": 22 | main() 23 | -------------------------------------------------------------------------------- /webapp_django/mykafka/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import get_consumer, get_producer, send_topic_data 2 | 3 | __all__ = [ 4 | 'get_consumer', 5 | 'get_producer', 6 | 'send_topic_data' 7 | ] -------------------------------------------------------------------------------- /webapp_django/mykafka/client.py: -------------------------------------------------------------------------------- 1 | import json 2 | from kafka import KafkaConsumer 3 | from kafka import KafkaProducer 4 | from django.conf import settings 5 | from decouple import Config, RepositoryEnv 6 | 7 | BASE_DIR = settings.BASE_DIR 8 | REPO_DIR = BASE_DIR.parent 9 | CLUSTER_ENV_PATH = REPO_DIR / "cluster.env" 10 | 11 | config = Config(RepositoryEnv(str(CLUSTER_ENV_PATH))) 12 | 13 | KAFKA_BROKER_1=config('KAFKA_BROKER_1', default=None) 14 | KAFKA_BROKER_2 = config("KAFKA_BROKER_2", default=None) 15 | KAFKA_BROKER_3 = config("KAFKA_BROKER_3", default=None) 16 | KAFKA_BROKER_4 = config("KAFKA_BROKER_4", default=None) 17 | KAFKA_BROKER_5 = config("KAFKA_BROKER_5", default=None) 18 | bootstrap_servers = [KAFKA_BROKER_1, KAFKA_BROKER_2, KAFKA_BROKER_3, KAFKA_BROKER_4, KAFKA_BROKER_5] 19 | 20 | def get_consumer(topics=["default"], from_beginning=False): 21 | kafka_config = { 22 | "bootstrap_servers": bootstrap_servers, 23 | } 24 | if from_beginning: 25 | kafka_config['auto_offset_reset'] = 'smallest' 26 | return KafkaConsumer( 27 | *topics, 28 | **kafka_config 29 | ) 30 | 31 | 32 | def get_producer(): 33 | return KafkaProducer(bootstrap_servers=bootstrap_servers) 34 | 35 | def send_topic_data(data:dict, topic:str="some_example"): 36 | producer = get_producer() 37 | data_json = json.dumps(data) 38 | data_ready = data_json.encode('utf-8') 39 | return producer.send(topic, data_ready) -------------------------------------------------------------------------------- /webapp_django/orders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codingforentrepreneurs/Coding-with-Kafka/f00b5c292cc68c9ce1c5269c330b4e2a04f49915/webapp_django/orders/__init__.py -------------------------------------------------------------------------------- /webapp_django/orders/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | from .models import Order 5 | 6 | 7 | class OrderAdmin(admin.ModelAdmin): 8 | readonly_fields = ['finalize_url', 'timestamp', 'updated'] 9 | 10 | admin.site.register(Order, OrderAdmin) -------------------------------------------------------------------------------- /webapp_django/orders/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class OrdersConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "orders" 7 | -------------------------------------------------------------------------------- /webapp_django/orders/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codingforentrepreneurs/Coding-with-Kafka/f00b5c292cc68c9ce1c5269c330b4e2a04f49915/webapp_django/orders/management/__init__.py -------------------------------------------------------------------------------- /webapp_django/orders/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codingforentrepreneurs/Coding-with-Kafka/f00b5c292cc68c9ce1c5269c330b4e2a04f49915/webapp_django/orders/management/commands/__init__.py -------------------------------------------------------------------------------- /webapp_django/orders/management/commands/listen.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | from orders.models import Order 3 | import json 4 | import mykafka 5 | 6 | # python manage.py listen 7 | class Command(BaseCommand): 8 | help = 'Process orders' 9 | 10 | def handle(self, *args, **options): 11 | # Your logic to process orders goes here 12 | consumer = mykafka.get_consumer(topics=['order_update']) 13 | 14 | for message in consumer: 15 | try: 16 | self.process_message(message) 17 | except Exception as e: 18 | print(f'Error processing message: {e}') 19 | 20 | def process_message(self, message): 21 | raw_value = message.value 22 | value_str = raw_value.decode("utf-8") 23 | try: 24 | data = json.loads(value_str) 25 | except json.decoder.JSONDecodeError: 26 | data = None 27 | print("invalid json") 28 | # print(data, 'django_listener') 29 | data_type = data.get('type') 30 | order_shipped_type = f"orders/{Order.OrderStatus.SHIPPED}" 31 | if data_type == order_shipped_type: 32 | print(data) 33 | order_id = data.get('order_id') 34 | qs = Order.objects.filter(id__iexact=order_id).exclude(status=Order.OrderStatus.SHIPPED) 35 | if qs.exists(): 36 | qs.update(status=Order.OrderStatus.SHIPPED, is_shipped=True) 37 | -------------------------------------------------------------------------------- /webapp_django/orders/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.3 on 2024-03-27 18:26 2 | 3 | import uuid 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | initial = True 9 | 10 | dependencies = [] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name="Order", 15 | fields=[ 16 | ( 17 | "id", 18 | models.UUIDField( 19 | default=uuid.uuid4, 20 | editable=False, 21 | primary_key=True, 22 | serialize=False, 23 | ), 24 | ), 25 | ("product_name", models.CharField(blank=True, max_length=100)), 26 | ( 27 | "status", 28 | models.CharField( 29 | choices=[ 30 | ("pending", "Pending"), 31 | ("processed", "Processed"), 32 | ("shipped", "Shipped"), 33 | ], 34 | default="pending", 35 | max_length=20, 36 | ), 37 | ), 38 | ("timestamp", models.DateTimeField(auto_now_add=True)), 39 | ("updated", models.DateTimeField(auto_now=True)), 40 | ], 41 | ), 42 | ] 43 | -------------------------------------------------------------------------------- /webapp_django/orders/migrations/0002_order_is_shipped.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.3 on 2024-03-27 19:09 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("orders", "0001_initial"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="order", 14 | name="is_shipped", 15 | field=models.BooleanField(default=False), 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /webapp_django/orders/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codingforentrepreneurs/Coding-with-Kafka/f00b5c292cc68c9ce1c5269c330b4e2a04f49915/webapp_django/orders/migrations/__init__.py -------------------------------------------------------------------------------- /webapp_django/orders/models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from django.db import models 4 | from django.db.models.signals import post_save 5 | 6 | from django.utils.safestring import mark_safe 7 | import mykafka 8 | 9 | from . import utils as orders_utils 10 | 11 | 12 | class Order(models.Model): 13 | class OrderStatus(models.TextChoices): 14 | PENDING = 'pending', 'Pending' 15 | PROCESSED = 'processed', 'Processed' 16 | SHIPPED = 'shipped', 'Shipped' 17 | 18 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 19 | product_name = models.CharField(max_length=100, blank=True) 20 | status = models.CharField( 21 | max_length=20, 22 | choices=OrderStatus.choices, 23 | default=OrderStatus.PENDING 24 | ) 25 | is_shipped = models.BooleanField(default=False) 26 | timestamp = models.DateTimeField(auto_now_add=True) 27 | updated = models.DateTimeField(auto_now=True) 28 | 29 | def __str__(self): 30 | return str(self.id) 31 | 32 | def save(self, *args, **kwargs): 33 | if not self.product_name: 34 | self.product_name = orders_utils.generate_product_name() 35 | if self.is_shipped: 36 | self.status = Order.OrderStatus.SHIPPED 37 | super().save(*args, **kwargs) 38 | 39 | def ready_to_ship(self): 40 | return self.status == self.OrderStatus.PROCESSED 41 | 42 | def shipped(self): 43 | return self.status == self.OrderStatus.SHIPPED 44 | 45 | def serialize(self): 46 | return { 47 | "id": str(self.id), 48 | "product_name": self.product_name, 49 | "status": self.status, 50 | "timestamp": self.timestamp.timestamp() 51 | } 52 | 53 | def finalize_url(self): 54 | fastapi_url = f"http://localhost:8000/order/{self.id}" 55 | return mark_safe(f"{fastapi_url}") 56 | 57 | 58 | 59 | 60 | def order_did_update(sender, instance, *args, **kwargs): 61 | if instance.ready_to_ship() and not instance.is_shipped: 62 | instance_data = instance.serialize() 63 | topic_status = Order.OrderStatus.PROCESSED 64 | topic_data = { 65 | "type": f"orders/{topic_status}", 66 | "object": instance_data, 67 | } 68 | # print(topic_data) 69 | result = mykafka.send_topic_data(topic_data, topic=f'order_update') 70 | # print(result) 71 | 72 | 73 | post_save.connect(order_did_update, sender=Order) -------------------------------------------------------------------------------- /webapp_django/orders/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /webapp_django/orders/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | def generate_product_name(): 3 | # Sample list of product names (you can replace this with a larger dataset) 4 | product_names = [ 5 | "Widget", "Gadget", "Tool", "Accessory", "Appliance", "Device", "Equipment", 6 | "Utensil", "Implement", "Contraption", "Instrument", "Apparatus", "Fixture", 7 | "Furniture", "Merchandise", "Article", "Commodity", "Merchandise", "Asset" 8 | ] 9 | a = random.choice(product_names) 10 | b = random.choice(product_names) 11 | if b == a: 12 | return f"{a}" 13 | return f"{a} {b}" 14 | -------------------------------------------------------------------------------- /webapp_django/orders/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | 3 | # Create your views here. 4 | -------------------------------------------------------------------------------- /webapp_django/requirements.txt: -------------------------------------------------------------------------------- 1 | Django 2 | python-decouple 3 | kafka-python -------------------------------------------------------------------------------- /webapp_fastapi/main.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from typing import Union 3 | from contextlib import asynccontextmanager 4 | import json 5 | 6 | from aiokafka import AIOKafkaProducer 7 | from fastapi import FastAPI,Request, Response 8 | from decouple import Config, RepositoryEnv 9 | 10 | 11 | 12 | BASE_DIR = pathlib.Path(__file__).resolve().parent.parent 13 | CLUSTER_ENV_PATH = BASE_DIR / "cluster.env" 14 | config = Config(RepositoryEnv(str(CLUSTER_ENV_PATH))) 15 | 16 | KAFKA_BROKER_1=config('KAFKA_BROKER_1', default=None) 17 | KAFKA_BROKER_2 = config("KAFKA_BROKER_2", default=None) 18 | KAFKA_BROKER_3 = config("KAFKA_BROKER_3", default=None) 19 | KAFKA_BROKER_4 = config("KAFKA_BROKER_4", default=None) 20 | KAFKA_BROKER_5 = config("KAFKA_BROKER_5", default=None) 21 | bootstrap_servers = [KAFKA_BROKER_1, KAFKA_BROKER_2, KAFKA_BROKER_3, KAFKA_BROKER_4, KAFKA_BROKER_5] 22 | 23 | workers = {} 24 | 25 | 26 | @asynccontextmanager 27 | async def lifespan(app: FastAPI): 28 | producer = AIOKafkaProducer(bootstrap_servers=bootstrap_servers) 29 | await producer.start() 30 | workers['producer'] = producer 31 | yield 32 | try: 33 | await producer.stop() 34 | except: 35 | pass 36 | workers.clear() 37 | 38 | 39 | 40 | app = FastAPI(lifespan=lifespan) 41 | 42 | 43 | @app.middleware("http") 44 | async def kafka_page_view_event(request: Request, call_next): 45 | # after the request, before response 46 | page_view_event = { 47 | "type": "page/view", 48 | "path": request.url.path, 49 | "method": request.method, 50 | "status_code": None, 51 | "headers": {k: v for k, v in request.headers.items()}, 52 | } 53 | producer = workers.get("producer") 54 | try: 55 | response = await call_next(request) 56 | except Exception as e: 57 | page_error = { 58 | "type": "page/view/error", 59 | "error": str(e), 60 | "status_code": 500 61 | } 62 | print(page_error) 63 | event_data = {**page_view_event, **page_error} 64 | if producer is not None: 65 | data = json.dumps(event_data).encode("utf-8") 66 | topic = "webapp_page_view" 67 | await producer.send_and_wait(topic, data) 68 | return Response(content="

Internal Server Error

500 error

") 69 | event_data = {**page_view_event} 70 | # after the response but before it's returned 71 | event_data['status_code'] = response.status_code 72 | if producer is not None: 73 | data = json.dumps(event_data).encode("utf-8") 74 | # print(data) 75 | topic = "webapp_page_view" 76 | await producer.send_and_wait(topic, data) 77 | return response 78 | 79 | 80 | @app.get("/") 81 | async def read_root(order_id:str=None): 82 | # raise Exception("Server error") 83 | # print(order_id) 84 | # event_data = {"type": "orders/start", "order_id": order_id} 85 | # data = json.dumps(event_data).encode("utf-8") 86 | # producer = workers.get("producer") 87 | # if producer is not None: 88 | # topic = "some_topic" 89 | # await producer.send_and_wait(topic, data) 90 | return {"Hello": "World", "BASE_DIR": BASE_DIR} 91 | 92 | 93 | @app.get("/order/{order_id}") 94 | async def read_order(order_id:str=None): 95 | print(order_id) 96 | event_data = {"type": "orders/shipped", "order_id": order_id} 97 | data = json.dumps(event_data).encode("utf-8") 98 | producer = workers.get("producer") 99 | if producer is not None: 100 | topic = "order_update" 101 | await producer.send_and_wait(topic, data) 102 | return {"Hello": "World", "BASE_DIR": BASE_DIR} 103 | 104 | 105 | 106 | @app.get("/items/{item_id}") 107 | def read_item(item_id: int, q: Union[str, None] = None): 108 | return {"item_id": item_id, "q": q} 109 | 110 | 111 | @app.get("/about") 112 | async def read_about(): 113 | return {"Hello": "World"} 114 | -------------------------------------------------------------------------------- /webapp_fastapi/requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | # uvicorn main:app --reload 3 | # main.py has app = FastAPI() -> main:app 4 | uvicorn 5 | python-decouple 6 | aiokafka -------------------------------------------------------------------------------- /webapp_fastapi/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | uvicorn main:app --reload --------------------------------------------------------------------------------