├── .gitignore ├── LICENSE ├── README.md ├── django-tutorial ├── db.sqlite3 ├── django_tutorial │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── manage.py ├── musics │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models.py │ ├── tests.py │ └── views.py └── templates │ └── hello_django.html ├── docker-elk ├── docker-compose.yml ├── elasticsearch │ ├── Dockerfile │ └── config │ │ └── elasticsearch.yml ├── kibana │ ├── Dockerfile │ └── config │ │ └── kibana.yml └── logstash │ ├── Dockerfile │ ├── config │ └── logstash.yml │ └── pipeline │ └── logstash.conf ├── docker-logging └── docker-compose.yml └── python-logging └── demo_logging.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-elk-tutorial 2 | 3 | docker-elk-tutorial📝 4 | 5 | * [Youtube Tutorial PART 1 - ELK 簡介](https://youtu.be/T_sLKn3vXa4) 6 | * [Youtube Tutorial PART 2 - docker ELK 環境建立](https://youtu.be/4JybtoFgC8g) 7 | * [Youtube Tutorial PART 3 - 透過 python 送 log 到 ELK](https://youtu.be/EpEJGLzIK6A) 8 | * [Youtube Tutorial PART 4 - logging for Django + ELK](https://youtu.be/_bkx0FfNRpQ) 9 | * [Youtube Tutorial PART 5 - docker logging + ELK](https://youtu.be/gTqAjea4Ncg) 10 | 11 | * 進階 [Youtube Tutorial - Linux 教學 - docker-elk-tutorial 7.6.0](https://youtu.be/iWFasUQ1tNQ) - [docker-elk-tutorial 7.6.0](https://github.com/twtrubiks/docker-elk-tutorial/tree/elk-7.6.0) 12 | 13 | ## 簡介 14 | 15 | * [Youtube Tutorial PART 1 - ELK 簡介](https://youtu.be/T_sLKn3vXa4) 16 | 17 | docker-elk :question: 這是什麼:question: 他可以吃嗎:confused: 18 | 19 | 重點在 **ELK** ，他是由三個東西所組成的。 20 | 21 | [Elasticsearch](https://www.elastic.co/) ( E ) 22 | 23 | ![img](https://i.imgur.com/qSbJRSv.png) 24 | 25 | [Logstash](https://www.elastic.co/products/logstash) ( L ) 26 | 27 | ![img](https://i.imgur.com/7sQUVqy.png) 28 | 29 | [Kibana](https://www.elastic.co/products/kibana) ( K ) 30 | 31 | ![img](https://i.imgur.com/eajQh99.png) 32 | 33 | 基本上，整個工作流程是這樣 34 | 35 | ![img](https://i.imgur.com/ZTDCjnD.png) 36 | 37 | 步驟一 38 | 39 | Logstash 蒐集從 docker or 其他地方的 log 資訊，這個步驟主要是因為我們可以透過 [logstash.conf](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/docker-elk/logstash/pipeline/logstash.conf) 過濾 40 | 41 | 以及解析我們需要的資訊。 42 | 43 | 步驟二 44 | 45 | Logstash 將處理完後的 log 資訊轉發到 Elasticsearch 進行 index。 46 | 47 | 步驟三 48 | 49 | 最後使用者可以透過 Kibana 分析以及視覺化所要的資料。 50 | 51 | 以上就是整個工作的流程，那他有什麼用呢 :confused: 52 | 53 | 像是分散式系統好了，之前介紹的 [docker swawm](https://github.com/twtrubiks/docker-swarm-tutorial)，每個容器的 log都進去一個一個看一定會累死， 54 | 55 | 所以這時候就可以統一把 log 送到 docker-elk 中，方便統一管理以及分析。 56 | 57 | 使用者的 log 非常重要，如果可以從 log 中分析出使用者愛好以及習慣，就可以推薦他類似的東西或 58 | 59 | 進行改善，當然，有一點很重要，就是這些 log 必須 **處理** 過，你可能會和我說可以用 AI（ AI 正夯 :expressionless: ） 60 | 61 | 但這不是這次的重點:relaxed: 62 | 63 | 由於這篇文章我會採用 Docker 建立 docker-elk，所以建議對 Docker 要有一定的認識，如果你不了解 64 | 65 | Docker ，可參考我之前的 Docker 教學文章 66 | 67 | * [Docker 基本教學 - 從無到有 Docker-Beginners-Guide](https://github.com/twtrubiks/docker-tutorial) 68 | 69 | 透過這篇文章，你將會學會 70 | 71 | * [docker ELK 環境建立](https://github.com/twtrubiks/docker-elk-tutorial#docker-elk-%E7%92%B0%E5%A2%83%E5%BB%BA%E7%AB%8B) 72 | 73 | * [透過 python 送 log 到 ELK](https://github.com/twtrubiks/docker-elk-tutorial#%E9%80%8F%E9%81%8E-python-%E9%80%81-log-%E5%88%B0-elk) 74 | 75 | * [logging for Django + ELK](https://github.com/twtrubiks/docker-elk-tutorial#logging-for-django--elk) - Django 如何設定 logging 以及發送 logging 到 ELK 中 76 | 77 | * [docker logging + ELK](https://github.com/twtrubiks/docker-elk-tutorial#docker-logging--elk) - 將 docker logs 發送到 docker ELK 中 78 | 79 | ## docker ELK 環境建立 80 | 81 | * [Youtube Tutorial PART 2 - docker ELK 環境建立](https://youtu.be/4JybtoFgC8g) 82 | 83 | 我們直接使用 [docker-elk](https://github.com/deviantony/docker-elk) 這邊的 docker-compose.yml 即可，但因為我擔心版本會 84 | 85 | 更新（ 導致怪問題 ），所以我放一份到我自己的目錄下，建議閱讀一下 [docker-elk](https://github.com/deviantony/docker-elk) 86 | 87 | 中的 README.md，先到 [docker-elk](https://github.com/twtrubiks/docker-elk-tutorial/tree/master/docker-elk) 目錄底下 88 | 89 | > cd docker-elk 90 | 91 | 直接執行以下指令 92 | 93 | > docker-compose up 94 | 95 | 第一次會比較慢，因為要 pull image 而且還要初始化 :sleeping: 96 | 97 | 這時候可以起來運動一下拉拉筋 :relaxed: 98 | 99 | 也可以用 `docker ps` 確認 docker-elk 都有正常運行 100 | 101 | ![img](https://i.imgur.com/OrprV0K.png) 102 | 103 | [docker-compose.yml](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/docker-elk/docker-compose.yml) 如果沒有特別修改，默認的 port 可參考下方 104 | 105 | ```conf 106 | 5000: Logstash TCP input 107 | 9200: Elasticsearch HTTP 108 | 9300: Elasticsearch TCP transport 109 | 5601: Kibana 110 | ``` 111 | 112 | 以上是預設的，這邊我多加上一個 UDP 的 port 113 | 114 | ```conf 115 | 12201: Logstash UDP input 116 | ``` 117 | 118 | 那要如何加，首先，在 docker-elk/[docker-compose.yml](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/docker-elk/docker-compose.yml) 中加上 `12201:12201/udp` 119 | 120 | ```yml 121 | logstash: 122 | build: 123 | context: logstash/ 124 | volumes: 125 | - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro 126 | - ./logstash/pipeline:/usr/share/logstash/pipeline:ro 127 | ports: 128 | - "5000:5000" 129 | - "12201:12201/udp" 130 | environment: 131 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 132 | networks: 133 | - elk 134 | depends_on: 135 | - elasticsearch 136 | ``` 137 | 138 | 接著在 docker-elk/logstash/pipeline/[logstash.conf](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/docker-elk/logstash/pipeline/logstash.conf) 底下加上 udp 139 | 140 | ```conf 141 | input { 142 | tcp { 143 | port => 5000 144 | } 145 | udp { 146 | port => 12201 147 | } 148 | } 149 | 150 | ## Add your filters / logstash plugins configuration here 151 | 152 | output { 153 | elasticsearch { 154 | hosts => "elasticsearch:9200" 155 | } 156 | } 157 | 158 | ``` 159 | 160 | [logstash.conf](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/docker-elk/logstash/pipeline/logstash.conf) 可以設定的真的非常多，像是 filters ，大家可以自行去摸索，這邊先不介紹:smirk: 161 | 162 | 確認啟動成功後，我們可以先來看看 Elasticsearch，瀏覽 163 | [http://localhost:9200/](http://localhost:9200/) 164 | 165 | ![img](https://i.imgur.com/4FcbOCm.png) 166 | 167 | 接著再來看看 Kibana（ 有時候你會發現無法瀏覽，這是因為還在初始化 ） 168 | 169 | 等待初始化完畢後，可以瀏覽 [http://localhost:5601/](http://localhost:5601/)，你應該會看到 170 | 171 | ![img](https://i.imgur.com/f9aYPd3.png) 172 | 173 | 我們需要先設定 index pattern，MAC 或 Linux 用戶直接使用以下指令 174 | 175 | ```cmd 176 | curl -XPOST -D- "http://localhost:5601/api/saved_objects/index-pattern" \ 177 | -H "Content-Type: application/json" \ 178 | -H "kbn-version: 6.1.0" \ 179 | -d "{'attributes':{'title':'logstash-*','timeFieldName':'@timestamp'}}" 180 | ``` 181 | 182 | 如果你是 Windows 用戶，請用其他方法，雖然 Windows 也有 curl，但我裝上去執行指令， 183 | 184 | 他都會報錯說 josn格式錯誤，所以我直接改用 [Postman](https://www.getpostman.com/) 185 | 186 | ![img](https://i.imgur.com/lHh7thR.png) 187 | 188 | 如果一切順利，你應該會看到 response 189 | 190 | ![img](https://i.imgur.com/ideT84S.png) 191 | 192 | 接著重新整理 [http://localhost:5601/](http://localhost:5601/)，你應該會看到 index pattern 建立成功 193 | 194 | ![img](https://i.imgur.com/qB55XQp.png) 195 | 196 | 接著我們可以嘗試送送看 log , 如果你是 MAC 或 Linux 用戶，你可以使用以下指令 197 | 198 | ```cmd 199 | nc localhost 5000 < README.md 200 | ``` 201 | 202 | 上面這段指令其實只是將 README.md 往 logstash ( [http://localhost:5000/](http://localhost:5000/) ) 送資料， 203 | 204 | 可以透過 Kibana 觀看結果，會發現有一堆 [README.md](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/README.md) 的文字 205 | 206 | ![img](https://i.imgur.com/eSdsdcZ.png) 207 | 208 | 如果你是 Windows 用戶，請跳過這段 :laughing: 209 | 210 | 直接用 python 來測試吧:smirk: 211 | 212 | ## 透過 python 送 log 到 ELK 213 | 214 | * [Youtube Tutorial PART 3 - 透過 python 送 log 到 ELK](https://youtu.be/EpEJGLzIK6A) 215 | 216 | 剛剛簡單的介紹 ELK，現在讓我們透過 python 送 log 到 ELK 吧 :satisfied: 217 | 218 | 建議大家可以先了解一下 python 中的 [logging](https://docs.python.org/3.6/howto/logging.html)， 219 | 220 | 也可以參考這個簡單的範例 [logging_tutorial.py](https://github.com/twtrubiks/python-notes/blob/master/logging_tutorial.py)。 221 | 222 | 要使用 python 發送 log 到 ELK，請先執行下列指令 223 | 224 | [python-logstash](https://github.com/vklochan/python-logstash) 225 | 226 | > pip install python-logstash 227 | 228 | 接著執行以下程式碼 python-logging/[demo_logging.py](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/python-logging/demo_logging.py) 229 | 230 | ```python 231 | import logging 232 | import logstash 233 | import sys 234 | 235 | host = 'localhost' 236 | 237 | test_logger = logging.getLogger('python-logstash-logger') 238 | test_logger.setLevel(logging.INFO) 239 | 240 | # UDP 241 | # test_logger.addHandler(logstash.LogstashHandler(host, 12201, version=1)) 242 | 243 | # TCP 244 | test_logger.addHandler(logstash.TCPLogstashHandler(host, 5000, version=1)) 245 | 246 | test_logger.error('python-logstash: test logstash error message.') 247 | test_logger.info('python-logstash: test logstash info message.') 248 | test_logger.warning('python-logstash: test logstash warning message.') 249 | 250 | # add extra field to logstash message 251 | extra = { 252 | 'test_string': 'python version: ' + repr(sys.version_info), 253 | 'test_boolean': True, 254 | 'test_dict': {'a': 1, 'b': 'c'}, 255 | 'test_float': 1.23, 256 | 'test_integer': 123, 257 | 'test_list': [1, 2, '3'], 258 | } 259 | test_logger.info('python-logstash: test extra fields', extra=extra) 260 | print('done,please see kibana') 261 | ``` 262 | 263 | 接著可以到 Kibana 觀看 264 | 265 | ![img](https://i.imgur.com/mmPfRs6.png) 266 | 267 | log 訊息的確是我們剛剛送出去的 268 | 269 | ![img](https://i.imgur.com/Rm4bVgQ.png) 270 | 271 | 如果你要測試 UDP 的部份，就把 TCP 註解，UDP 打開（ 取消註解 ）， 272 | 273 | 這樣以後我們就可以將我們需要記錄的 log 資料通通都送到 ELK 中管理 :thumbsup: 274 | 275 | ## logging for Django + ELK 276 | 277 | * [Youtube Tutorial PART 4 - logging for Django + ELK](https://youtu.be/_bkx0FfNRpQ) 278 | 279 | 剛剛介紹了如何透過 python 送 log 到 ELK 中，現在要教大家如何在 Django 中設定 logging :smirk: 280 | 281 | 如果不了解什麼是 Django，可參考我之前寫的 [Django 基本教學 - 從無到有 Django-Beginners-Guide 📝](https://github.com/twtrubiks/django-tutorial) 282 | 283 | 一樣請記得安裝 [python-logstash](https://github.com/vklochan/python-logstash) :blush: 284 | 285 | > pip install python-logstash 286 | 287 | 我們就依照 [這篇](https://github.com/twtrubiks/django-tutorial) 的範例繼續介紹， 288 | 289 | 先將 django-tutorial/django-tutorial/[settings.py](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/django-tutorial/django-tutorial/settings.py) 加入下方程式碼 290 | 291 | ```python 292 | LOGGING = { 293 | 'version': 1, 294 | 'disable_existing_loggers': False, 295 | 'formatters': { 296 | 'verbose': { 297 | 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' 298 | }, 299 | 'simple': { 300 | 'format': '%(levelname)s %(message)s' 301 | }, 302 | }, 303 | 'handlers': { 304 | 'console': { 305 | 'level': 'INFO', 306 | 'class': 'logging.StreamHandler', 307 | 'formatter': 'simple' 308 | }, 309 | 'logstash': { 310 | 'level': 'WARNING', 311 | 'class': 'logstash.TCPLogstashHandler', 312 | 'host': 'localhost', 313 | 'port': 5000, # Default value: 5000 314 | 'version': 1, 315 | 'message_type': 'django_logstash', # 'type' field in logstash message. Default value: 'logstash'. 316 | 'fqdn': False, # Fully qualified domain name. Default value: false. 317 | 'tags': ['django.request'], # list of tags. Default: None. 318 | }, 319 | }, 320 | 'loggers': { 321 | 'django': { 322 | 'handlers': ['console'], 323 | 'level': 'INFO', 324 | 'propagate': True, 325 | }, 326 | 'django.request': { 327 | 'handlers': ['logstash'], 328 | 'level': 'WARNING', 329 | 'propagate': True, 330 | }, 331 | } 332 | } 333 | ``` 334 | 335 | 詳細的 django logging 可參考官網 [https://docs.djangoproject.com/en/2.0/topics/logging/](https://docs.djangoproject.com/en/2.0/topics/logging/)， 336 | 337 | 這邊要稍微提一下 [django.request](https://docs.djangoproject.com/en/2.0/topics/logging/#django-request) 338 | 339 | ```txt 340 | django.request 341 | Log messages related to the handling of requests. 342 | 5XX responses are raised as ERROR messages; 343 | 4XX responses are raised as WARNING messages. 344 | ``` 345 | 346 | 接著到 django-tutorial/musics/[views.py](https://github.com/twtrubiks/django-tutorial/blob/master/musics/views.py)中修改程式碼 347 | 348 | ```python 349 | from django.shortcuts import render 350 | 351 | from musics.models import Music 352 | from django.http import Http404 353 | 354 | # Create your views here. 355 | def hello_view(request): 356 | musics = Music.objects.all() 357 | # raise Exception('error !!!!') 358 | # raise Http404("sorry 404") 359 | return render(request, 'hello_django.html', { 360 | 'data': "Hello Django ", 361 | 'musics': musics, 362 | }) 363 | ``` 364 | 365 | 以上註解的兩個地方，可以自行玩玩看，然後到 Kibana 中觀看， 366 | 367 | 如果不太理解，可參考影片的說明 [Youtube Tutorial PART 4 - logging for Django + ELK](https://youtu.be/_bkx0FfNRpQ) 368 | 369 | `raise Exception('error !!!!')` 這行等於是 5XX responses，也就是 ERROR messages， 370 | 371 | `raise Http404("sorry 404")` 這行等於是 ˋXX responses，也就是 WARNING messages。 372 | 373 | ## docker logging + ELK 374 | 375 | * [Youtube Tutorial PART 5 - docker logging + ELK](https://youtu.be/gTqAjea4Ncg) 376 | 377 | 既然都講到這裡了，一定要來說一下如何將 docker 的 log 送到 ELK 中， 378 | 379 | 先來個 tcp 的簡單範例 380 | 381 | ```cmd 382 | docker run --log-driver=syslog --log-opt syslog-address=tcp://0.0.0.0:5000 --log-opt syslog-facility=daemon alpine echo hello world tcp 383 | ``` 384 | 385 | ![img](https://i.imgur.com/CNlAb98.png) 386 | 387 | 到 Kibana 觀看 388 | 389 | ![img](https://i.imgur.com/tWSFH1B.png) 390 | 391 | 再來個 udp 的簡單範例 392 | 393 | ```cmd 394 | docker run --log-driver=gelf --log-opt gelf-address=udp://0.0.0.0:12201 alpine echo hello world udp 395 | ``` 396 | 397 | ![img](https://i.imgur.com/DZpe15V.png) 398 | 399 | 這邊我覺得奇怪的是，如果用 gelf 送出去的 log 都會變成亂碼， 400 | 401 | 如果有人知道原因再請解答:sweat_smile: 402 | 403 | ![img](https://i.imgur.com/TobRFjw.png) 404 | 405 | docker logging 詳細可參考 [https://docs.docker.com/engine/admin/logging/overview/](https://docs.docker.com/engine/admin/logging/overview/) 406 | 407 | 那如果我希望寫在 docker-compose 中呢？ 408 | 409 | 請看 docker-logging/[docker-compose.yml](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/docker-logging/docker-compose.yml) 410 | 411 | ```python 412 | version: '3.3' 413 | services: 414 | 415 | db: 416 | # container_name: 'postgres' 417 | image: postgres 418 | environment: 419 | POSTGRES_PASSWORD: password123 420 | ports: 421 | - "5432:5432" 422 | # (HOST:CONTAINER) 423 | volumes: 424 | - pgdata:/var/lib/postgresql/data/ 425 | 426 | web: 427 | # build: ./api 428 | # command: python manage.py runserver 0.0.0.0:8000 429 | image: twtrubiks/my_django 430 | restart: always 431 | volumes: 432 | - api_data:/docker_api 433 | # (HOST:CONTAINER) 434 | ports: 435 | - "8000:8000" 436 | # (HOST:CONTAINER) 437 | depends_on: 438 | - db 439 | 440 | logging: 441 | driver: syslog 442 | options: 443 | syslog-address: tcp://0.0.0.0:5000 444 | tag: web-container-tcp 445 | 446 | # logging: 447 | # driver: gelf 448 | # options: 449 | # gelf-address: udp://0.0.0.0:12201 450 | # tag: web-container-udp 451 | 452 | volumes: 453 | api_data: 454 | pgdata: 455 | ``` 456 | 457 | 以上這個範例是從 [Docker 基本教學 - 從無到有 Docker-Beginners-Guide](https://github.com/twtrubiks/docker-tutorial)修改過來的， 458 | 459 | 一樣執行 `docker-compose up`， 460 | 461 | ![img](https://i.imgur.com/FjiW7K9.png) 462 | 463 | 接著到 Kibana 中可以看到 log 資訊 464 | 465 | ![img](https://i.imgur.com/Y0F8BSD.png) 466 | 467 | ## 後記： 468 | 469 | 這篇文章主要是帶大家對 ELK 有一些基本的觀念，因為 ELK 可以玩的東西真的非常的多， 470 | 471 | 坑很大，像是前面所說的 [logstash.conf](https://github.com/twtrubiks/docker-elk-tutorial/blob/master/docker-elk/logstash/pipeline/logstash.conf) 中可以設定的參數，像是 filters 之類的....... 472 | 473 | 又或是 Kibana 如何呈現精美的圖表，甚至將 docker-elk 佈署到 Swarm 中，都可以玩玩 474 | 475 | 看，所以大家有興趣可以再自行深入研究:smile: 476 | 477 | 我本來是想要透過 Django 結合 Haystack 做個全文檢索的範例，但因為 Haystack 對於 478 | 479 | ElasticSearch 的版本只支援到 2.X ( ElasticSearch 都出到 6.X 了 )，最後就沒有將這範例 480 | 481 | 寫出來了:sweat_smile: 482 | 483 | [elasticsearch-py](https://github.com/elastic/elasticsearch-py) 這個 library 也可以看看，我用 6.x 版本測試，還是有一點問題，問題 484 | 485 | 如果解決再分享給各位:laughing: 486 | 487 | ## 執行環境 488 | 489 | * Python 3.6.2 490 | 491 | ## Reference 492 | 493 | * [docker-elk](https://github.com/deviantony/docker-elk) 494 | 495 | * [python-logstash](https://github.com/vklochan/python-logstash) 496 | 497 | * [Django](https://www.djangoproject.com/) 498 | 499 | ## Donation 500 | 501 | 文章都是我自己研究內化後原創，如果有幫助到您，也想鼓勵我的話，歡迎請我喝一杯咖啡:laughing: 502 | 503 | ![alt tag](https://i.imgur.com/LRct9xa.png) 504 | 505 | [贊助者付款](https://payment.opay.tw/Broadcaster/Donate/9E47FDEF85ABE383A0F5FC6A218606F8) 506 | 507 | ## License 508 | 509 | MIT license 510 | -------------------------------------------------------------------------------- /django-tutorial/db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twtrubiks/docker-elk-tutorial/5870c1367fd88d61e816fe715dbf26dc8923cfc7/django-tutorial/db.sqlite3 -------------------------------------------------------------------------------- /django-tutorial/django_tutorial/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twtrubiks/docker-elk-tutorial/5870c1367fd88d61e816fe715dbf26dc8923cfc7/django-tutorial/django_tutorial/__init__.py -------------------------------------------------------------------------------- /django-tutorial/django_tutorial/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for django_tutorial project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.10.6. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.10/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = '&2y%^#rywhy^rpxd8y%a0rpt7z6ed6lt4+7s=nwqx!b5t5nr5%' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'musics' 41 | ] 42 | 43 | MIDDLEWARE = [ 44 | 'django.middleware.security.SecurityMiddleware', 45 | 'django.contrib.sessions.middleware.SessionMiddleware', 46 | 'django.middleware.common.CommonMiddleware', 47 | 'django.middleware.csrf.CsrfViewMiddleware', 48 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 49 | 'django.contrib.messages.middleware.MessageMiddleware', 50 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 51 | ] 52 | 53 | ROOT_URLCONF = 'django_tutorial.urls' 54 | 55 | TEMPLATES = [ 56 | { 57 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 58 | 'DIRS': [os.path.join(BASE_DIR, 'templates')] 59 | , 60 | 'APP_DIRS': True, 61 | 'OPTIONS': { 62 | 'context_processors': [ 63 | 'django.template.context_processors.debug', 64 | 'django.template.context_processors.request', 65 | 'django.contrib.auth.context_processors.auth', 66 | 'django.contrib.messages.context_processors.messages', 67 | ], 68 | }, 69 | }, 70 | ] 71 | 72 | WSGI_APPLICATION = 'django_tutorial.wsgi.application' 73 | 74 | 75 | # Database 76 | # https://docs.djangoproject.com/en/1.10/ref/settings/#databases 77 | 78 | DATABASES = { 79 | 'default': { 80 | 'ENGINE': 'django.db.backends.sqlite3', 81 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 82 | } 83 | } 84 | 85 | 86 | LOGGING = { 87 | 'version': 1, 88 | 'disable_existing_loggers': False, 89 | 'formatters': { 90 | 'verbose': { 91 | 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' 92 | }, 93 | 'simple': { 94 | 'format': '%(levelname)s %(message)s' 95 | }, 96 | }, 97 | 'handlers': { 98 | 'console': { 99 | 'level': 'INFO', 100 | 'class': 'logging.StreamHandler', 101 | 'formatter': 'simple' 102 | }, 103 | 'logstash': { 104 | 'level': 'WARNING', 105 | 'class': 'logstash.TCPLogstashHandler', 106 | 'host': 'localhost', 107 | 'port': 5000, # Default value: 5000 108 | 'version': 1, 109 | 'message_type': 'django_logstash', # 'type' field in logstash message. Default value: 'logstash'. 110 | 'fqdn': False, # Fully qualified domain name. Default value: false. 111 | 'tags': ['django.request'], # list of tags. Default: None. 112 | }, 113 | }, 114 | 'loggers': { 115 | 'django': { 116 | 'handlers': ['console'], 117 | 'level': 'INFO', 118 | 'propagate': True, 119 | }, 120 | 'django.request': { 121 | 'handlers': ['logstash'], 122 | 'level': 'WARNING', 123 | 'propagate': True, 124 | }, 125 | } 126 | } 127 | 128 | # Password validation 129 | # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators 130 | 131 | AUTH_PASSWORD_VALIDATORS = [ 132 | { 133 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 134 | }, 135 | { 136 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 137 | }, 138 | { 139 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 140 | }, 141 | { 142 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 143 | }, 144 | ] 145 | 146 | 147 | # Internationalization 148 | # https://docs.djangoproject.com/en/1.10/topics/i18n/ 149 | 150 | LANGUAGE_CODE = 'en-us' 151 | 152 | TIME_ZONE = 'UTC' 153 | 154 | USE_I18N = True 155 | 156 | USE_L10N = True 157 | 158 | USE_TZ = True 159 | 160 | 161 | # Static files (CSS, JavaScript, Images) 162 | # https://docs.djangoproject.com/en/1.10/howto/static-files/ 163 | 164 | STATIC_URL = '/static/' 165 | -------------------------------------------------------------------------------- /django-tutorial/django_tutorial/urls.py: -------------------------------------------------------------------------------- 1 | """django_tutorial URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.10/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import url 17 | from django.contrib import admin 18 | from musics.views import hello_view 19 | 20 | urlpatterns = [ 21 | url(r'^admin/', admin.site.urls), 22 | url(r'^hello/', hello_view), 23 | ] 24 | -------------------------------------------------------------------------------- /django-tutorial/django_tutorial/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for django_tutorial project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_tutorial.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /django-tutorial/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_tutorial.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /django-tutorial/musics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twtrubiks/docker-elk-tutorial/5870c1367fd88d61e816fe715dbf26dc8923cfc7/django-tutorial/musics/__init__.py -------------------------------------------------------------------------------- /django-tutorial/musics/admin.py: -------------------------------------------------------------------------------- 1 | # Register your models here. 2 | from django.contrib import admin 3 | from musics.models import Music 4 | 5 | admin.site.register(Music) 6 | -------------------------------------------------------------------------------- /django-tutorial/musics/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class MusicsConfig(AppConfig): 5 | name = 'musics' 6 | -------------------------------------------------------------------------------- /django-tutorial/musics/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | TYPE_CHOICES = ( 4 | ('T1', 'type 1'), 5 | ('T2', 'type 2'), 6 | ('T3', 'type 3'), 7 | ('T4', 'type 4'), 8 | ) 9 | 10 | 11 | # Create your models here. 12 | class Music(models.Model): 13 | song = models.TextField(default="song") 14 | singer = models.TextField(default="AKB48") 15 | last_modify_date = models.DateTimeField(auto_now=True) 16 | created = models.DateTimeField(auto_now_add=True) 17 | type = models.CharField( 18 | max_length=2, 19 | choices=TYPE_CHOICES, 20 | default="T1" 21 | ) 22 | 23 | class Meta: 24 | db_table = "music" 25 | 26 | def display_type_name(self): 27 | return self.get_type_display() 28 | -------------------------------------------------------------------------------- /django-tutorial/musics/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /django-tutorial/musics/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | 3 | from musics.models import Music 4 | from django.http import Http404 5 | 6 | 7 | # Create your views here. 8 | def hello_view(request): 9 | musics = Music.objects.all() 10 | # raise Exception('error !!!!') 11 | # raise Http404("sorry 404") 12 | return render(request, 'hello_django.html', { 13 | 'data': "Hello Django ", 14 | 'musics': musics, 15 | }) 16 | -------------------------------------------------------------------------------- /django-tutorial/templates/hello_django.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | {{ data }} 9 | {% for music in musics %} 10 |

id : {{ music.id }}

11 |

song : {{ music.song }}

12 |

singer : {{ music.singer }}

13 |

type : {{ music.type }}

14 |

display_type_name : {{ music.display_type_name }}

15 | {% endfor %} 16 | 17 | 18 | -------------------------------------------------------------------------------- /docker-elk/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | elasticsearch: 6 | build: 7 | context: elasticsearch/ 8 | volumes: 9 | - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro 10 | ports: 11 | - "9200:9200" 12 | - "9300:9300" 13 | environment: 14 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 15 | networks: 16 | - elk 17 | 18 | logstash: 19 | build: 20 | context: logstash/ 21 | volumes: 22 | - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro 23 | - ./logstash/pipeline:/usr/share/logstash/pipeline:ro 24 | ports: 25 | - "5000:5000" 26 | - "12201:12201/udp" 27 | environment: 28 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 29 | networks: 30 | - elk 31 | depends_on: 32 | - elasticsearch 33 | 34 | kibana: 35 | build: 36 | context: kibana/ 37 | volumes: 38 | - ./kibana/config/:/usr/share/kibana/config:ro 39 | ports: 40 | - "5601:5601" 41 | networks: 42 | - elk 43 | depends_on: 44 | - elasticsearch 45 | 46 | networks: 47 | 48 | elk: 49 | driver: bridge 50 | -------------------------------------------------------------------------------- /docker-elk/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/elasticsearch-docker 2 | FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.0 3 | 4 | # Add your elasticsearch plugins setup here 5 | # Example: RUN elasticsearch-plugin install analysis-icu 6 | -------------------------------------------------------------------------------- /docker-elk/elasticsearch/config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Elasticsearch configuration from elasticsearch-docker. 3 | ## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml 4 | # 5 | cluster.name: "docker-cluster" 6 | network.host: 0.0.0.0 7 | 8 | # minimum_master_nodes need to be explicitly set when bound on a public IP 9 | # set to 1 to allow single node clusters 10 | # Details: https://github.com/elastic/elasticsearch/pull/17288 11 | discovery.zen.minimum_master_nodes: 1 12 | 13 | ## Use single node discovery in order to disable production mode and avoid bootstrap checks 14 | ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html 15 | # 16 | discovery.type: single-node 17 | -------------------------------------------------------------------------------- /docker-elk/kibana/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/kibana-docker 2 | FROM docker.elastic.co/kibana/kibana-oss:6.1.0 3 | 4 | # Add your kibana plugins setup here 5 | # Example: RUN kibana-plugin install 6 | -------------------------------------------------------------------------------- /docker-elk/kibana/config/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Kibana configuration from kibana-docker. 3 | ## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml 4 | # 5 | server.name: kibana 6 | server.host: "0" 7 | elasticsearch.url: http://elasticsearch:9200 8 | -------------------------------------------------------------------------------- /docker-elk/logstash/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/logstash-docker 2 | FROM docker.elastic.co/logstash/logstash-oss:6.1.0 3 | 4 | # Add your logstash plugins setup here 5 | # Example: RUN logstash-plugin install logstash-filter-json 6 | -------------------------------------------------------------------------------- /docker-elk/logstash/config/logstash.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Logstash configuration from logstash-docker. 3 | ## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml 4 | # 5 | http.host: "0.0.0.0" 6 | path.config: /usr/share/logstash/pipeline -------------------------------------------------------------------------------- /docker-elk/logstash/pipeline/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => 5000 4 | } 5 | udp { 6 | port => 12201 7 | } 8 | } 9 | 10 | ## Add your filters / logstash plugins configuration here 11 | 12 | output { 13 | elasticsearch { 14 | hosts => "elasticsearch:9200" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /docker-logging/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | services: 3 | 4 | db: 5 | # container_name: 'postgres' 6 | image: postgres 7 | environment: 8 | POSTGRES_PASSWORD: password123 9 | ports: 10 | - "5432:5432" 11 | # (HOST:CONTAINER) 12 | volumes: 13 | - pgdata:/var/lib/postgresql/data/ 14 | 15 | web: 16 | # build: ./api 17 | # command: python manage.py runserver 0.0.0.0:8000 18 | image: twtrubiks/my_django 19 | restart: always 20 | volumes: 21 | - api_data:/docker_api 22 | # (HOST:CONTAINER) 23 | ports: 24 | - "8000:8000" 25 | # (HOST:CONTAINER) 26 | depends_on: 27 | - db 28 | 29 | logging: 30 | driver: syslog 31 | options: 32 | syslog-address: tcp://0.0.0.0:5000 33 | tag: web-container-tcp 34 | 35 | # logging: 36 | # driver: gelf 37 | # options: 38 | # gelf-address: udp://0.0.0.0:12201 39 | # tag: web-container-udp 40 | 41 | volumes: 42 | api_data: 43 | pgdata: -------------------------------------------------------------------------------- /python-logging/demo_logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logstash 3 | import sys 4 | 5 | host = 'localhost' 6 | 7 | test_logger = logging.getLogger('python-logstash-logger') 8 | test_logger.setLevel(logging.INFO) 9 | 10 | # UDP 11 | # test_logger.addHandler(logstash.LogstashHandler(host, 12201, version=1)) 12 | 13 | # TCP 14 | test_logger.addHandler(logstash.TCPLogstashHandler(host, 5000, version=1)) 15 | 16 | test_logger.error('python-logstash: test logstash error message.') 17 | test_logger.info('python-logstash: test logstash info message.') 18 | test_logger.warning('python-logstash: test logstash warning message.') 19 | 20 | # add extra field to logstash message 21 | extra = { 22 | 'test_string': 'python version: ' + repr(sys.version_info), 23 | 'test_boolean': True, 24 | 'test_dict': {'a': 1, 'b': 'c'}, 25 | 'test_float': 1.23, 26 | 'test_integer': 123, 27 | 'test_list': [1, 2, '3'], 28 | } 29 | test_logger.info('python-logstash: test extra fields', extra=extra) 30 | print('done,please see kibana') 31 | --------------------------------------------------------------------------------