├── .env ├── .gitignore ├── README.md ├── app ├── Dockerfile ├── app.py ├── log_config.py └── requirements.txt ├── docker-compose.yml └── docker ├── README.md ├── elasticsearch ├── Dockerfile └── config │ └── elasticsearch.yml ├── kibana ├── Dockerfile └── config │ └── kibana.yml └── logstash ├── Dockerfile ├── config └── logstash.yml └── pipeline └── logstash.conf /.env: -------------------------------------------------------------------------------- 1 | ELK_VERSION=6.7.0 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | __pycache__/ 3 | 4 | # Dev 5 | 6 | .vscode 7 | .python-version 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Structlog + ELK stack 2 | 3 | A proof of concept of structured logging from Python to an Elasticsearch + Logstash + Kibana environment using [Docker ELK](https://github.com/deviantony/docker-elk), [structlog](https://www.structlog.org/) and [python-logstash](https://github.com/vklochan/python-logstash). 4 | 5 | ## Setup 6 | 7 | 1. `docker-compose up`, and wait for a bit, the app will start up first, then Elasticsearch and Kibana and finally Logstash 8 | 2. As described in the [docker-elk initial setup](https://github.com/deviantony/docker-elk#default-kibana-index-pattern-creation) run the fllowing command to create the initial logstash index 9 | 10 | ``` 11 | $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ 12 | -H 'Content-Type: application/json' \ 13 | -H 'kbn-version: 6.1.0' \ 14 | -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' 15 | ``` 16 | 17 | 3. `curl http://localhost:8888/`, should print "Hello, World" and forward a first entry. You should see Logstash output in the terminal. 18 | 4. Visit [`http://localhost:5601/`](http://localhost:5601/) to open up Kibana, click on Discover and you should see two entries under a `logstash-*` index. 19 | 5. Visit [`http://localhost:8888/boom/`](http://localhost:8888/boom/) to log an exception. 20 | -------------------------------------------------------------------------------- /app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7.3-stretch 2 | RUN mkdir /opt/app 3 | WORKDIR /opt/app 4 | RUN pip install -U pip setuptools wheel 5 | RUN python3 -m venv /opt/app 6 | ENV PATH="/opt/app/bin:${PATH}" 7 | COPY requirements.txt /tmp/requirements.txt 8 | RUN pip install -r /tmp/requirements.txt 9 | COPY . /opt/app 10 | -------------------------------------------------------------------------------- /app/app.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | 3 | from flask import Flask 4 | import structlog 5 | 6 | from log_config import configure_logging 7 | 8 | app = Flask(__name__) 9 | # should be called once on app initialisation 10 | configure_logging() 11 | logger = structlog.get_logger() 12 | 13 | 14 | @app.route('/') 15 | def root(): 16 | # binds `request_id` to the current context 17 | log = logger.new(request_id=str(uuid4())) 18 | helper_function() 19 | log.info('an info', source='root') 20 | return 'Hello, world!' 21 | 22 | 23 | def helper_function(): 24 | # binds `helper_key: helper_value` to the current context 25 | logger.bind(helper_key='helper_value') 26 | # an event is logged with a message, optional keys as kwargs and context 27 | logger.warning('a warning', source='helper') 28 | 29 | 30 | @app.errorhandler(Exception) 31 | def base_error_handler(error): 32 | # exception logging includes traceback information 33 | logger.exception(error) 34 | return 'Woops, our bad, sorry!', 500 35 | 36 | 37 | @app.route('/boom/') 38 | def boom(): 39 | log = logger.new(request_id=str(uuid4())) 40 | log.info('About to raise') 41 | raise Exception('Test exception') 42 | 43 | 44 | if __name__ == '__main__': 45 | app.run() 46 | -------------------------------------------------------------------------------- /app/log_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import structlog 4 | import logstash 5 | 6 | 7 | def configure_logging(): 8 | structlog.configure( 9 | processors=[ 10 | structlog.stdlib.filter_by_level, 11 | structlog.stdlib.add_log_level, 12 | structlog.stdlib.PositionalArgumentsFormatter(), 13 | structlog.processors.StackInfoRenderer(), 14 | structlog.processors.format_exc_info, 15 | structlog.processors.UnicodeDecoder(), 16 | # required as LogstashHandler uses `extra` for logging JSON values 17 | structlog.stdlib.render_to_log_kwargs, 18 | ], 19 | # required to mimic Flask's threadlocal context allowing 20 | # logging during the whole request process 21 | context_class=structlog.threadlocal.wrap_dict(dict), 22 | logger_factory=structlog.stdlib.LoggerFactory(), 23 | wrapper_class=structlog.stdlib.BoundLogger, 24 | cache_logger_on_first_use=True, 25 | ) 26 | 27 | stream_handler = logging.StreamHandler() 28 | stream_handler.setFormatter( 29 | structlog.stdlib.ProcessorFormatter( 30 | processor=structlog.dev.ConsoleRenderer()) 31 | ) 32 | 33 | app_logger = logging.getLogger('app') 34 | app_logger.addHandler(stream_handler) 35 | app_logger.setLevel(logging.INFO) 36 | 37 | # for TCP use TCPLogstashHandler and port 5000 38 | logstash_handler = logstash.LogstashHandler( 39 | 'logstash', 5959, version=1) 40 | app_logger.addHandler(logstash_handler) 41 | -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | structlog[dev] 3 | python-logstash 4 | colorama==0.3.9 5 | twisted 6 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | elasticsearch: 6 | build: 7 | context: docker/elasticsearch/ 8 | args: 9 | ELK_VERSION: $ELK_VERSION 10 | volumes: 11 | - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro 12 | ports: 13 | - "9200:9200" 14 | - "9300:9300" 15 | environment: 16 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 17 | networks: 18 | - elk 19 | 20 | logstash: 21 | build: 22 | context: docker/logstash/ 23 | args: 24 | ELK_VERSION: $ELK_VERSION 25 | volumes: 26 | - ./docker/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro 27 | - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro 28 | ports: 29 | - "5000:5000" 30 | - "9600:9600" 31 | environment: 32 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 33 | networks: 34 | - elk 35 | depends_on: 36 | - elasticsearch 37 | 38 | kibana: 39 | build: 40 | context: docker/kibana/ 41 | args: 42 | ELK_VERSION: $ELK_VERSION 43 | volumes: 44 | - ./docker/kibana/config/:/usr/share/kibana/config:ro 45 | ports: 46 | - "5601:5601" 47 | networks: 48 | - elk 49 | depends_on: 50 | - elasticsearch 51 | 52 | app: 53 | build: 54 | context: app/ 55 | ports: 56 | - "8888:8888" 57 | networks: 58 | - elk 59 | depends_on: 60 | - logstash 61 | command: python -m twisted --log-format text web -p tcp:port=8888 --wsgi app.app 62 | 63 | networks: 64 | 65 | elk: 66 | driver: bridge 67 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Docker ELK stack 2 | 3 | [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | [![Elastic Stack version](https://img.shields.io/badge/ELK-6.1.0-blue.svg?style=flat)](https://github.com/deviantony/docker-elk/issues/212) 5 | [![Build Status](https://api.travis-ci.org/deviantony/docker-elk.svg?branch=master)](https://travis-ci.org/deviantony/docker-elk) 6 | 7 | Run the latest version of the ELK (Elasticsearch, Logstash, Kibana) stack with Docker and Docker Compose. 8 | 9 | It will give you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch 10 | and the visualization power of Kibana. 11 | 12 | Based on the official Docker images: 13 | 14 | * [elasticsearch](https://github.com/elastic/elasticsearch-docker) 15 | * [logstash](https://github.com/elastic/logstash-docker) 16 | * [kibana](https://github.com/elastic/kibana-docker) 17 | 18 | **Note**: Other branches in this project are available: 19 | 20 | * ELK 6 with X-Pack support: https://github.com/deviantony/docker-elk/tree/x-pack 21 | * ELK 6 in Vagrant: https://github.com/deviantony/docker-elk/tree/vagrant 22 | * ELK 6 with Search Guard: https://github.com/deviantony/docker-elk/tree/searchguard 23 | 24 | ## Contents 25 | 26 | 1. [Requirements](#requirements) 27 | * [Host setup](#host-setup) 28 | * [SELinux](#selinux) 29 | 2. [Getting started](#getting-started) 30 | * [Bringing up the stack](#bringing-up-the-stack) 31 | * [Initial setup](#initial-setup) 32 | 3. [Configuration](#configuration) 33 | * [How can I tune the Kibana configuration?](#how-can-i-tune-the-kibana-configuration) 34 | * [How can I tune the Logstash configuration?](#how-can-i-tune-the-logstash-configuration) 35 | * [How can I tune the Elasticsearch configuration?](#how-can-i-tune-the-elasticsearch-configuration) 36 | * [How can I scale out the Elasticsearch cluster?](#how-can-i-scale-up-the-elasticsearch-cluster) 37 | 4. [Storage](#storage) 38 | * [How can I persist Elasticsearch data?](#how-can-i-persist-elasticsearch-data) 39 | 5. [Extensibility](#extensibility) 40 | * [How can I add plugins?](#how-can-i-add-plugins) 41 | * [How can I enable the provided extensions?](#how-can-i-enable-the-provided-extensions) 42 | 6. [JVM tuning](#jvm-tuning) 43 | * [How can I specify the amount of memory used by a service?](#how-can-i-specify-the-amount-of-memory-used-by-a-service) 44 | * [How can I enable a remote JMX connection to a service?](#how-can-i-enable-a-remote-jmx-connection-to-a-service) 45 | 46 | ## Requirements 47 | 48 | ### Host setup 49 | 50 | 1. Install [Docker](https://www.docker.com/community-edition#/download) version **1.10.0+** 51 | 2. Install [Docker Compose](https://docs.docker.com/compose/install/) version **1.6.0+** 52 | 3. Clone this repository 53 | 54 | ### SELinux 55 | 56 | On distributions which have SELinux enabled out-of-the-box you will need to either re-context the files or set SELinux 57 | into Permissive mode in order for docker-elk to start properly. For example on Redhat and CentOS, the following will 58 | apply the proper context: 59 | 60 | ```console 61 | $ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ 62 | ``` 63 | 64 | ## Usage 65 | 66 | ### Bringing up the stack 67 | 68 | **Note**: In case you switched branch or updated a base image - you may need to run `docker-compose build` first 69 | 70 | Start the ELK stack using `docker-compose`: 71 | 72 | ```console 73 | $ docker-compose up 74 | ``` 75 | 76 | You can also choose to run it in background (detached mode): 77 | 78 | ```console 79 | $ docker-compose up -d 80 | ``` 81 | 82 | Give Kibana a few seconds to initialize, then access the Kibana web UI by hitting 83 | [http://localhost:5601](http://localhost:5601) with a web browser. 84 | 85 | By default, the stack exposes the following ports: 86 | * 5000: Logstash TCP input. 87 | * 9200: Elasticsearch HTTP 88 | * 9300: Elasticsearch TCP transport 89 | * 5601: Kibana 90 | 91 | **WARNING**: If you're using `boot2docker`, you must access it via the `boot2docker` IP address instead of `localhost`. 92 | 93 | **WARNING**: If you're using *Docker Toolbox*, you must access it via the `docker-machine` IP address instead of 94 | `localhost`. 95 | 96 | Now that the stack is running, you will want to inject some log entries. The shipped Logstash configuration allows you 97 | to send content via TCP: 98 | 99 | ```console 100 | $ nc localhost 5000 < /path/to/logfile.log 101 | ``` 102 | 103 | ## Initial setup 104 | 105 | ### Default Kibana index pattern creation 106 | 107 | When Kibana launches for the first time, it is not configured with any index pattern. 108 | 109 | #### Via the Kibana web UI 110 | 111 | **NOTE**: You need to inject data into Logstash before being able to configure a Logstash index pattern via the Kibana web 112 | UI. Then all you have to do is hit the *Create* button. 113 | 114 | Refer to [Connect Kibana with 115 | Elasticsearch](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html) for detailed instructions 116 | about the index pattern configuration. 117 | 118 | #### On the command line 119 | 120 | Create an index pattern via the Kibana API: 121 | 122 | ```console 123 | $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ 124 | -H 'Content-Type: application/json' \ 125 | -H 'kbn-version: 6.1.0' \ 126 | -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' 127 | ``` 128 | 129 | The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the first time. 130 | 131 | ## Configuration 132 | 133 | **NOTE**: Configuration is not dynamically reloaded, you will need to restart the stack after any change in the 134 | configuration of a component. 135 | 136 | ### How can I tune the Kibana configuration? 137 | 138 | The Kibana default configuration is stored in `kibana/config/kibana.yml`. 139 | 140 | It is also possible to map the entire `config` directory instead of a single file. 141 | 142 | ### How can I tune the Logstash configuration? 143 | 144 | The Logstash configuration is stored in `logstash/config/logstash.yml`. 145 | 146 | It is also possible to map the entire `config` directory instead of a single file, however you must be aware that 147 | Logstash will be expecting a 148 | [`log4j2.properties`](https://github.com/elastic/logstash-docker/tree/master/build/logstash/config) file for its own 149 | logging. 150 | 151 | ### How can I tune the Elasticsearch configuration? 152 | 153 | The Elasticsearch configuration is stored in `elasticsearch/config/elasticsearch.yml`. 154 | 155 | You can also specify the options you want to override directly via environment variables: 156 | 157 | ```yml 158 | elasticsearch: 159 | 160 | environment: 161 | network.host: "_non_loopback_" 162 | cluster.name: "my-cluster" 163 | ``` 164 | 165 | ### How can I scale out the Elasticsearch cluster? 166 | 167 | Follow the instructions from the Wiki: [Scaling out 168 | Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster) 169 | 170 | ## Storage 171 | 172 | ### How can I persist Elasticsearch data? 173 | 174 | The data stored in Elasticsearch will be persisted after container reboot but not after container removal. 175 | 176 | In order to persist Elasticsearch data even after removing the Elasticsearch container, you'll have to mount a volume on 177 | your Docker host. Update the `elasticsearch` service declaration to: 178 | 179 | ```yml 180 | elasticsearch: 181 | 182 | volumes: 183 | - /path/to/storage:/usr/share/elasticsearch/data 184 | ``` 185 | 186 | This will store Elasticsearch data inside `/path/to/storage`. 187 | 188 | **NOTE:** beware of these OS-specific considerations: 189 | * **Linux:** the [unprivileged `elasticsearch` user][esuser] is used within the Elasticsearch image, therefore the 190 | mounted data directory must be owned by the uid `1000`. 191 | * **macOS:** the default Docker for Mac configuration allows mounting files from `/Users/`, `/Volumes/`, `/private/`, 192 | and `/tmp` exclusively. Follow the instructions from the [documentation][macmounts] to add more locations. 193 | 194 | [esuser]: https://github.com/elastic/elasticsearch-docker/blob/016bcc9db1dd97ecd0ff60c1290e7fa9142f8ddd/templates/Dockerfile.j2#L22 195 | [macmounts]: https://docs.docker.com/docker-for-mac/osxfs/ 196 | 197 | ## Extensibility 198 | 199 | ### How can I add plugins? 200 | 201 | To add plugins to any ELK component you have to: 202 | 203 | 1. Add a `RUN` statement to the corresponding `Dockerfile` (eg. `RUN logstash-plugin install logstash-filter-json`) 204 | 2. Add the associated plugin code configuration to the service configuration (eg. Logstash input/output) 205 | 3. Rebuild the images using the `docker-compose build` command 206 | 207 | ### How can I enable the provided extensions? 208 | 209 | A few extensions are available inside the [`extensions`](extensions) directory. These extensions provide features which 210 | are not part of the standard Elastic stack, but can be used to enrich it with extra integrations. 211 | 212 | The documentation for these extensions is provided inside each individual subdirectory, on a per-extension basis. Some 213 | of them require manual changes to the default ELK configuration. 214 | 215 | ## JVM tuning 216 | 217 | ### How can I specify the amount of memory used by a service? 218 | 219 | By default, both Elasticsearch and Logstash start with [1/4 of the total host 220 | memory](https://docs.oracle.com/javase/8/docs/technotes/guides/vm/gctuning/parallel.html#default_heap_size) allocated to 221 | the JVM Heap Size. 222 | 223 | The startup scripts for Elasticsearch and Logstash can append extra JVM options from the value of an environment 224 | variable, allowing the user to adjust the amount of memory that can be used by each component: 225 | 226 | | Service | Environment variable | 227 | |---------------|----------------------| 228 | | Elasticsearch | ES_JAVA_OPTS | 229 | | Logstash | LS_JAVA_OPTS | 230 | 231 | To accomodate environments where memory is scarce (Docker for Mac has only 2 GB available by default), the Heap Size 232 | allocation is capped by default to 256MB per service in the `docker-compose.yml` file. If you want to override the 233 | default JVM configuration, edit the matching environment variable(s) in the `docker-compose.yml` file. 234 | 235 | For example, to increase the maximum JVM Heap Size for Logstash: 236 | 237 | ```yml 238 | logstash: 239 | 240 | environment: 241 | LS_JAVA_OPTS: "-Xmx1g -Xms1g" 242 | ``` 243 | 244 | ### How can I enable a remote JMX connection to a service? 245 | 246 | As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the docker 247 | host. 248 | 249 | Update the `{ES,LS}_JAVA_OPTS` environment variable with the following content (I've mapped the JMX service on the port 250 | 18080, you can change that). Do not forget to update the `-Djava.rmi.server.hostname` option with the IP address of your 251 | Docker host (replace **DOCKER_HOST_IP**): 252 | 253 | ```yml 254 | logstash: 255 | 256 | environment: 257 | LS_JAVA_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=18080 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=DOCKER_HOST_IP -Dcom.sun.management.jmxremote.local.only=false" 258 | ``` 259 | -------------------------------------------------------------------------------- /docker/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ELK_VERSION 2 | 3 | # https://github.com/elastic/elasticsearch-docker 4 | FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION} 5 | 6 | # Add your elasticsearch plugins setup here 7 | # Example: RUN elasticsearch-plugin install analysis-icu 8 | -------------------------------------------------------------------------------- /docker/elasticsearch/config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Elasticsearch configuration from elasticsearch-docker. 3 | ## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml 4 | # 5 | cluster.name: "docker-cluster" 6 | network.host: 0.0.0.0 7 | 8 | # minimum_master_nodes need to be explicitly set when bound on a public IP 9 | # set to 1 to allow single node clusters 10 | # Details: https://github.com/elastic/elasticsearch/pull/17288 11 | discovery.zen.minimum_master_nodes: 1 12 | 13 | ## Use single node discovery in order to disable production mode and avoid bootstrap checks 14 | ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html 15 | # 16 | discovery.type: single-node 17 | -------------------------------------------------------------------------------- /docker/kibana/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ELK_VERSION 2 | 3 | # https://github.com/elastic/kibana-docker 4 | FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION} 5 | 6 | # Add your kibana plugins setup here 7 | # Example: RUN kibana-plugin install 8 | -------------------------------------------------------------------------------- /docker/kibana/config/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Kibana configuration from kibana-docker. 3 | ## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml 4 | # 5 | server.name: kibana 6 | server.host: "0" 7 | elasticsearch.url: http://elasticsearch:9200 8 | -------------------------------------------------------------------------------- /docker/logstash/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ELK_VERSION 2 | 3 | # https://github.com/elastic/logstash-docker 4 | FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} 5 | 6 | # Add your logstash plugins setup here 7 | # Example: RUN logstash-plugin install logstash-filter-json 8 | -------------------------------------------------------------------------------- /docker/logstash/config/logstash.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Logstash configuration from logstash-docker. 3 | ## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml 4 | # 5 | http.host: "0.0.0.0" 6 | path.config: /usr/share/logstash/pipeline 7 | -------------------------------------------------------------------------------- /docker/logstash/pipeline/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => 5000 4 | codec => json 5 | } 6 | udp { 7 | port => 5959 8 | codec => json 9 | } 10 | } 11 | 12 | ## Add your filters / logstash plugins configuration here 13 | 14 | output { 15 | elasticsearch { 16 | hosts => "elasticsearch:9200" 17 | } 18 | stdout { 19 | codec => rubydebug 20 | } 21 | } 22 | --------------------------------------------------------------------------------