├── elasticsearch ├── config │ └── .placeholder └── Dockerfile ├── .gitattributes ├── kibana ├── Dockerfile └── config │ └── kibana.yml ├── logstash ├── Dockerfile ├── pipeline │ └── logstash.conf └── config │ └── logstash.yml ├── LICENSE ├── docker-compose.yml └── README.md /elasticsearch/config/.placeholder: -------------------------------------------------------------------------------- 1 | Ensure the existence of the parent folder. 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Declare files that will always have LF line endings on checkout. 2 | *.sh text eol=lf -------------------------------------------------------------------------------- /kibana/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/kibana-docker 2 | FROM docker.elastic.co/kibana/kibana:5.2.2 3 | 4 | # Add your kibana plugins setup here 5 | # Example: RUN kibana-plugin install 6 | -------------------------------------------------------------------------------- /logstash/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/logstash-docker 2 | FROM docker.elastic.co/logstash/logstash:5.2.2 3 | 4 | # Add your logstash plugins setup here 5 | # Example: RUN logstash-plugin install logstash-filter-json 6 | -------------------------------------------------------------------------------- /elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/elasticsearch-docker 2 | FROM docker.elastic.co/elasticsearch/elasticsearch:5.2.2 3 | 4 | # Add your elasticsearch plugins setup here 5 | # Example: RUN elasticsearch-plugin install analysis-icu 6 | -------------------------------------------------------------------------------- /logstash/pipeline/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => 5000 4 | } 5 | } 6 | 7 | ## Add your filters / logstash plugins configuration here 8 | 9 | output { 10 | elasticsearch { 11 | hosts => "elasticsearch:9200" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /logstash/config/logstash.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Logstash configuration from logstash-docker. 3 | ## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml 4 | # 5 | http.host: "0.0.0.0" 6 | 7 | ## Disable X-Pack 8 | ## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html 9 | ## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling 10 | # 11 | xpack.monitoring.enabled: false 12 | -------------------------------------------------------------------------------- /kibana/config/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Kibana configuration from kibana-docker. 3 | ## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml 4 | # 5 | server.name: kibana 6 | server.host: "0" 7 | elasticsearch.url: http://elasticsearch:9200 8 | 9 | ## Disable X-Pack 10 | ## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html 11 | ## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling 12 | # 13 | xpack.security.enabled: false 14 | xpack.monitoring.enabled: false 15 | xpack.graph.enabled: false 16 | xpack.reporting.enabled: false 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Anthony Lapenna 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | elasticsearch: 6 | build: elasticsearch/ 7 | ports: 8 | - "9200:9200" 9 | - "9300:9300" 10 | environment: 11 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 12 | # disable X-Pack 13 | # see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html 14 | # https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling 15 | xpack.security.enabled: "false" 16 | xpack.monitoring.enabled: "false" 17 | xpack.graph.enabled: "false" 18 | xpack.watcher.enabled: "false" 19 | networks: 20 | - elk 21 | 22 | logstash: 23 | build: logstash/ 24 | volumes: 25 | - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml 26 | - ./logstash/pipeline:/usr/share/logstash/pipeline 27 | ports: 28 | - "5000:5000" 29 | environment: 30 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 31 | networks: 32 | - elk 33 | depends_on: 34 | - elasticsearch 35 | 36 | kibana: 37 | build: kibana/ 38 | volumes: 39 | - ./kibana/config/:/usr/share/kibana/config 40 | ports: 41 | - "5601:5601" 42 | networks: 43 | - elk 44 | depends_on: 45 | - elasticsearch 46 | 47 | networks: 48 | 49 | elk: 50 | driver: bridge 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker ELK stack 2 | 3 | [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | 5 | Run the latest version of the ELK (Elasticsearch, Logstash, Kibana) stack with Docker and Docker-compose. 6 | 7 | It will give you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and the visualization power of Kibana. 8 | 9 | Based on the official images: 10 | 11 | * [elasticsearch](https://github.com/elastic/elasticsearch-docker) 12 | * [logstash](https://github.com/elastic/logstash-docker) 13 | * [kibana](https://github.com/elastic/kibana-docker) 14 | 15 | **Note**: Other branches in this project are available: 16 | 17 | * ELK 5 with X-Pack support: https://github.com/deviantony/docker-elk/tree/x-pack 18 | * ELK 5 in Vagrant: https://github.com/deviantony/docker-elk/tree/vagrant 19 | * ELK 5 with Search Guard: https://github.com/deviantony/docker-elk/tree/searchguard 20 | 21 | # Requirements 22 | 23 | ## Setup 24 | 25 | 1. Install [Docker](http://docker.io). 26 | 2. Install [Docker-compose](http://docs.docker.com/compose/install/) **version >= 1.6**. 27 | 3. Clone this repository 28 | 29 | ## Increase `vm.max_map_count` on your host 30 | 31 | You need to increase the `vm.max_map_count` kernel setting on your Docker host. 32 | To do this follow the recommended instructions from the Elastic documentation: [Install Elasticsearch with Docker](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode) 33 | 34 | ## SELinux 35 | 36 | On distributions which have SELinux enabled out-of-the-box you will need to either re-context the files or set SELinux into Permissive mode in order for docker-elk to start properly. 37 | For example on Redhat and CentOS, the following will apply the proper context: 38 | 39 | ```bash 40 | $ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ 41 | ``` 42 | 43 | # Usage 44 | 45 | Start the ELK stack using *docker-compose*: 46 | 47 | ```bash 48 | $ docker-compose up 49 | ``` 50 | 51 | You can also choose to run it in background (detached mode): 52 | 53 | ```bash 54 | $ docker-compose up -d 55 | ``` 56 | 57 | Now that the stack is running, you'll want to inject logs in it. The shipped logstash configuration allows you to send content via tcp: 58 | 59 | ```bash 60 | $ nc localhost 5000 < /path/to/logfile.log 61 | ``` 62 | 63 | And then access Kibana UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser. 64 | 65 | *NOTE*: You'll need to inject data into logstash before being able to configure a logstash index pattern in Kibana. Then all you should have to do is to hit the create button. 66 | 67 | Refer to [Connect Kibana with Elasticsearch](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html) for detailed instructions about the index pattern configuration. 68 | 69 | By default, the stack exposes the following ports: 70 | * 5000: Logstash TCP input. 71 | * 9200: Elasticsearch HTTP 72 | * 9300: Elasticsearch TCP transport 73 | * 5601: Kibana 74 | 75 | *WARNING*: If you're using *boot2docker*, you must access it via the *boot2docker* IP address instead of *localhost*. 76 | 77 | *WARNING*: If you're using *Docker Toolbox*, you must access it via the *docker-machine* IP address instead of *localhost*. 78 | 79 | # Configuration 80 | 81 | *NOTE*: Configuration is not dynamically reloaded, you will need to restart the stack after any change in the configuration of a component. 82 | 83 | ## How can I tune Kibana configuration? 84 | 85 | The Kibana default configuration is stored in `kibana/config/kibana.yml`. 86 | 87 | ## How can I tune Logstash configuration? 88 | 89 | The logstash configuration is stored in `logstash/config/logstash.yml`. 90 | 91 | It is also possible to map the entire `config` directory inside the container in the `docker-compose.yml`. Update the logstash container declaration to: 92 | 93 | ```yml 94 | logstash: 95 | build: logstash/ 96 | volumes: 97 | - ./logstash/pipeline:/usr/share/logstash/pipeline 98 | - ./logstash/config:/usr/share/logstash/config 99 | ports: 100 | - "5000:5000" 101 | networks: 102 | - elk 103 | depends_on: 104 | - elasticsearch 105 | ``` 106 | 107 | In the above example the folder `logstash/config` is mapped onto the container `/usr/share/logstash/config` so you can create more than one file in that folder if you'd like to. However, you must be aware that config files will be read from the directory in alphabetical order, and that Logstash will be expecting a [`log4j2.properties`](https://github.com/elastic/logstash-docker/tree/master/build/logstash/config) file for its own logging. 108 | 109 | ## How can I tune Elasticsearch configuration? 110 | 111 | The Elasticsearch container is using the [shipped configuration](https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml). 112 | 113 | If you want to override the default configuration, create a file `elasticsearch/config/elasticsearch.yml` and add your configuration in it. 114 | 115 | Then, you'll need to map your configuration file inside the container in the `docker-compose.yml`. Update the elasticsearch container declaration to: 116 | 117 | ```yml 118 | elasticsearch: 119 | build: elasticsearch/ 120 | ports: 121 | - "9200:9200" 122 | - "9300:9300" 123 | environment: 124 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 125 | networks: 126 | - elk 127 | volumes: 128 | - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml 129 | ``` 130 | 131 | You can also specify the options you want to override directly via environment variables: 132 | 133 | ```yml 134 | elasticsearch: 135 | build: elasticsearch/ 136 | ports: 137 | - "9200:9200" 138 | - "9300:9300" 139 | environment: 140 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 141 | network.host: "_non_loopback_" 142 | cluster.name: "my-cluster" 143 | networks: 144 | - elk 145 | ``` 146 | 147 | ## How can I scale up the Elasticsearch cluster? 148 | 149 | Follow the instructions from the Wiki: [Scaling up Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster) 150 | 151 | # Storage 152 | 153 | ## How can I store Elasticsearch data? 154 | 155 | The data stored in Elasticsearch will be persisted after container reboot but not after container removal. 156 | 157 | In order to persist Elasticsearch data even after removing the Elasticsearch container, you'll have to mount a volume on your Docker host. Update the elasticsearch container declaration to: 158 | 159 | ```yml 160 | elasticsearch: 161 | build: elasticsearch/ 162 | ports: 163 | - "9200:9200" 164 | - "9300:9300" 165 | environment: 166 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 167 | network.host: "_non_loopback_" 168 | cluster.name: "my-cluster" 169 | networks: 170 | - elk 171 | volumes: 172 | - /path/to/storage:/usr/share/elasticsearch/data 173 | ``` 174 | 175 | This will store Elasticsearch data inside `/path/to/storage`. 176 | 177 | # Extensibility 178 | 179 | ## How can I add plugins? 180 | 181 | To add plugins to any ELK component you have to: 182 | 183 | 1. Add a `RUN` statement to the corresponding `Dockerfile` (eg. `RUN logstash-plugin install logstash-filter-json`) 184 | 2. Add the associated plugin code configuration to the service configuration (eg. Logstash input/output) 185 | 186 | # JVM tuning 187 | 188 | ## How can I specify the amount of memory used by a service? 189 | 190 | By default, both Elasticsearch and Logstash start with [1/4 of the total host memory](https://docs.oracle.com/javase/8/docs/technotes/guides/vm/gctuning/parallel.html#default_heap_size) allocated to the JVM Heap Size. 191 | 192 | The startup scripts for Elasticsearch and Logstash can append extra JVM options from the value of an environment variable, allowing the user to adjust the amount of memory that can be used by each component: 193 | 194 | | Service | Environment variable | 195 | |---------------|----------------------| 196 | | Elasticsearch | ES_JAVA_OPTS | 197 | | Logstash | LS_JAVA_OPTS | 198 | 199 | To accomodate environments where memory is scarce (Docker for Mac has only 2 GB available by default), the Heap Size allocation is capped by default to 256MB per service within the `docker-compose.yml` file. If you want to override the default JVM configuration, edit the matching environment variable(s) in the `docker-compose.yml` file. 200 | 201 | For example, to increase the maximum JVM Heap Size for Logstash: 202 | 203 | ```yml 204 | logstash: 205 | build: logstash/ 206 | volumes: 207 | - ./logstash/pipeline:/usr/share/logstash/pipeline 208 | ports: 209 | - "5000:5000" 210 | networks: 211 | - elk 212 | depends_on: 213 | - elasticsearch 214 | environment: 215 | LS_JAVA_OPTS: "-Xmx1g -Xms1g" 216 | ``` 217 | 218 | ## How can I enable a remote JMX connection to a service? 219 | 220 | As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the docker host. 221 | 222 | Update the *{ES,LS}_JAVA_OPTS* environment variable with the following content (I've mapped the JMX service on the port 18080, you can change that). Do not forget to update the *-Djava.rmi.server.hostname* option with the IP address of your Docker host (replace **DOCKER_HOST_IP**): 223 | 224 | ```yml 225 | logstash: 226 | build: logstash/ 227 | volumes: 228 | - ./logstash/pipeline:/usr/share/logstash/pipeline 229 | ports: 230 | - "5000:5000" 231 | networks: 232 | - elk 233 | depends_on: 234 | - elasticsearch 235 | environment: 236 | LS_JAVA_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=18080 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=DOCKER_HOST_IP -Dcom.sun.management.jmxremote.local.only=false" 237 | ``` 238 | --------------------------------------------------------------------------------