├── .gitignore ├── README.md ├── curator └── config │ ├── actions.yml │ └── config.yml ├── docker-compose.yml ├── elasticsearch └── config │ └── elasticsearch.yml ├── kibana └── config │ └── kibana.yml ├── logstash └── config │ └── logstash.conf └── webapp ├── Dockerfile ├── filebeat.yml └── start.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ELK stack from scratch, with Docker 2 | =================================== 3 | 4 | ## Run (stack) 5 | ``` 6 | # run (daemon) 7 | docker-compose up -d 8 | # show logs 9 | docker-compose logs -f 10 | ``` 11 | 12 | ## Run the webapp to monitor 13 | ``` 14 | docker build ./webapp -t dockerelkstack_webapp 15 | docker run --network dockerelkstack_logging --link redis:redis -p 80:80 -d --name webapp dockerelkstack_webapp 16 | ``` 17 | Then navigate on the site (port 80, source: https://github.com/sbilly/joli-admin) 18 | 19 | ## Index management with curator 20 | ``` 21 | docker run --network dockerelkstack_logging --link elastic:elasticsearch -v "$PWD/curator/config":/config --rm bobrik/curator:4.0.4 --config /config/config.yml /config/actions.yml 22 | ``` 23 | -------------------------------------------------------------------------------- /curator/config/actions.yml: -------------------------------------------------------------------------------- 1 | actions: 2 | 1: 3 | action: close 4 | description: >- 5 | Close indices older than 14 day (based on index name), for logstash-* indices. 6 | options: 7 | delete_aliases: False 8 | timeout_override: 9 | continue_if_exception: False 10 | disable_action: False 11 | filters: 12 | - filtertype: pattern 13 | kind: prefix 14 | value: logstash- 15 | exclude: 16 | - filtertype: age 17 | source: name 18 | direction: older 19 | timestring: '%Y.%m.%d' 20 | unit: days 21 | unit_count: 14 22 | exclude: 23 | 2: 24 | action: delete_indices 25 | description: >- 26 | Delete indices older than 30 days (based on index name), for logstash-* indices. 27 | Ignore the error if the filter does not result in an actionable list of indices (ignore_empty_list) and exit cleanly. 28 | options: 29 | ignore_empty_list: True 30 | timeout_override: 31 | continue_if_exception: False 32 | disable_action: False 33 | filters: 34 | - filtertype: pattern 35 | kind: prefix 36 | value: logstash- 37 | exclude: 38 | - filtertype: age 39 | source: name 40 | direction: older 41 | timestring: '%Y.%m.%d' 42 | unit: days 43 | unit_count: 30 44 | exclude: 45 | -------------------------------------------------------------------------------- /curator/config/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | client: 5 | hosts: 6 | - elasticsearch 7 | port: 9200 8 | url_prefix: 9 | use_ssl: False 10 | certificate: 11 | client_cert: 12 | client_key: 13 | aws_key: 14 | aws_secret_key: 15 | aws_region: 16 | ssl_no_validate: False 17 | http_auth: 18 | timeout: 30 19 | master_only: False 20 | 21 | logging: 22 | loglevel: INFO 23 | logfile: 24 | logformat: default 25 | blacklist: ['elasticsearch', 'urllib3'] -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | services: 4 | # brocker 5 | redis: 6 | image: redis:3.2.6 7 | container_name: redis 8 | ports: 9 | - 6379:6379 10 | volumes: 11 | - redis-data:/data 12 | networks: 13 | - logging 14 | # index, search & agregation 15 | elasticsearch: 16 | image: elasticsearch:5.1.2 17 | container_name: elastic 18 | environment: 19 | - ES_JAVA_OPTS=-Xms1g -Xmx1g 20 | ports: 21 | - 9200:9200 22 | - 9300:9300 23 | volumes: 24 | - $PWD/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml 25 | - "es-data:/usr/share/elasticsearch/data" 26 | networks: 27 | - logging 28 | # UI 29 | kibana: 30 | image: kibana:5.1.2 31 | container_name: kibana 32 | ports: 33 | - 5601:5601 34 | volumes: 35 | - $PWD/kibana/config/kibana.yml:/etc/kibana/kibana.yml 36 | networks: 37 | - logging 38 | depends_on: 39 | - elasticsearch 40 | # indexer 41 | logstash: 42 | image: logstash:5.1.2 43 | container_name: logstash 44 | command: logstash -f /config/ 45 | environment: 46 | - JAVA_OPTS=-Xms1g -Xmx1g 47 | volumes: 48 | - $PWD/logstash/config:/config 49 | networks: 50 | - logging 51 | depends_on: 52 | - elasticsearch 53 | - redis 54 | 55 | volumes: 56 | es-data: 57 | driver: local 58 | redis-data: 59 | driver: local 60 | 61 | networks: 62 | logging: 63 | driver: bridge 64 | -------------------------------------------------------------------------------- /elasticsearch/config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | network.host: 0.0.0.0 2 | 3 | discovery.zen.minimum_master_nodes: 1 4 | 5 | bootstrap.seccomp: false 6 | 7 | -------------------------------------------------------------------------------- /kibana/config/kibana.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 5601 3 | host: 0.0.0.0 4 | basePath: "" 5 | 6 | elasticsearch: 7 | url: "http://elasticsearch:9200" 8 | -------------------------------------------------------------------------------- /logstash/config/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | redis { 3 | host => redis 4 | port => 6379 5 | key => "logstash" 6 | data_type => "list" 7 | codec => json 8 | } 9 | } 10 | 11 | filter { 12 | if [type] == "nginx-access" { 13 | grok { 14 | match => { "message" => "%{COMBINEDAPACHELOG}" } 15 | } 16 | } 17 | } 18 | 19 | output { 20 | elasticsearch { 21 | hosts => ["elasticsearch:9200"] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /webapp/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx 2 | 3 | RUN apt-get update \ 4 | && apt-get install curl sudo git -y \ 5 | && curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.1.2-amd64.deb \ 6 | && sudo dpkg -i filebeat-5.1.2-amd64.deb \ 7 | && rm -rf /var/log/nginx/*.log \ 8 | && touch /var/log/nginx/access.log \ 9 | && touch /var/log/nginx/error.log \ 10 | && git clone https://github.com/sbilly/joli-admin \ 11 | && mv joli-admin/joli/* /usr/share/nginx/html/ 12 | 13 | ADD filebeat.yml /etc/filebeat/filebeat.yml 14 | 15 | ADD ./start.sh /usr/local/bin/start.sh 16 | RUN chmod +x /usr/local/bin/start.sh 17 | CMD [ "/usr/local/bin/start.sh" ] 18 | -------------------------------------------------------------------------------- /webapp/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat: 2 | prospectors: 3 | - 4 | paths: 5 | - "/var/log/nginx/access.log" 6 | document_type: nginx-access 7 | output: 8 | redis: 9 | hosts: ["redis"] 10 | port: 6379 11 | key: "logstash" 12 | -------------------------------------------------------------------------------- /webapp/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /etc/init.d/nginx start 4 | /etc/init.d/filebeat start 5 | 6 | tail -f /var/log/nginx/access.log -f /var/log/nginx/error.log 7 | --------------------------------------------------------------------------------