├── .gitattributes
├── .github
    └── workflows
    │   └── docker-publish-to-dockerhub.yml
├── Dockerfile
├── LICENSE
├── README-short.txt
├── README.md
├── docker-compose.yml
├── docs
    └── index.md
├── elasticsearch-default
├── elasticsearch-init
├── elasticsearch-logrotate
├── elasticsearch.yml
├── kibana-init
├── kibana-logrotate
├── kibana.yml
├── logstash-beats.crt
├── logstash-beats.key
├── logstash-conf
    ├── 02-beats-input.conf
    ├── 10-syslog.conf
    ├── 11-nginx.conf
    └── 30-output.conf
├── logstash-init
├── logstash-logrotate
├── nginx-filebeat
    ├── Dockerfile
    ├── filebeat.yml
    ├── html
    │   ├── favicon.ico
    │   └── index.html
    ├── logstash-beats.crt
    └── start.sh
├── nginx.pattern
├── pipelines.yml
└── start.sh


/.gitattributes:
--------------------------------------------------------------------------------
1 | * text eol=lf
2 | 


--------------------------------------------------------------------------------
/.github/workflows/docker-publish-to-dockerhub.yml:
--------------------------------------------------------------------------------
 1 | name: "Docker: Docker Hub"
 2 | 
 3 | on:
 4 |   push:
 5 |     tags: [ '*' ]
 6 | 
 7 | env:
 8 |   REGISTRY: docker.io
 9 |   IMAGE_NAME: sebp/elk
10 | 
11 | jobs:
12 |   build_and_publish:
13 |   
14 |     runs-on: ubuntu-latest
15 |     permissions:
16 |       contents: read
17 |       packages: write
18 | 
19 |     steps:
20 |       - name: Checkout repository
21 |         uses: actions/checkout@v2
22 | 
23 | 
24 |       - name: Log into registry ${{ env.REGISTRY }}
25 |         if: github.event_name != 'pull_request'
26 |         uses: docker/login-action@v1
27 |         with:
28 |           registry: ${{ env.REGISTRY }}
29 |           username: ${{ secrets.DOCKERHUB_USERNAME }}
30 |           password: ${{ secrets.DOCKERHUB_TOKEN }}
31 | 
32 |       - name: Extract Docker metadata
33 |         id: meta
34 |         uses: docker/metadata-action@v3
35 |         with:
36 |           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
37 |           flavor: |
38 |             latest=true
39 |             prefix=
40 |             suffix=
41 | 
42 |       - name: Build and push Docker image
43 |         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
44 |         with:
45 |           context: .
46 |           push: ${{ github.event_name != 'pull_request' }}
47 |           tags: ${{ steps.meta.outputs.tags }}
48 |           labels: ${{ steps.meta.outputs.labels }}, latest
49 | 
50 |       - name: Docker Hub Description
51 |         uses: peter-evans/dockerhub-description@v2
52 |         with:
53 |           username: ${{ secrets.DOCKERHUB_USERNAME }}
54 |           password: ${{ secrets.DOCKERHUB_PASSWORD }}
55 |           repository: ${{ env.IMAGE_NAME }}
56 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
  1 | # Dockerfile for ELK stack
  2 | # Elasticsearch, Logstash, Kibana 8.17.6
  3 | 
  4 | # Build with:
  5 | # docker build -t <repo-user>/elk .
  6 | 
  7 | # Run with:
  8 | # docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it --name elk <repo-user>/elk
  9 | 
 10 | # replace with master-arm64 for ARM64
 11 | ARG IMAGE=focal-1.1.0
 12 | 
 13 | FROM phusion/baseimage:${IMAGE}
 14 | MAINTAINER Sebastien Pujadas http://pujadas.net
 15 | ENV \
 16 |  REFRESHED_AT=2020-06-20
 17 | 
 18 | 
 19 | ###############################################################################
 20 | #                                INSTALLATION
 21 | ###############################################################################
 22 | 
 23 | ### install prerequisites (cURL, gosu, tzdata, JDK for Logstash)
 24 | 
 25 | RUN set -x \
 26 |  && apt update -qq \
 27 |  && apt install -qqy --no-install-recommends ca-certificates curl gosu tzdata openjdk-11-jdk-headless \
 28 |  && apt clean \
 29 |  && rm -rf /var/lib/apt/lists/* \
 30 |  && gosu nobody true \
 31 |  && set +x
 32 | 
 33 | 
 34 | ### set current package version
 35 | 
 36 | ARG ELK_VERSION=8.17.6
 37 | 
 38 | # base version (i.e. remove OSS prefix) for Elasticsearch and Kibana (no OSS version since 7.11.0)
 39 | ARG ELK_BASE_VERSION=8.17.6
 40 | 
 41 | # replace with aarch64 for ARM64 systems
 42 | ARG ARCH=x86_64 
 43 | 
 44 | 
 45 | ### install Elasticsearch
 46 | 
 47 | # predefine env vars, as you can't define an env var that references another one in the same block
 48 | ENV \
 49 |  ES_VERSION=${ELK_BASE_VERSION} \
 50 |  ES_HOME=/opt/elasticsearch
 51 | 
 52 | ENV \
 53 |  ES_PACKAGE=elasticsearch-${ES_VERSION}-linux-${ARCH}.tar.gz \
 54 |  ES_GID=991 \
 55 |  ES_UID=991 \
 56 |  ES_PATH_CONF=/etc/elasticsearch \
 57 |  ES_PATH_BACKUP=/var/backups
 58 | 
 59 | RUN DEBIAN_FRONTEND=noninteractive \
 60 |  && mkdir ${ES_HOME} \
 61 |  && curl -O https://artifacts.elastic.co/downloads/elasticsearch/${ES_PACKAGE} \
 62 |  && tar xzf ${ES_PACKAGE} -C ${ES_HOME} --strip-components=1 \
 63 |  && rm -f ${ES_PACKAGE} \
 64 |  && groupadd -r elasticsearch -g ${ES_GID} \
 65 |  && useradd -r -s /usr/sbin/nologin -M -d ${ES_HOME} -c "Elasticsearch service user" -u ${ES_UID} -g elasticsearch elasticsearch \
 66 |  && mkdir -p /var/log/elasticsearch ${ES_PATH_CONF} ${ES_PATH_CONF}/scripts ${ES_PATH_CONF}/jvm.options.d /var/lib/elasticsearch ${ES_PATH_BACKUP} \
 67 |  && chown -R elasticsearch:elasticsearch ${ES_HOME} /var/log/elasticsearch /var/lib/elasticsearch ${ES_PATH_CONF} ${ES_PATH_BACKUP}
 68 | 
 69 | 
 70 | ### install Logstash
 71 | 
 72 | ENV \
 73 |  LOGSTASH_VERSION=${ELK_VERSION} \
 74 |  LOGSTASH_HOME=/opt/logstash
 75 | 
 76 | ENV \
 77 |  LOGSTASH_PACKAGE=logstash-${LOGSTASH_VERSION}-linux-${ARCH}.tar.gz \
 78 |  LOGSTASH_GID=992 \
 79 |  LOGSTASH_UID=992 \
 80 |  LOGSTASH_PATH_CONF=/etc/logstash \
 81 |  LOGSTASH_PATH_SETTINGS=${LOGSTASH_HOME}/config
 82 | 
 83 | RUN mkdir ${LOGSTASH_HOME} \
 84 |  && curl -O https://artifacts.elastic.co/downloads/logstash/${LOGSTASH_PACKAGE} \
 85 |  && tar xzf ${LOGSTASH_PACKAGE} -C ${LOGSTASH_HOME} --strip-components=1 \
 86 |  && rm -f ${LOGSTASH_PACKAGE} \
 87 |  && groupadd -r logstash -g ${LOGSTASH_GID} \
 88 |  && useradd -r -s /usr/sbin/nologin -M -d ${LOGSTASH_HOME} -c "Logstash service user" -u ${LOGSTASH_UID} -g logstash logstash \
 89 |  && mkdir -p /var/log/logstash ${LOGSTASH_PATH_CONF}/conf.d \
 90 |  && chown -R logstash:logstash ${LOGSTASH_HOME} /var/log/logstash ${LOGSTASH_PATH_CONF}
 91 | 
 92 | 
 93 | ### install Kibana
 94 | 
 95 | ENV \
 96 |  KIBANA_VERSION=${ELK_BASE_VERSION} \
 97 |  KIBANA_HOME=/opt/kibana
 98 | 
 99 | ENV \
100 |  KIBANA_PACKAGE=kibana-${KIBANA_VERSION}-linux-${ARCH}.tar.gz \
101 |  KIBANA_GID=993 \
102 |  KIBANA_UID=993
103 | 
104 | RUN mkdir ${KIBANA_HOME} \
105 |  && curl -O https://artifacts.elastic.co/downloads/kibana/${KIBANA_PACKAGE} \
106 |  && tar xzf ${KIBANA_PACKAGE} -C ${KIBANA_HOME} --strip-components=1 \
107 |  && rm -f ${KIBANA_PACKAGE} \
108 |  && groupadd -r kibana -g ${KIBANA_GID} \
109 |  && useradd -r -s /usr/sbin/nologin -d ${KIBANA_HOME} -c "Kibana service user" -u ${KIBANA_UID} -g kibana kibana \
110 |  && mkdir -p /var/log/kibana \
111 |  && chown -R kibana:kibana ${KIBANA_HOME} /var/log/kibana
112 | 
113 | 
114 | ###############################################################################
115 | #                              START-UP SCRIPTS
116 | ###############################################################################
117 | 
118 | ### Elasticsearch
119 | 
120 | ADD ./elasticsearch-init /etc/init.d/elasticsearch
121 | RUN sed -i -e 's#^ES_HOME=$#ES_HOME='$ES_HOME'#' /etc/init.d/elasticsearch \
122 |  && chmod +x /etc/init.d/elasticsearch
123 | 
124 | 
125 | ### Logstash
126 | 
127 | ADD ./logstash-init /etc/init.d/logstash
128 | RUN sed -i -e 's#^LS_HOME=$#LS_HOME='$LOGSTASH_HOME'#' /etc/init.d/logstash \
129 |  && chmod +x /etc/init.d/logstash
130 | 
131 | 
132 | ### Kibana
133 | 
134 | ADD ./kibana-init /etc/init.d/kibana
135 | RUN sed -i -e 's#^KIBANA_HOME=$#KIBANA_HOME='$KIBANA_HOME'#' /etc/init.d/kibana \
136 |  && chmod +x /etc/init.d/kibana
137 | 
138 | 
139 | ###############################################################################
140 | #                               CONFIGURATION
141 | ###############################################################################
142 | 
143 | ### configure Elasticsearch
144 | 
145 | ADD ./elasticsearch.yml ${ES_PATH_CONF}/elasticsearch.yml
146 | ADD ./elasticsearch-default /etc/default/elasticsearch
147 | RUN cp ${ES_HOME}/config/log4j2.properties ${ES_HOME}/config/jvm.options \
148 |     ${ES_PATH_CONF} \
149 |  && chown -R elasticsearch:elasticsearch ${ES_PATH_CONF} \
150 |  && chmod -R +r ${ES_PATH_CONF}
151 | 
152 | 
153 | ### configure Logstash
154 | 
155 | # certs/keys for Beats and Lumberjack input
156 | RUN mkdir -p /etc/pki/tls/{certs,private}
157 | ADD ./logstash-beats.crt /etc/pki/tls/certs/logstash-beats.crt
158 | ADD ./logstash-beats.key /etc/pki/tls/private/logstash-beats.key
159 | 
160 | # pipelines
161 | ADD pipelines.yml ${LOGSTASH_PATH_SETTINGS}/pipelines.yml
162 | 
163 | # filters
164 | ADD ./logstash-conf/*.conf ${LOGSTASH_PATH_CONF}/conf.d/
165 | 
166 | # patterns
167 | ADD ./nginx.pattern ${LOGSTASH_HOME}/patterns/nginx
168 | RUN chown -R logstash:logstash ${LOGSTASH_HOME}/patterns
169 | 
170 | # Fix permissions
171 | RUN chmod -R +r ${LOGSTASH_PATH_CONF} ${LOGSTASH_PATH_SETTINGS} \
172 |  && chown -R logstash:logstash ${LOGSTASH_PATH_SETTINGS}
173 | 
174 | 
175 | ### configure logrotate
176 | 
177 | ADD ./elasticsearch-logrotate /etc/logrotate.d/elasticsearch
178 | ADD ./logstash-logrotate /etc/logrotate.d/logstash
179 | ADD ./kibana-logrotate /etc/logrotate.d/kibana
180 | RUN chmod 644 /etc/logrotate.d/elasticsearch \
181 |  && chmod 644 /etc/logrotate.d/logstash \
182 |  && chmod 644 /etc/logrotate.d/kibana
183 | 
184 | 
185 | ### configure Kibana
186 | 
187 | ADD ./kibana.yml ${KIBANA_HOME}/config/kibana.yml
188 | 
189 | 
190 | ###############################################################################
191 | #                                   START
192 | ###############################################################################
193 | 
194 | ADD ./start.sh /usr/local/bin/start.sh
195 | RUN chmod +x /usr/local/bin/start.sh
196 | 
197 | EXPOSE 5601 9200 9300 9600 5044
198 | VOLUME /var/lib/elasticsearch
199 | 
200 | CMD [ "/usr/local/bin/start.sh" ]
201 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright 2015 Sébastien Pujadas
 2 | 
 3 | Licensed under the Apache License, Version 2.0 (the "License");
 4 | you may not use this file except in compliance with the License.
 5 | You may obtain a copy of the License at
 6 | 
 7 |     http://www.apache.org/licenses/LICENSE-2.0
 8 | 
 9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.


--------------------------------------------------------------------------------
/README-short.txt:
--------------------------------------------------------------------------------
1 | Collect, search and visualise log data with ELK (Elasticsearch 8.17.6, Logstash 8.17.6, Kibana 8.17.6).
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Elasticsearch, Logstash, Kibana (ELK) Docker image
 2 | 
 3 | [![](https://images.microbadger.com/badges/image/sebp/elk.svg)](https://microbadger.com/images/sebp/elk "Get your own image badge on microbadger.com") [![Documentation Status](https://readthedocs.org/projects/elk-docker/badge/?version=latest)](http://elk-docker.readthedocs.io/?badge=latest)
 4 | 
 5 | This Docker image provides a convenient centralised log server and log management web interface, by packaging Elasticsearch, Logstash, and Kibana, collectively known as ELK.
 6 | 
 7 | ### Documentation
 8 | 
 9 | See the [ELK Docker image documentation web page](http://elk-docker.readthedocs.io/) for complete instructions on how to use this image.
10 | 
11 | ### Docker Hub
12 | 
13 | This image is hosted on Docker Hub at [https://hub.docker.com/r/sebp/elk/](https://hub.docker.com/r/sebp/elk/).
14 | 
15 | The following tags are available:
16 | 
17 | - `latest`, `8.17.6`: ELK 8.17.6.
18 | 
19 | - `oss-8.17.6` (ELK OSS 8.17.6), `8.17.3` (8.17.3), `oss-8.17.3` (OSS 8.17.3), `8.15.1` (8.15.1), `oss-8.15.1` (OSS 8.15.1), `8.15.0` (8.15.0), `oss-8.15.0` (OSS 8.15.0), `8.14.3` (8.14.3), `8.14.1` (8.14.1), `oss-8.14.1` (OSS 8.14.1), `8.14.0` (8.14.0), `oss-8.14.0` (OSS 8.14.0), `8.13.4` (8.13.4), `oss-8.13.4` (OSS 8.13.4), `8.13.3` (8.13.3), `oss-8.13.3` (OSS 8.13.3), `8.13.2` (8.13.2), `oss-8.13.2` (OSS 8.13.2), `8.13.1` (8.13.1), `oss-8.13.1` (OSS 8.13.1), `8.13.0` (8.13.0), `oss-8.13.0` (OSS 8.13.0), `8.12.2` (8.12.2), `oss-8.12.2` (OSS 8.12.2), `8.12.1` (8.12.1), `oss-8.12.1` (OSS 8.12.1), `8.12.0` (8.12.0), `oss-8.12.0` (OSS 8.12.0), `8.11.4` (8.11.4), `oss-8.11.4` (OSS 8.11.4), `8.11.3` (8.11.3), `oss-8.11.3` (OSS 8.11.3), `8.11.2` (8.11.2), `oss-8.11.2` (OSS 8.11.2), `8.11.1` (8.11.1), `oss-8.11.1` (OSS 8.11.1), `8.10.4` (8.10.4), `oss-8.10.4` (OSS 8.10.4), `8.9.2` (8.9.2), `oss-8.9.2` (OSS 8.9.2), `8.9.1` (8.9.1), `oss-8.9.1` (OSS 8.9.1), `8.9.0` (8.9.0), `oss-8.9.0` (OSS 8.9.0), `8.8.2` (8.8.2), `oss-8.8.2` (OSS 8.8.2), `8.8.1` (8.8.1), `oss-8.8.1` (OSS 8.8.1), `8.8.0` (8.8.0), `oss-8.8.0` (OSS 8.8.0), `8.7.1` (8.7.1), `oss-8.7.1` (OSS 8.7.1), `8.7.0` (8.7.0), `oss-8.7.0` (OSS 8.7.0), `8.6.2` (8.6.2), `oss-8.6.2` (OSS 8.6.2), `8.6.1` (8.6.1), `oss-8.6.1` (OSS 8.6.1), `8.6.0` (8.6.0), `oss-8.6.0` (OSS 8.6.0), `8.5.3` (8.5.3), `oss-8.5.3` (OSS 8.5.3), `8.5.2` (8.5.2), `oss-8.5.2` (OSS 8.5.2), `8.5.1` (8.5.1), `oss-8.5.1` (OSS 8.5.1), `8.5.0` (8.5.0), `oss-8.5.0` (OSS 8.5.0), `8.4.3` (8.4.3), `oss-8.4.3` (OSS 8.4.3), `8.4.2` (8.4.2), `oss-8.4.2` (OSS 8.4.2), `8.4.1` (8.4.1), `oss-8.4.1` (OSS 8.4.1), `8.4.0` (8.4.0), `oss-8.4.0` (OSS 8.4.0), `8.3.3` (8.3.3), `oss-8.3.3` (OSS 8.3.3), `8.3.2` (8.3.2), `oss-8.3.2` (OSS 8.3.2), `8.3.1` (8.3.1), `oss-8.3.1` (OSS 8.3.1), `8.3.0` (8.3.0), `oss-8.3.0` (OSS 8.3.0), `8.2.3` (8.2.3), `oss-8.2.3` (OSS 8.2.3), `8.2.2` (8.2.2), `oss-8.2.2` (OSS 8.2.2), `8.2.1` (8.2.1), `oss-8.2.1` (OSS 8.2.1), `8.2.0` (8.2.0), `oss-8.2.0` (OSS 8.2.0), `8.1.3` (8.1.3), `oss-8.1.3` (OSS 8.1.3), `8.1.2` (8.1.2), `oss-8.1.2` (OSS 8.1.2), `8.1.1` (8.1.1), `oss-8.1.1` (OSS 8.1.1), `8.1.0` (8.1.0), `oss-8.1.0` (OSS 8.1.0), `8.0.1` (8.0.1), `oss-8.0.1` (OSS 8.0.1), `8.0.0` (8.0.0), `oss-8.0.0` (OSS 8.0.0).
20 | 
21 | - `7.17.1` (ELK 7.17.5), `oss-7.17.1` (OSS 7.17.5), `7.17.1` (7.17.1), `oss-7.17.1` (OSS 7.17.1), `7.17.0` (7.17.0), `oss-7.17.0` (OSS 7.17.0), `7.16.3` (7.16.3), `oss-7.16.3` (OSS 7.16.3), `7.16.2` (7.16.2), `oss-7.16.2` (OSS 7.16.2), `7.16.1` (7.16.1), `oss-7.16.1` (OSS 7.16.1), `7.16.0` (7.16.0), `oss-7.16.0` (OSS 7.16.0), `7.15.2` (7.15.2), `oss-7.15.2` (OSS 7.15.2), `7.15.1` (7.15.1), `oss-7.15.1` (OSS 7.15.1), `7.15.0` (7.15.0), `oss-7.15.0` (OSS 7.15.0), `7.14.2` (7.14.2), `oss-7.14.2` (OSS 7.14.2), `7.14.1` (7.14.1), `oss-7.14.1` (OSS 7.14.1), `7.14.0` (7.14.0), `oss-7.14.0` (OSS 7.14.0), `7.13.4` (7.13.4), `oss-7.13.4` (OSS 7.13.4), `7.13.3` (7.13.3), `oss-7.13.3` (OSS 7.13.3), `7.13.2` (7.13.2), `oss-7.13.2` (OSS 7.13.2), `7.13.1` (7.13.1), `oss-7.13.1` (OSS 7.13.1), `7.13.0` (7.13.0), `oss-7.13.0` (OSS 7.13.0), `7.12.1` (7.12.1), `oss-7.12.1` (OSS 7.12.1), `7.12.0` (7.12.0), `oss-7.12.0` (OSS 7.12.0), `7.11.2` (7.11.2), `oss-7.11.2` (OSS 7.11.2), `7.11.1` (7.11.1), `oss-7.11.1` (OSS 7.11.1), `oss-7.11.0` (OSS 7.11.0), `7.11.0` (7.11.0), `oss-7.10.2` (OSS 7.10.2), `7.10.2` (7.10.2), `oss-7.10.1` (OSS 7.10.1), `7.10.1` (7.10.1), `oss-7.10.0` (OSS 7.10.0), `7.10.0` (7.10.0), `oss-793` (OSS 7.9.3), `793` (7.9.3), `oss-792` (OSS 7.9.2), `792` (7.9.2), `oss-791` (OSS 7.9.1), `791` (7.9.1), `oss-790` (OSS 7.9.0), `790` (7.9.0), `oss-781` (OSS 7.8.1), `781` (7.8.1), `oss-780` (OSS 7.8.0), `780` (7.8.0), `771` (7.7.1), `770` (7.7.0), `762` (7.6.2), `761` (7.6.1), `760` (7.6.0), `752` (7.5.2), `751` (7.5.1), `750` (7.5.0), `742` (7.4.2), `741` (7.4.1), `740` (7.4.0), `732` (7.3.2), `731` (7.3.1), `730` (7.3.0), `721` (7.2.1), `720` (7.2.0), `711` (7.1.1), `710` (7.1.0), `701` (7.0.1), `700` (7.0.0).
22 | 
23 | - `6.8.22` (ELK 6.8.22), `683` (6.8.3), `681` (ELK 6.8.2), `681` (ELK 6.8.1), `680` (ELK 6.8.0), `672` (ELK 6.7.2), `671` (ELK 6.7.1), `670` (6.7.0), `662` (6.6.2), `661` (6.6.1), `660` (6.6.0), `651` (6.5.1), `650` (6.5.0), `643` (6.4.3), `642` (6.4.2), `641` (6.4.1), `640` (6.4.0), `632` (6.3.2), `631` (6.3.1), `630` (6.3.0), `624` (6.2.4), `623` (6.2.3), `622` (6.2.2), `621` (6.2.1), `620` (6.2.0), `613` (6.1.3), `612` (6.1.2), `611` (6.1.1), `610` (6.1.0), `601` (6.0.1), `600` (6.0.0).
24 | 
25 | - `5615` (ELK version 5.6.15), `568` (5.6.8), `564` (5.6.4), `563` (5.6.3), `562` (5.6.2), `561` (5.6.1), `560` (5.6.0), `553` (5.5.3), `552` (5.5.2), `551` (5.5.1), `550` (5.5.0), `543` (5.4.3), `542` (5.4.2), `541` (5.4.1), `540` (5.4.0), `532` (5.3.2), `531` (5.3.1), `530` (5.3.0), `522` (5.2.2), `521` (5.2.1), `520` (5.2.0), `512` (5.1.2), `511` (5.1.1), `502` (5.0.2), `es501_l501_k501` (5.0.1), `es500_l500_k500` (5.0.0).
26 | 
27 | - `es241_l240_k461`: Elasticsearch 2.4.1, Logstash 2.4.0, and Kibana 4.6.1.
28 | 
29 | - `es240_l240_k460`: Elasticsearch 2.4.0, Logstash 2.4.0, and Kibana 4.6.0.
30 | 
31 | - `es235_l234_k454`: Elasticsearch 2.3.5, Logstash 2.3.4, and Kibana 4.5.4.
32 | 
33 | - `es234_l234_k453`: Elasticsearch 2.3.4, Logstash 2.3.4, and Kibana 4.5.3.
34 | 
35 | - `es234_l234_k452`: Elasticsearch 2.3.4, Logstash 2.3.4, and Kibana 4.5.2.
36 | 
37 | - `es233_l232_k451`: Elasticsearch 2.3.3, Logstash 2.3.2, and Kibana 4.5.1.
38 | 
39 | - `es232_l232_k450`: Elasticsearch 2.3.2, Logstash 2.3.2, and Kibana 4.5.0.
40 | 
41 | - `es231_l231_k450`: Elasticsearch 2.3.1, Logstash 2.3.1, and Kibana 4.5.0.
42 | 
43 | - `es230_l230_k450`: Elasticsearch 2.3.0, Logstash 2.3.0, and Kibana 4.5.0.
44 | 
45 | - `es221_l222_k442`: Elasticsearch 2.2.1, Logstash 2.2.2, and Kibana 4.4.2.
46 | 
47 | - `es220_l222_k441`: Elasticsearch 2.2.0, Logstash 2.2.2, and Kibana 4.4.1.
48 | 
49 | - `es220_l220_k440`: Elasticsearch 2.2.0, Logstash 2.2.0, and Kibana 4.4.0.
50 | 
51 | - `E1L1K4`: Elasticsearch 1.7.3, Logstash 1.5.5, and Kibana 4.1.2.
52 | 
53 | **Note** – See the documentation page for more information on pulling specific combinations of versions of Elasticsearch, Logstash and Kibana.
54 | 
55 | ### About
56 | 
57 | Written by [Sébastien Pujadas](https://pujadas.net), released under the [Apache 2 license](https://www.apache.org/licenses/LICENSE-2.0).
58 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | # Docker Compose file for ELK stack
 2 | 
 3 | # Build with:
 4 | # docker-compose build elk
 5 | 
 6 | # Run with:
 7 | # docker-compose up
 8 | 
 9 | elk:
10 |   build: .
11 |   ports:
12 |     - "5601:5601"
13 |     - "9200:9200"
14 |     - "5044:5044"
15 | 


--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
  1 | # Elasticsearch, Logstash, Kibana (ELK) Docker image documentation
  2 | 
  3 | This web page documents how to use the [sebp/elk](https://hub.docker.com/r/sebp/elk/) Docker image, which provides a convenient centralised log server and log management web interface, by packaging [Elasticsearch](https://www.elastic.co/products/elasticsearch), [Logstash](http://logstash.net/), and [Kibana](https://www.elastic.co/products/kibana), collectively known as ELK.
  4 | 
  5 | ### Contents ###
  6 | 
  7 | - [Prerequisites](#prerequisites)
  8 | - [Installation](#installation)
  9 | 	- [Pulling specific version combinations](#specific-version-combinations)
 10 | - [Usage](#usage)
 11 | 	- [Running the container using Docker Compose](#running-with-docker-compose)
 12 | 	- [Creating a dummy log entry](#creating-dummy-log-entry)
 13 | 	- [Starting services selectively](#selective-services)
 14 | 	- [Overriding start-up variables](#overriding-variables)
 15 | 	- [Pre-hooks and post-hooks](#pre-post-hooks)
 16 | - [Forwarding logs](#forwarding-logs)
 17 | 	- [Forwarding logs with Filebeat](#forwarding-logs-filebeat)
 18 | 	- [Connecting a Docker container to an ELK container running on the same host](#connecting-containers)
 19 | - [Building the image](#building-image)
 20 | 	- [Building the image for ARM64](#building-image-arm64)
 21 | - [Tweaking the image](#tweaking-image)
 22 | 	- [Updating Logstash's configuration](#updating-logstash-configuration)
 23 | 	- [Updating Elasticsearch’s configuration](#updating-elasticsearch-configuration)
 24 | 	- [Installing Elasticsearch plugins](#installing-elasticsearch-plugins)
 25 | 	- [Installing Logstash plugins](#installing-logstash-plugins)
 26 | 	- [Installing Kibana plugins](#installing-kibana-plugins)
 27 | - [Persisting log data](#persisting-log-data)
 28 | - [Snapshot and restore](#snapshot-restore)
 29 | - [Setting up an Elasticsearch cluster](#elasticsearch-cluster)
 30 | 	- [Running Elasticsearch nodes on different hosts](#elasticsearch-cluster-different-hosts)
 31 | 	- [Running Elasticsearch nodes on a single host](#elasticsearch-cluster-single-host)
 32 | 	- [Optimising your Elasticsearch cluster](#optimising-elasticsearch-cluster)
 33 | - [Security considerations](#security-considerations)
 34 | 	- [Notes on certificates](#certificates)
 35 | 	- [Disabling SSL/TLS](#disabling-ssl-tls)
 36 | - [Frequently encountered issues](#frequent-issues)
 37 |   - [Elasticsearch is not starting (1): `max virtual memory areas vm.max_map_count [65530] likely too low, increase to at least [262144]`](#es-not-starting-max-map-count)
 38 |   - [Elasticsearch is not starting (2): `cat: /var/log/elasticsearch/elasticsearch.log: No such file or directory`](#es-not-starting-not-enough-memory)
 39 |   - [Elasticsearch is not starting (3): bootstrap tests](#es-not-starting-bootstrap-tests)
 40 |   - [Elasticsearch is not starting (4): no errors in log](#es-not-starting-timeout)
 41 |   - [Elasticsearch is suddenly stopping after having started properly](#es-suddenly-stopping)
 42 |   - [Miscellaneous](#issues-misc)
 43 | 
 44 | - [Assorted hints](#assorted-hints)
 45 | - [Troubleshooting](#troubleshooting)
 46 |   - [If Elasticsearch isn't starting...](#es-not-starting)
 47 |   - [If your log-emitting client doesn't seem to be able to reach Logstash...](#logstash-unreachable)
 48 |   - [Additional tips](#general-troubleshooting)
 49 | - [Reporting issues](#reporting-issues)
 50 | - [Breaking changes](#breaking-changes)
 51 | - [Release notes](#release-notes)
 52 | - [References](#references)
 53 | - [About](#about)
 54 | 
 55 | ## Prerequisites<a name="prerequisites"></a>
 56 | 
 57 | To run a container using this image, you will need the following:
 58 | 
 59 | - **Docker**
 60 | 
 61 | 	Install [Docker](https://docker.com/), either using a native package (Linux) or wrapped in a virtual machine (Windows, OS X – e.g. using [Boot2Docker](http://boot2docker.io/) or [Vagrant](https://www.vagrantup.com/)).
 62 | 
 63 | 	**Note** – As the *sebp/elk* image is based on a Linux image, users of Docker for Windows will need to ensure that [Docker is using Linux containers](https://docs.docker.com/docker-for-windows/#switch-between-windows-and-linux-containers).
 64 | 	
 65 | - **A minimum of 4GB RAM assigned to Docker**
 66 | 
 67 | 	Elasticsearch alone needs at least 2GB of RAM to run.
 68 | 
 69 | 	With Docker for Mac, the amount of RAM dedicated to Docker can be set using the UI: see [How to increase docker-machine memory Mac](http://stackoverflow.com/questions/32834082/how-to-increase-docker-machine-memory-mac/39720010#39720010) (Stack Overflow).
 70 | 
 71 | 	In Docker Desktop for Windows, [use the *Advanced* tab to adjust limits on resources available to Docker](https://docs.docker.com/docker-for-windows/#:~:text=Memory%3A%20By%20default%2C%20Docker%20Desktop,swap%20file%20size%20as%20needed).
 72 | 
 73 | - **A limit on mmap counts equal to 262,144 or more**
 74 | 
 75 | 	**<span style="color:red">!! This is the most frequent reason for Elasticsearch failing to start since Elasticsearch version 5 was released.</span>**
 76 | 
 77 | 	On Linux, use `sysctl vm.max_map_count` on the host to view the current value, and see [Elasticsearch's documentation on virtual memory](https://www.elastic.co/guide/en/elasticsearch/reference/5.0/vm-max-map-count.html#vm-max-map-count) for guidance on how to change this value. Note that the limits **must be changed on the host**; they cannot be changed from within a container.
 78 | 
 79 | 	If using Docker for Mac, then you will need to start the container with the `MAX_MAP_COUNT` environment variable (see [Overriding start-up variables](#overriding-variables)) set to at least 262144 (using e.g. `docker`'s `-e` option) to make Elasticsearch set the limits on mmap counts at start-up time.
 80 | 
 81 | - **Access to TCP port 5044 from log-emitting clients**
 82 | 
 83 | 	Other ports may need to be explicitly opened: see [Usage](#usage) for the complete list of ports that are exposed.
 84 | 	
 85 | - **Configuration of Elasticsearch to run on Apple M2**
 86 | 
 87 | 	Elasticsearch’s configuration file `elasticsearch.yml` needs to be updated with the following configuration item: `bootstrap.system_call_filter: false`. See [Updating Elasticsearch’s configuration](#updating-elasticsearch-configuration) for guidance.
 88 | 
 89 | ## Installation <a name="installation"></a>
 90 | 
 91 | To pull this image from the [Docker registry](https://hub.docker.com/r/sebp/elk/), open a shell prompt and enter:
 92 | 
 93 | 	$ sudo docker pull sebp/elk
 94 | 
 95 | **Note** – This image has been built automatically from the source files in the [source Git repository on GitHub](https://github.com/spujadas/elk-docker). If you want to build the image yourself, see the [Building the image](#building-image) section.
 96 | 
 97 | ### Pulling specific version combinations <a name="specific-version-combinations"></a> 
 98 | 
 99 | Specific version combinations of Elasticsearch, Logstash and Kibana can be pulled by using tags.
100 | 
101 | For instance, the image containing Elasticsearch 1.7.3, Logstash 1.5.5, and Kibana 4.1.2 (which is the last image using the Elasticsearch 1.x and Logstash 1.x branches) bears the tag `E1L1K4`, and can therefore be pulled using `sudo docker pull sebp/elk:E1L1K4`.
102 | 
103 | The available tags are listed on [Docker Hub's sebp/elk image page](https://hub.docker.com/r/sebp/elk/) or GitHub repository page.
104 | 
105 | By default, if no tag is indicated (or if using the tag `latest`), the latest version of the image will be pulled.
106 | 
107 | ## Usage <a name="usage"></a>
108 | 
109 | Run a container from the image with the following command:
110 | 
111 | 	$ sudo docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it --name elk sebp/elk
112 | 
113 | **Note** – The whole ELK stack will be started. See the *[Starting services selectively](#selective-services)* section to selectively start part of the stack.
114 | 
115 | This command publishes the following ports, which are needed for proper operation of the ELK stack:
116 | 
117 | - 5601 (Kibana web interface).
118 | - 9200 (Elasticsearch JSON interface).
119 | - 5044 (Logstash Beats interface, receives logs from Beats such as Filebeat – see the *[Forwarding logs with Filebeat](#forwarding-logs-filebeat)* section).
120 | 
121 | The image exposes (but does not publish):
122 | 
123 | - Elasticsearch's transport interface on port 9300. Use the `-p 9300:9300` option with the `docker` command above to publish it. This transport interface is notably used by [Elasticsearch's Java client API](https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/index.html), and to run Elasticsearch in a cluster.
124 | 
125 | - [Logstash's monitoring API](https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html) on port 9600. Use the `-p 9600:9600` option with the `docker` command above to publish it.
126 | 
127 | 
128 | The figure below shows how the pieces fit together.
129 | 
130 | ![](https://user-images.githubusercontent.com/930566/85201704-cc2c0e80-b301-11ea-8d59-aeda442a034d.png)
131 | 
132 | Access Kibana's web interface by browsing to `http://<your-host>:5601`, where `<your-host>` is the hostname or IP address of the host Docker is running on (see note), e.g. `localhost` if running a local native version of Docker, or the IP address of the virtual machine if running a VM-hosted version of Docker (see note).
133 | 
134 | **Note** – To configure and/or find out the IP address of a VM-hosted Docker installation, see [https://docs.docker.com/installation/windows/](https://docs.docker.com/installation/windows/) (Windows) and [https://docs.docker.com/installation/mac/](https://docs.docker.com/installation/mac/) (OS X) for guidance if using Boot2Docker. If you're using [Vagrant](https://www.vagrantup.com/), you'll need to set up port forwarding (see [https://docs.vagrantup.com/v2/networking/forwarded_ports.html](https://docs.vagrantup.com/v2/networking/forwarded_ports.html).
135 | 
136 | You can stop the container with `^C`, and start it again with `sudo docker start elk`.
137 | 
138 | As from Kibana version 4.0.0, you won't be able to see anything (not even an empty dashboard) until something has been logged (see the *[Creating a dummy log entry](#creating-dummy-log-entry)* sub-section below on how to test your set-up, and the *[Forwarding logs](#forwarding-logs)* section on how to forward logs from regular applications).
139 | 
140 | When filling in the index pattern in Kibana (default is `logstash-*`), note that in this image, Logstash uses an output plugin that is configured to work with Beat-originating input (e.g. as produced by Filebeat, see [Forwarding logs with Filebeat](#forwarding-logs-filebeat)) and that logs will be indexed with a `<beatname>-` prefix (e.g. `filebeat-` when using Filebeat).
141 | 
142 | ### Running the container using Docker Compose <a name="running-with-docker-compose"></a>
143 | 
144 | If you're using [Docker Compose](https://docs.docker.com/compose/) to manage your Docker services (and if not you really should as it will make your life much easier!), then you can create an entry for the ELK Docker image by adding the following lines to your `docker-compose.yml` file:
145 | 
146 | 	elk:
147 | 	  image: sebp/elk
148 | 	  ports:
149 | 	    - "5601:5601"
150 | 	    - "9200:9200"
151 | 	    - "5044:5044"
152 | 
153 | You can then start the ELK container like this:
154 | 
155 | 	$ sudo docker-compose up elk
156 | 
157 | ### Creating a dummy log entry <a name="creating-dummy-log-entry"></a>
158 | 
159 | If you haven't got any logs yet and want to manually create a dummy log entry for test purposes (for instance to see the dashboard), first start the container as usual (`sudo docker run ...` or `docker-compose up ...`).
160 | 
161 | In another terminal window, find out the name of the container running ELK, which is displayed in the last column of the output of the `sudo docker ps` command.
162 | 
163 | 	$ sudo docker ps
164 | 	CONTAINER ID        IMAGE                  ...   NAMES
165 | 	86aea21cab85        elkdocker_elk:latest   ...   elkdocker_elk_1
166 | 
167 | Open a shell prompt in the container and type (replacing `<container-name>` with the name of the container, e.g. `elkdocker_elk_1` in the example above):
168 | 
169 | 	$ sudo docker exec -it <container-name> /bin/bash
170 | 
171 | At the prompt, enter:
172 | 
173 | 	# /opt/logstash/bin/logstash --path.data /tmp/logstash/data \
174 | 		-e 'input { stdin { } } output { elasticsearch { hosts => ["localhost"] } }'
175 | 
176 | Wait for Logstash to start (as indicated by the message `The stdin plugin is now waiting for input:`), then type some dummy text followed by Enter to create a log entry:
177 | 
178 | 	this is a dummy entry
179 | 
180 | **Note** – You can create as many entries as you want. Use `^C` to go back to the bash prompt.
181 | 
182 | If you browse to `http://<your-host>:9200/_search?pretty&size=1000` (e.g. [http://localhost:9200/_search?pretty&size=1000](http://localhost:9200/_search?pretty&size=1000) for a local native instance of Docker) you'll see that Elasticsearch has indexed the entry:
183 | 
184 | 	{
185 | 	  ...
186 | 	  "hits": {
187 | 	    ...
188 | 	    "hits": [ {
189 | 	      "_index": "logstash-...",
190 | 	      "_type": "logs",
191 | 	      ...
192 | 	      "_source": { "message": "this is a dummy entry", "@version": "1", "@timestamp": ... }
193 | 	    } ]
194 | 	  }
195 | 	}
196 | 
197 | You can now browse to Kibana's web interface at `http://<your-host>:5601` (e.g. [http://localhost:5601](http://localhost:5601) for a local native instance of Docker).
198 | 
199 | Make sure that the drop-down "Time Filter field name" field is pre-populated with the value `@timestamp`, then click on "Create", and you're good to go.
200 | 
201 | ### Starting services selectively <a name="selective-services"></a>
202 | 
203 | By default, when starting a container, all three of the ELK services (Elasticsearch, Logstash, Kibana) are started.
204 | 
205 | The following environment variables may be used to selectively start a subset of the services:
206 | 
207 | - `ELASTICSEARCH_START`: if set and set to anything other than `1`, then Elasticsearch will not be started.
208 | 
209 | - `LOGSTASH_START`: if set and set to anything other than `1`, then Logstash will not be started.
210 | 
211 | - `KIBANA_START`: if set and set to anything other than `1`, then Kibana will not be started.
212 | 
213 | For example, the following command starts Elasticsearch only:
214 | 
215 | 	$ sudo docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it \
216 | 		-e LOGSTASH_START=0 -e KIBANA_START=0 --name elk sebp/elk
217 | 
218 | Note that if the container is to be started with Elasticsearch _disabled_, then:
219 | 
220 | - If Logstash is enabled, then you need to make sure that the configuration file for Logstash's Elasticsearch output plugin (`/etc/logstash/conf.d/30-output.conf`) points to a host belonging to the Elasticsearch cluster rather than `localhost` (which is the default in the ELK image, since by default Elasticsearch and Logstash run together), e.g.:
221 | 	
222 | 		output {
223 | 		  elasticsearch { hosts => ["elk-master.example.com"] }
224 | 		}
225 | 
226 | - Similarly, if Kibana is enabled, then Kibana's `kibana.yml` configuration file must first be updated to make the `elasticsearch.url` setting (default value: `"http://localhost:9200"`) point to a running instance of Elasticsearch.
227 | 
228 | ### Overriding start-up variables <a name="overriding-variables"></a>
229 | 
230 | The following environment variables can be used to override the defaults used to start up the services:
231 | 
232 | - `TZ`: the container's time zone (see [list of valid time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)), e.g. `America/Los_Angeles` (default is `Etc/UTC`, i.e. UTC).
233 | 
234 | - `ES_HEAP_SIZE`: Elasticsearch heap size (default is 50% of container memory, 31G max)
235 | 
236 | 	Specifying a heap size – e.g. `2g` – will set both the min and max to the provided value. To set the min and max values separately, see the `ES_JAVA_OPTS` below. 
237 | 
238 | - `ES_JAVA_OPTS`: additional Java options for Elasticsearch (default: `""`)
239 | 
240 | 	For instance, to set the min and max heap size to 512MB and 2G, set this environment variable to `-Xms512m -Xmx2g`.
241 | 
242 | - `ES_CONNECT_RETRY`: number of seconds to wait for Elasticsearch to be up before starting Logstash and/or Kibana (default: `30`) 
243 | 
244 | - `ES_PROTOCOL`: protocol to use to ping Elasticsearch's JSON interface URL (default: `http`)
245 | 
246 | 	Note that this variable is only used to test if Elasticsearch is up when starting up the services. It is not used to update Elasticsearch's URL in Logstash's and Kibana's configuration files.  
247 | 
248 | - `CLUSTER_NAME`: the name of the Elasticsearch cluster (default: automatically resolved when the container starts if Elasticsearch requires no user authentication).
249 | 
250 | 	The name of the Elasticsearch cluster is used to set the name of the Elasticsearch log file that the container displays when running. By default the name of the cluster is resolved automatically at start-up time (and populates `CLUSTER_NAME`) by querying Elasticsearch's REST API anonymously. However, when Elasticsearch requires user authentication (as is the case by default when running X-Pack for instance), this query fails and the container stops as it assumes that Elasticsearch is not running properly. Therefore, the `CLUSTER_NAME` environment variable can be used to specify the name of the cluster and bypass the (failing) automatic resolution.
251 | 
252 | - `LS_HEAP_SIZE`: Logstash heap size (default: `"500m"`)
253 | 
254 | - `LS_OPTS`: Logstash options (default: `"--auto-reload"` in images with tags `es231_l231_k450` and `es232_l232_k450`, `""` in `latest`; see [Breaking changes](#breaking-changes))
255 | 
256 | - `NODE_OPTIONS`: Node options for Kibana (default: `"--max-old-space-size=250"`)
257 | 
258 | - `MAX_MAP_COUNT`: limit on mmap counts (default: system default)
259 | 
260 | 	**Warning** – This setting is system-dependent: not all systems allow this limit to be set from within the container, you may need to set this from the host before starting the container (see [Prerequisites](#prerequisites)). 
261 | 
262 | - `MAX_OPEN_FILES`: maximum number of open files (default: system default; Elasticsearch needs this amount to be equal to at least 65536)
263 | 
264 | - `KIBANA_CONNECT_RETRY`: number of seconds to wait for Kibana to be up before running the post-hook script (see [Pre-hooks and post-hooks](#pre-post-hooks)) (default: `30`) 
265 | 
266 | - `ES_HEAP_DISABLE` and `LS_HEAP_DISABLE`: disable `HeapDumpOnOutOfMemoryError` for Elasticsearch and Logstash respectively if non-zero (default: `HeapDumpOnOutOfMemoryError` is enabled).
267 | 
268 | 	Setting these environment variables avoids potentially large heap dumps if the services run out of memory.
269 | 
270 | 
271 | As an illustration, the following command starts the stack, running Elasticsarch with a 2GB heap size and Logstash with a 1GB heap size:
272 | 
273 | 	$ sudo docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it \
274 | 		-e ES_HEAP_SIZE="2g" -e LS_HEAP_SIZE="1g" --name elk sebp/elk
275 | 
276 | ### Pre-hooks and post-hooks<a name="pre-post-hooks"></a>
277 | 
278 | Before starting the ELK services, the container will run the script at `/usr/local/bin/elk-pre-hooks.sh` if it exists and is executable.
279 | 
280 | This can in particular be used to expose custom environment variables (in addition to the [default ones supported by the image](#overriding-variables)) to Elasticsearch and Logstash by amending their corresponding `/etc/default` files.
281 | 
282 | For instance, to expose the custom `MY_CUSTOM_VAR` environment variable to Elasticsearch, add an executable `/usr/local/bin/elk-pre-hooks.sh` to the container (e.g. by `ADD`-ing it to a custom `Dockerfile` that extends the base image, or by bind-mounting the file at runtime), with the following contents:
283 | 
284 | 	cat << EOF >> /etc/default/elasticsearch
285 | 	MY_CUSTOM_VAR=$MY_CUSTOM_VAR
286 | 	export MY_CUSTOM_VAR 
287 | 	EOF
288 | 
289 | After starting the ELK services, the container will run the script at `/usr/local/bin/elk-post-hooks.sh` if it exists and is executable.
290 | 
291 | This can for instance be used to add index templates to Elasticsearch or to add index patterns to Kibana after the services have started. 
292 | 
293 | ## Forwarding logs <a name="forwarding-logs"></a>
294 | 
295 | Forwarding logs from a host relies on a forwarding agent that collects logs (e.g. from log files, from the syslog daemon) and sends them to our instance of Logstash.
296 | 
297 | As configured in this image, Logstash expects logs from a [Beats](https://www.elastic.co/products/beats) shipper (e.g. Filebeat) over a secure (SSL/TLS) connection.
298 | 
299 | **Note** – See [this comment](https://github.com/spujadas/elk-docker/issues/264#issuecomment-481191669) for guidance on how to set up a vanilla HTTP listener.
300 | 
301 | ### Forwarding logs with Filebeat <a name="forwarding-logs-filebeat"></a>
302 | 
303 | Install [Filebeat](https://www.elastic.co/products/beats/filebeat) on the host you want to collect and forward logs from (see the *[References](#references)* section for links to detailed instructions).
304 | 
305 | **Note** – Make sure that the version of Filebeat is the same as the version of the ELK image.
306 | 
307 | #### Example Filebeat set-up and configuration
308 | 
309 | **Note** – The `nginx-filebeat` subdirectory of the [source Git repository on GitHub](https://github.com/spujadas/elk-docker) contains a sample `Dockerfile` which enables you to create a Docker image that implements the steps below.
310 | 
311 | Here is a sample `/etc/filebeat/filebeat.yml` configuration file for Filebeat, that forwards syslog and authentication logs, as well as [nginx](http://nginx.org/) logs.
312 | 
313 | 	output:
314 | 	  logstash:
315 | 	    enabled: true
316 | 	    hosts:
317 | 	      - elk:5044
318 | 	    timeout: 15
319 | 	    ssl:
320 | 	      certificate_authorities:
321 | 	  	      - /etc/pki/tls/certs/logstash-beats.crt
322 | 	
323 | 	filebeat:
324 | 	  inputs:
325 | 	    -
326 | 	      paths:
327 | 	        - /var/log/syslog
328 | 	        - /var/log/auth.log
329 | 	      document_type: syslog
330 | 	    -
331 | 	      paths:
332 | 	        - "/var/log/nginx/*.log"
333 | 	      fields_under_root: true
334 | 		  fields:
335 | 		    type: nginx-access
336 | 
337 | In the sample configuration file, make sure that you replace `elk` in `elk:5044` with the hostname or IP address of the ELK-serving host.
338 | 
339 | You'll also need to copy the `logstash-beats.crt` file (which contains the certificate authority's certificate – or server certificate as the certificate is self-signed – for Logstash's Beats input plugin; see [Security considerations](#security-considerations) for more information on certificates) from the [source repository of the ELK image](https://github.com/spujadas/elk-docker) to `/etc/pki/tls/certs/logstash-beats.crt`.
340 | 
341 | **Note** – Alternatively, when using Filebeat on a Windows machine, instead of using the `certificate_authorities` configuration option, the certificate from `logstash-beats.crt` can be installed in Windows' Trusted Root Certificate Authorities store. 
342 | 
343 | **Note** – The ELK image includes configuration items (`/etc/logstash/conf.d/11-nginx.conf` and `/opt/logstash/patterns/nginx`) to parse nginx access logs, as forwarded by the Filebeat instance above.
344 | 
345 | If you're starting Filebeat for the first time, you should load the default index template in Elasticsearch. *At the time of writing, in version 6, loading the index template in Elasticsearch doesn't work, see [Known issues](#known-issues).*
346 | 
347 | Start Filebeat:
348 | 
349 | 		sudo /etc/init.d/filebeat start
350 | 
351 | #### Note on processing multiline log entries
352 | 
353 | In order to process multiline log entries (e.g. stack traces) as a single event using Filebeat, you may want to consider [Filebeat's multiline option](https://www.elastic.co/blog/beats-1-1-0-and-winlogbeat-released), which was introduced in Beats 1.1.0, as a handy alternative to altering Logstash's configuration files to use [Logstash's multiline codec](https://www.elastic.co/guide/en/logstash/current/plugins-codecs-multiline.html).
354 | 
355 | ### Connecting a Docker container to an ELK container running on the same host<a name="connecting-containers"></a>
356 | 
357 | If you want to forward logs from a Docker container to the ELK container on a host, then you need to connect the two containers.
358 | 
359 | **Note** – The log-emitting Docker container must have Filebeat running in it for this to work.
360 | 
361 | First of all, create an isolated, user-defined `bridge` network (we'll call it `elknet`):
362 | 
363 | 	$ sudo docker network create -d bridge elknet
364 | 
365 | Now start the ELK container, giving it a name (e.g. `elk`) using the `--name` option, and specifying the network it must connect to (`elknet` in this example):
366 | 
367 | 	$ sudo docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it \
368 | 		--name elk --network=elknet sebp/elk
369 | 
370 | Then start the log-emitting container on the same network (replacing `your/image` with the name of the Filebeat-enabled image you're forwarding logs from):
371 | 
372 | 	$ sudo docker run -p 80:80 -it --network=elknet your/image
373 | 
374 | From the perspective of the log emitting container, the ELK container is now known as `elk`, which is the hostname to be used under `hosts` in the `filebeat.yml` configuration file.
375 | 
376 | For more information on networking with Docker, see [Docker's documentation on working with `network` commands](https://docs.docker.com/engine/userguide/networking/work-with-networks/). 
377 | 
378 | #### Linking containers without a user-defined network
379 | 
380 | _This is the legacy way of connecting containers over the Docker's default `bridge` network, using links, which are a [deprecated legacy feature of Docker which may eventually be removed](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/)._
381 | 
382 | First of all, give the ELK container a name (e.g. `elk`) using the `--name` option:
383 | 
384 | 	$ sudo docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it --name elk sebp/elk
385 | 
386 | Then start the log-emitting container with the `--link` option (replacing `your/image` with the name of the Filebeat-enabled image you're forwarding logs from):
387 | 
388 | 	$ sudo docker run -p 80:80 -it --link elk:elk your/image
389 | 
390 | From the perspective of the log emitting container, the ELK container is now known as `elk`, which is the hostname to be used under `hosts` in the `filebeat.yml` configuration file.
391 | 
392 | With Compose here's what example entries for a (locally built log-generating) container and an ELK container might look like in the `docker-compose.yml` file.
393 | 
394 | 	yourapp:
395 | 	  image: your/image
396 | 	  ports:
397 | 	    - "80:80"
398 | 	  links:
399 | 	    - elk
400 | 	
401 | 	elk:
402 | 	  image: sebp/elk
403 | 	  ports:
404 | 	    - "5601:5601"
405 | 	    - "9200:9200"
406 | 	    - "5044:5044"
407 | 
408 | ## Building the image <a name="building-image"></a>
409 | 
410 | To build the Docker image from the source files, first clone the [Git repository](https://github.com/spujadas/elk-docker), go to the root of the cloned directory (i.e. the directory that contains `Dockerfile`), and:
411 | 
412 | - If you're using the vanilla `docker` command then run `sudo docker build -t <repository-name> .`, where `<repository-name>` is the repository name to be applied to the image, which you can then use to run the image with the `docker run` command.
413 | 
414 | - If you're using Compose then run `sudo docker-compose build elk`, which uses the `docker-compose.yml` file from the source repository to build the image. You can then run the built image with `sudo docker-compose up`.
415 | 
416 | ### Building the image for ARM64 <a name="building-image-arm64"></a>
417 | 
418 | To build the image for ARM64 (e.g. Raspberry Pi), run the following command:
419 | 
420 | 	$ sudo docker build --build-arg IMAGE=master-arm64 --build-arg ARCH=aarch64 .
421 | 
422 | **Note** – The OSS version of the image cannot be built for ARM64.  
423 | 
424 | ## Tweaking the image <a name="tweaking-image"></a>
425 | 
426 | There are several approaches to tweaking the image:
427 | 
428 | - Use the image as a base image and extend it, adding files (e.g. configuration files to process logs sent by log-producing applications, plugins for Elasticsearch) and overwriting files (e.g. configuration files, certificate and private key files) as required. See Docker's [Dockerfile Reference page](https://docs.docker.com/reference/builder/) for more information on writing a `Dockerfile`.
429 | 
430 | - Replace existing files by bind-mounting local files to files in the container. See Docker's [Manage data in containers](https://docs.docker.com/engine/userguide/containers/dockervolumes/) page for more information on volumes in general and bind-mounting in particular.
431 | 
432 | - Fork the source Git repository and hack away.
433 | 
434 | The next few subsections present some typical use cases.
435 | 
436 | ### Updating Logstash's configuration <a name="updating-logstash-configuration"></a>
437 | 
438 | Generally speaking, the directory layout for Logstash is the one described [here](https://www.elastic.co/guide/en/logstash/current/dir-layout.html#zip-targz-layout). 
439 | 
440 | Logstash's settings are defined by the configuration files (e.g. `logstash.yml`, `jvm.options`, `pipelines.yml`) located in `/opt/logstash/config`.
441 | 
442 | Out of the box the image's `pipelines.yml` configuration file defines a default pipeline, made of the files (e.g. `01-lumberjack-input.conf`, `02-beats-input.conf`) located in `/etc/logstash/conf.d`.
443 | 
444 | **Note** – Somewhat confusingly, the term "configuration file" may be used to refer to the files defining  Logstash's settings or those defining its pipelines (which are probably the ones you want to tweak the most).
445 | 
446 | To modify an existing configuration file (be it a high-level Logstash configuration file, or a pipeline configuration file), you can bind-mount a local configuration file to a configuration file within the container at runtime. For instance, if you want to replace the image's `30-output.conf` configuration file with your local file `/path/to/your-30-output.conf`, then you would add the following `-v` option to your `docker` command line:
447 | 
448 | 	$ sudo docker run ... \
449 | 		-v /path/to/your-30-output.conf:/etc/logstash/conf.d/30-output.conf \
450 | 		...
451 | 
452 | To create your own image with updated or additional configuration files, you can create a `Dockerfile` that extends the original image, with contents such as the following:
453 | 
454 | 	FROM sebp/elk
455 | 	
456 | 	# overwrite existing file
457 | 	ADD /path/to/your-30-output.conf /etc/logstash/conf.d/30-output.conf
458 | 	
459 | 	# add new file
460 | 	ADD /path/to/new-12-some-filter.conf /etc/logstash/conf.d/12-some-filter.conf
461 | 
462 | Then build the extended image using the `docker build` syntax. 
463 | 
464 | ### Updating Elasticsearch’s configuration <a name="updating-elasticsearch-configuration"></a>
465 | 
466 | The directory layout for Elasticsearch is described [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html)
467 | 
468 | Elasticsearch’s configuration file is `elasticsearch.yml`, located in `/etc/elasticsearch`.
469 | 
470 | The easiest way to modify this configuration file is to bind-mount a local configuration file to a configuration file within the container at runtime, as outlined in the introduction of the parent section.
471 | 
472 | 
473 | 
474 | ### Installing Elasticsearch plugins <a name="installing-elasticsearch-plugins"></a>
475 | 
476 | Elasticsearch's home directory in the image is `/opt/elasticsearch`, its [plugin management script](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-plugins.html) (`elasticsearch-plugin`) resides in the `bin` subdirectory, and plugins are installed in `plugins`.
477 | 
478 | Elasticsearch runs as the user `elasticsearch`. To avoid issues with permissions, it is therefore recommended to install Elasticsearch plugins as `elasticsearch`, using the `gosu` command (see below for an example, and references for further details).  
479 | 
480 | A `Dockerfile` like the following will extend the base image and install the [GeoIP processor plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest-geoip.html) (which adds information about the geographical location of IP addresses):
481 | 
482 | 	FROM sebp/elk
483 | 	
484 | 	ENV ES_HOME /opt/elasticsearch
485 | 	WORKDIR ${ES_HOME}
486 | 	
487 | 	RUN yes | CONF_DIR=/etc/elasticsearch gosu elasticsearch bin/elasticsearch-plugin \
488 | 		install -b ingest-geoip
489 | 
490 | You can now build the new image (see the *[Building the image](#building-image)* section above) and run the container in the same way as you did with the base image.
491 | 
492 | ### Installing Logstash plugins <a name="installing-logstash-plugins"></a>
493 | 
494 | The name of Logstash's home directory in the image is stored in the `LOGSTASH_HOME` environment variable (which is set to `/opt/logstash` in the base image). Logstash's plugin management script (`logstash-plugin`) is located in the `bin` subdirectory.
495 | 
496 | Logstash runs as the user `logstash`. To avoid issues with permissions, it is therefore recommended to install Logstash plugins as `logstash`, using the `gosu` command (see below for an example, and references for further details).  
497 | 
498 | The following `Dockerfile` can be used to extend the base image and install the [RSS input plugin](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-rss.html):
499 | 
500 | 	FROM sebp/elk
501 | 	
502 | 	WORKDIR ${LOGSTASH_HOME}
503 | 	RUN gosu logstash bin/logstash-plugin install logstash-input-rss
504 | 
505 | See the *[Building the image](#building-image)* section above for instructions on building the new image. You can then run a container based on this image using the same command line as the one in the *[Usage](#usage)* section.
506 | 
507 | ### Installing Kibana plugins <a name="installing-kibana-plugins"></a>
508 | 
509 | The name of Kibana's home directory in the image is stored in the `KIBANA_HOME` environment variable (which is set to `/opt/kibana` in the base image). Kibana's plugin management script (`kibana-plugin`) is located in the `bin` subdirectory, and plugins are installed in `installedPlugins`.
510 | 
511 | Kibana runs as the user `kibana`. To avoid issues with permissions, it is therefore recommended to install Kibana plugins as `kibana`, using the `gosu` command (see below for an example, and references for further details).  
512 | 
513 | A `Dockerfile` similar to the ones in the sections on Elasticsearch and Logstash plugins can be used to extend the base image and install a Kibana plugin.
514 | 
515 | ## Persisting log data <a name="persisting-log-data"></a>
516 | 
517 | In order to keep log data across container restarts, this image mounts `/var/lib/elasticsearch` — which is the directory that Elasticsearch stores its data in — as a volume.
518 | 
519 | You may however want to use a dedicated data volume to persist this log data, for instance to facilitate back-up and restore operations.
520 | 
521 | One way to do this is to mount a Docker named volume using `docker`'s `-v` option, as in:
522 | 
523 | 	$ sudo docker run -p 5601:5601 -p 9200:9200  -p 5044:5044 \
524 | 		-v elk-data:/var/lib/elasticsearch --name elk sebp/elk
525 | 
526 | This command mounts the named volume `elk-data` to `/var/lib/elasticsearch` (and automatically creates the volume if it doesn't exist; you could also pre-create it manually using `docker volume create elk-data`).
527 | 
528 | **Note** – By design, Docker never deletes a volume automatically (e.g. when no longer used by any container). Whilst this avoids accidental data loss, it also means that things can become messy if you're not managing your volumes properly (e.g. using the `-v` option when removing containers with `docker rm` to also delete the volumes... bearing in mind that the actual volume won't be deleted as long as at least one container is still referencing it, even if it's not running). You can keep track of existing volumes using `docker volume ls`.
529 | 
530 | See Docker's page on [Managing Data in Containers](https://docs.docker.com/engine/userguide/containers/dockervolumes/) and Container42's [Docker In-depth: Volumes](http://container42.com/2014/11/03/docker-indepth-volumes/) page for more information on managing data volumes.
531 | 
532 | In terms of permissions, Elasticsearch data is created by the image's `elasticsearch` user, with UID 991 and GID 991.
533 | 
534 | There is a [known situation](https://github.com/spujadas/elk-docker/issues/69) where SELinux denies access to the mounted volume when running in _enforcing_ mode. The workaround is to use the `setenforce 0` command to run SELinux in _permissive_ mode.
535 | 
536 | ## Snapshot and restore <a name="snapshot-restore"></a>
537 | 
538 | The `/var/backups` directory is registered as the snapshot repository (using the `path.repo` parameter in the `elasticsearch.yml` configuration file). A volume or bind-mount could be used to access this directory and the snapshots from outside the container.
539 | 
540 | For further information on snapshot and restore operations, see the official documentation on [Snapshot and Restore](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html). 
541 | 
542 | ## Setting up an Elasticsearch cluster <a name="elasticsearch-cluster"></a>
543 | 
544 | The ELK image can be used to run an Elasticsearch cluster, either on [separate hosts](#elasticsearch-cluster-different-hosts) or (mainly for test purposes) on a [single host](#elasticsearch-cluster-single-host), as described below.
545 | 
546 | For more (non-Docker-specific) information on setting up an Elasticsearch cluster, see the [Life Inside a Cluster section](https://www.elastic.co/guide/en/elasticsearch/guide/current/distributed-cluster.html) section of the Elasticsearch definitive guide.
547 | 
548 | ### Running Elasticsearch nodes on different hosts <a name="elasticsearch-cluster-different-hosts"></a>
549 | 
550 | To run cluster nodes on different hosts, you'll need to update Elasticsearch's `/etc/elasticsearch/elasticsearch.yml` file in the Docker image so that the nodes can find each other:
551 | 
552 | - Configure the [zen discovery module](http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html), by adding a `discovery.zen.ping.unicast.hosts` directive to point to the IP addresses or hostnames of hosts that should be polled to perform discovery when Elasticsearch is started on each node.
553 | 
554 | - Set up the `network.*` directives as follows:
555 | 
556 | 		network.host: 0.0.0.0
557 | 		network.publish_host: <reachable IP address or FQDN>
558 | 
559 | 	where `reachable IP address` refers to an IP address that other nodes can reach (e.g. a public IP address, or a routed private IP address, but *not* the Docker-assigned internal 172.x.x.x address).
560 | 
561 | - Publish port 9300
562 | 
563 | As an example, start an ELK container as usual on one host, which will act as the first master. Let's assume that the host is called *elk-master.example.com*.
564 | 
565 | Have a look at the cluster's health:
566 | 
567 | 	$ curl http://elk-master.example.com:9200/_cluster/health?pretty
568 | 	{
569 | 	  "cluster_name" : "elasticsearch",
570 | 	  "status" : "yellow",
571 | 	  "timed_out" : false,
572 | 	  "number_of_nodes" : 1,
573 | 	  "number_of_data_nodes" : 1,
574 | 	  "active_primary_shards" : 6,
575 | 	  "active_shards" : 6,
576 | 	  "relocating_shards" : 0,
577 | 	  "initializing_shards" : 0,
578 | 	  "unassigned_shards" : 6,
579 | 	  "delayed_unassigned_shards" : 6,
580 | 	  "number_of_pending_tasks" : 0,
581 | 	  "number_of_in_flight_fetch" : 0,
582 | 	  "task_max_waiting_in_queue_millis" : 0,
583 | 	  "active_shards_percent_as_number" : 50.0
584 | 	}
585 | 
586 | This shows that only one node is up at the moment, and the `yellow` status indicates that all primary shards are active, but not all replica shards are active.
587 | 
588 | Then, on another host, create a file named `elasticsearch-slave.yml` (let's say it's in `/home/elk`), with the following contents:
589 | 
590 | 	network.host: 0.0.0.0
591 | 	network.publish_host: <reachable IP address or FQDN>
592 | 	discovery.zen.ping.unicast.hosts: ["elk-master.example.com"]
593 | 
594 | You can now start an ELK container that uses this configuration file, using the following command (which mounts the configuration files on the host into the container):
595 | 
596 | 	$ sudo docker run -it --rm=true -p 9200:9200 -p 9300:9300 \
597 | 	  -v /home/elk/elasticsearch-slave.yml:/etc/elasticsearch/elasticsearch.yml \
598 | 	  sebp/elk
599 | 
600 | Once Elasticsearch is up, displaying the cluster's health on the original host now shows:
601 | 
602 | 	$ curl http://elk-master.example.com:9200/_cluster/health?pretty
603 | 	{
604 | 	  "cluster_name" : "elasticsearch",
605 | 	  "status" : "green",
606 | 	  "timed_out" : false,
607 | 	  "number_of_nodes" : 2,
608 | 	  "number_of_data_nodes" : 2,
609 | 	  "active_primary_shards" : 6,
610 | 	  "active_shards" : 12,
611 | 	  "relocating_shards" : 0,
612 | 	  "initializing_shards" : 0,
613 | 	  "unassigned_shards" : 0,
614 | 	  "delayed_unassigned_shards" : 0,
615 | 	  "number_of_pending_tasks" : 0,
616 | 	  "number_of_in_flight_fetch" : 0,
617 | 	  "task_max_waiting_in_queue_millis" : 0,
618 | 	  "active_shards_percent_as_number" : 100.0
619 | 	}
620 | 
621 | ### Running Elasticsearch nodes on a single host <a name="elasticsearch-cluster-single-host"></a>
622 | 
623 | Setting up Elasticsearch nodes to run on a single host is similar to running the nodes on different hosts, but the containers need to be linked in order for the nodes to discover each other.
624 | 
625 | Start the first node using the usual `docker` command on the host:
626 | 
627 | 	$ sudo docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it --name elk sebp/elk
628 | 
629 | Now, create a basic `elasticsearch-slave.yml` file containing the following lines:
630 | 
631 | 	network.host: 0.0.0.0
632 | 	discovery.zen.ping.unicast.hosts: ["elk"]
633 | 
634 | Start a node using the following command:
635 | 
636 | 	$ sudo docker run -it --rm=true \
637 | 	  -v /var/sandbox/elk-docker/elasticsearch-slave.yml:/etc/elasticsearch/elasticsearch.yml \
638 | 	  --link elk:elk --name elk-slave sebp/elk
639 | 
640 | Note that Elasticsearch's port is not published to the host's port 9200, as it was already published by the initial ELK container.
641 | 
642 | ### Optimising your Elasticsearch cluster <a name="optimising-elasticsearch-cluster"></a>
643 | 
644 | You can use the ELK image as is to run an Elasticsearch cluster, especially if you're just testing, but to optimise your set-up, you may want to have:
645 | 
646 | - One node running the complete ELK stack, using the ELK image as is.
647 | 
648 | - Several nodes running _only_ Elasticsearch (see *[Starting services selectively](#selective-services)*).
649 | 
650 | An even more optimal way to distribute Elasticsearch, Logstash and Kibana across several nodes or hosts would be to run only the required services on the appropriate nodes or hosts (e.g. Elasticsearch on several hosts, Logstash on a dedicated host, and Kibana on another dedicated host).
651 | 
652 | ## Security considerations <a name="security-considerations"></a>
653 | 
654 | As it stands this image is meant for local test use, and as such hasn't been secured: access to the ELK services is unrestricted, and default authentication server certificates and private keys for the Logstash input plugins are bundled with the image.
655 | 
656 | **Note** – In fact, since version 8 of the image, security has been explicitly disabled, see the [Release notes](#release-notes) section.
657 | 
658 | To harden this image, at the very least you would want to:
659 | 
660 | - Configure the services to run with security enabled, see [Start the Elastic Stack with security enabled](https://www.elastic.co/guide/en/elasticsearch/reference/8.0/configuring-stack-security.html) (Elasticsearch),  [Secure your connection to Elasticsearch](https://www.elastic.co/guide/en/logstash/8.0/ls-security.html) (Logstash), and [Configure security in Kibana](https://www.elastic.co/guide/en/kibana/8.0/using-kibana-with-security.html) (Kibana) for version 8 of the ELK services.
661 | 
662 | - Generate a new self-signed authentication certificate for the Logstash input plugins (see [Notes on certificates](#certificates)) or (better) get a proper certificate from a commercial provider (known as a certificate authority), and keep the private key private.
663 | 
664 |   If on the other hand you want to disable certificate-based server authentication (e.g. in a demo environment), see [Disabling SSL/TLS](#disabling-ssl-tls).
665 | 
666 | ### Notes on certificates <a name="certificates"></a>
667 | 
668 | Dummy server authentication certificates (`/etc/pki/tls/certs/logstash-*.crt`) and private keys (`/etc/pki/tls/private/logstash-*.key`) are included in the image.
669 | 
670 | **Note** – For Logstash 2.4.0 a PKCS#8-formatted private key must be used (see *[Breaking changes](#breaking-changes)* for guidance).
671 | 
672 | The certificates are assigned to hostname `*`, which means that they will work if you are using a single-part (i.e. no dots) domain name to reference the server from your client.
673 | 
674 | **Example** – In your client (e.g. Filebeat), sending logs to hostname `elk` will work, `elk.mydomain.com` will not (will produce an error along the lines of `x509: certificate is valid for *, not elk.mydomain.com`), neither will an IP address such as 192.168.0.1 (expect `x509: cannot validate certificate for 192.168.0.1 because it doesn't contain any IP SANs`).
675 | 
676 | If you cannot use a single-part domain name, then you could consider:
677 | 
678 | - Issuing a self-signed certificate with the right hostname using a variant of the commands given below.
679 | 
680 | - Issuing a certificate with the [IP address of the ELK stack in the subject alternative name field](https://github.com/elastic/logstash-forwarder/issues/221#issuecomment-48390920), even though this is bad practice in general as IP addresses are likely to change.
681 | 
682 | - Adding a single-part hostname (e.g. `elk`) to your client's `/etc/hosts` file.    
683 | 
684 | The following commands will generate a private key and a 10-year self-signed certificate issued to a server with hostname `elk` for the Beats input plugin:
685 | 
686 | 	$ cd /etc/pki/tls
687 | 	$ sudo openssl req -x509 -batch -nodes -subj "/CN=elk/" \
688 | 		-days 3650 -newkey rsa:2048 \
689 | 		-keyout private/logstash-beats.key -out certs/logstash-beats.crt
690 | 
691 | As another example, when running a non-predefined number of containers concurrently in a cluster with hostnames _directly_ under the `.mydomain.com` domain (e.g. `elk1.mydomain.com`, `elk2.mydomain.com`, etc.; _not_ `elk1.subdomain.mydomain.com`, `elk2.othersubdomain.mydomain.com` etc.), you could create a certificate assigned to the wildcard hostname `*.example.com` by using the following command (all other parameters are identical to the ones in the previous example).   
692 | 
693 | 	$ cd /etc/pki/tls
694 | 	$ sudo openssl req -x509 -batch -nodes -subj "/CN=*.example.com/" \
695 | 		-days 3650 -newkey rsa:2048 \
696 | 		-keyout private/logstash-beats.key -out certs/logstash-beats.crt
697 | 
698 | To make Logstash use the generated certificate to authenticate to a Beats client, extend the ELK image to overwrite (e.g. using the `Dockerfile` directive `ADD`):
699 | 
700 | - the certificate file (`logstash-beats.crt`) with `/etc/pki/tls/certs/logstash-beats.crt`.
701 | - the private key file (`logstash-beats.key`) with `/etc/pki/tls/private/logstash-beats.key`,
702 | 
703 | Additionally, remember to configure your Beats client to trust the newly created certificate using the `certificate_authorities` directive, as presented in [Forwarding logs with Filebeat](#forwarding-logs-filebeat).
704 | 
705 | ### Disabling SSL/TLS <a name="disabling-ssl-tls"></a>
706 | 
707 | Certificate-based server authentication requires log-producing clients to trust the server's root certificate authority's certificate, which can be an unnecessary hassle in zero-criticality environments (e.g. demo environments, sandboxes).
708 | 
709 | To disable certificate-based server authentication, remove all `ssl` and `ssl`-prefixed directives (e.g. `ssl_certificate`, `ssl_key`) in Logstash's input plugin configuration files.
710 | 
711 | For instance, with the default configuration files in the image, replace the contents of `02-beats-input.conf` (for Beats emitters) with:
712 | 
713 | 	input {
714 | 	  beats {
715 | 	    port => 5044
716 | 	  }
717 | 	}
718 | 
719 | ## Frequently encountered issues <a name="frequent-issues"></a>
720 | 
721 | ### Elasticsearch is not starting (1): `max virtual memory areas vm.max_map_count [65530] likely too low, increase to at least [262144]` <a name="es-not-starting-max-map-count"></a>
722 | 
723 | If the container stops and its logs include the message `max virtual memory areas vm.max_map_count [65530] likely too low, increase to at least [262144]`, then the limits on mmap counts are too low, see [Prerequisites](#prerequisites).
724 | 
725 | ### Elasticsearch is not starting (2): `cat: /var/log/elasticsearch/elasticsearch.log: No such file or directory` <a name="es-not-starting-not-enough-memory"></a>
726 | 
727 | If Elasticsearch's logs are *not* dumped (i.e. you get the following message: `cat: /var/log/elasticsearch/elasticsearch.log: No such file or directory`), then Elasticsearch did not have enough memory to start, see [Prerequisites](#prerequisites).
728 | 
729 | ### Elasticsearch is not starting (3): bootstrap tests <a name="es-not-starting-bootstrap-tests"></a>
730 | 
731 | **As from version 5**, if Elasticsearch is no longer starting, i.e. the `waiting for Elasticsearch to be up (xx/30)` counter goes up to 30 and the container exits with `Couln't start Elasticsearch. Exiting.` _and_ Elasticsearch's logs are dumped, then read the recommendations in the logs and consider that they *must* be applied.
732 | 
733 | In particular, in case (1) above, the message `max virtual memory areas vm.max_map_count [65530] likely too low, increase to at least [262144]` means that the host's limits on mmap counts **must** be set to at least 262144.
734 | 
735 | Another example is `max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]`. In this case, the host's limits on open files (as displayed by `ulimit -n`) must be increased (see [File Descriptors](https://www.elastic.co/guide/en/elasticsearch/reference/current/file-descriptors.html) in Elasticsearch documentation); and Docker's `ulimit` settings must be adjusted, either for the container (using [`docker run`'s `--ulimit` option](https://docs.docker.com/engine/reference/commandline/run/#set-ulimits-in-container---ulimit) or [Docker Compose's `ulimits` configuration option](https://docs.docker.com/compose/compose-file/#ulimits)) or globally (e.g. in `/etc/sysconfig/docker`, add `OPTIONS="--default-ulimit nofile=1024:65536"`). 
736 | 
737 | ### Elasticsearch is not starting (4): no errors in log <a name="es-not-starting-timeout"></a>
738 | 
739 | If Elasticsearch’s logs are dumped with no apparent error, then it may not have had enough time to start within the default window of 30 seconds.
740 | 
741 | In that case, you should set the `ES_CONNECT_RETRY` environment variable (see [Overriding start-up variables](#overriding-variables)) to a larger value to give Elasticsearch enough time to start running.
742 | 
743 | ### Elasticsearch is suddenly stopping after having started properly <a name="es-suddenly-stopping"></a>
744 | 
745 | With the default image, this is usually due to Elasticsearch running out of memory after the other services are started, and the corresponding process being (silently) killed.
746 | 
747 | As a reminder (see [Prerequisites](#prerequisites)), you should use no less than 3GB of memory to run the container... and possibly much more.
748 | 
749 | ### Miscellaneous <a name="issues-misc"></a>
750 | 
751 | Other known issues include:
752 | 
753 | - Elasticsearch not having enough time to start up with the default image settings: in that case [set the `ES_CONNECT_RETRY` environment variable](#overriding-variables) to a value larger than 30. (By default Elasticsearch has 30 seconds to start before other services are started, which may not be enough and cause the container to stop.)
754 | 
755 | - Incorrect proxy settings, e.g. if a proxy is defined for Docker, ensure that connections to `localhost` are not proxied (e.g. by using a `no_proxy` setting).
756 | 
757 | 
758 | ## Assorted hints <a name="assorted-hints"></a>
759 | 
760 | A `docker-compose.yml` file that provides a quick and easy ELK deployment with Kibana proxied through traefik with basic authentication can be found at https://github.com/spujadas/elk-docker/issues/374.
761 | 
762 | ## Troubleshooting <a name="troubleshooting"></a>
763 | 
764 | **Important** – If you need help to troubleshoot the configuration of Elasticsearch, Logstash, or Kibana, regardless of where the services are running (in a Docker container or not), please head over to the [Elastic forums](https://discuss.elastic.co/). The troubleshooting guidelines below only apply to running a container using the ELK Docker image.
765 | 
766 | Here are a few pointers to help you troubleshoot your containerised ELK.
767 | 
768 | ### If Elasticsearch isn't starting... <a name="es-not-starting"></a>
769 | 
770 | If the suggestions listed in [Frequently encountered issues](#frequent-issues) don't help, then an additional way of working out why Elasticsearch isn't starting is to:
771 | 
772 | - Start a container with the `bash` command:
773 | 	
774 | 		$ sudo docker run -it docker_elk bash
775 | 
776 | - Start Elasticsearch manually to look at what it outputs:
777 | 	
778 | 		$ ES_PATH_CONF=/etc/elasticsearch gosu elasticsearch /opt/elasticsearch/bin/elasticsearch \
779 | 			-Epath.logs=/var/log/elasticsearch \
780 | 			-Epath.data=/var/lib/elasticsearch
781 | 
782 | ### If your log-emitting client doesn't seem to be able to reach Logstash... <a name="logstash-unreachable"></a>
783 | 
784 | **Note** – Similar troubleshooting steps are applicable in set-ups where logs are sent directly to Elasticsearch.
785 | 
786 | Make sure that:
787 | 
788 | - You started the container with the right ports open (e.g. 5044 for Beats).
789 | 
790 | - If you are using Filebeat, its version is the same as the version of the ELK image/stack.
791 | 
792 | - The ports are reachable from the client machine (e.g. make sure the appropriate rules have been set up on your firewalls to authorise outbound flows from your client and inbound flows on your ELK-hosting machine).
793 | 
794 | - Your client is configured to connect to Logstash using TLS (or SSL) and that it trusts Logstash's self-signed certificate (or certificate authority if you replaced the default certificate with a proper certificate – see [Security considerations](#security-considerations)).
795 | 
796 |   	To check if Logstash is authenticating using the right certificate, check for errors in the output of
797 | 		
798 | 		$ openssl s_client -connect localhost:5044 -CAfile logstash-beats.crt
799 | 
800 | 	where `logstash-beats.crt` is the name of the file containing Logstash's self-signed certificate.
801 | 
802 | ### Additional tips <a name="general-troubleshooting"></a>
803 | 
804 | If the suggestions given above don't solve your issue, then you should have a look at:
805 | 
806 | - Your log-emitting client's logs.
807 | 
808 | - ELK's logs, by `docker exec`'ing into the running container (see [Creating a dummy log entry](#creating-dummy-log-entry)), turning on stdout log (see [plugins-outputs-stdout](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-stdout.html)), and checking Logstash's logs (located in `/var/log/logstash`), Elasticsearch's logs (in `/var/log/elasticsearch`), and Kibana's logs (in `/var/log/kibana`).
809 | 
810 | 	Note that ELK's logs are rotated daily and are deleted after a week, using logrotate. You can change this behaviour by overwriting the `elasticsearch`, `logstash` and `kibana` files in `/etc/logrotate.d`.  
811 | 
812 | ## Reporting issues <a name="reporting-issues"></a>
813 | 
814 | **Important** – For _non-Docker-related_ issues with Elasticsearch, Kibana, and Elasticsearch, report the issues on the appropriate [Elasticsearch](https://github.com/elastic/elasticsearch), [Logstash](https://github.com/elastic/logstash), or [Kibana](https://github.com/elastic/kibana) GitHub repository. 
815 | 
816 | You can report issues with this image using [GitHub's issue tracker](https://github.com/spujadas/elk-docker/issues) (please avoid raising issues as comments on Docker Hub, if only for the fact that the notification system is broken at the time of writing so there's a fair chance that I won't see it for a while).
817 | 
818 | Bearing in mind that the first thing I'll need to do is reproduce your issue, please provide as much relevant information (e.g. logs, configuration files, what you were expecting and what you got instead, any troubleshooting steps that you took, what _is_ working) as possible for me to do that.
819 | 
820 | [Pull requests](https://github.com/spujadas/elk-docker/pulls) are also welcome if you have found an issue and can solve it.
821 | 
822 | ## Breaking changes <a name="breaking changes"></a>
823 | 
824 | Here is the list of breaking changes that may have side effects when upgrading to later versions of the ELK image: 
825 | 
826 | - **Java 11**
827 | 
828 | 	*Applies to tags: `780` and later.*
829 | 
830 | 	Since tag `es234_l234_k452`, this image used Java 8.
831 | 
832 | 	As Java 8 will no longer be supported by the ELK stack, as of tag `780`, Elasticsearch uses the version of OpenJDK that it is bundled with (OpenJDK 11), and Logstash uses a separately installed OpenJDK 11 package. This may have unintended side effects on plugins that rely on Java.
833 | 
834 | 
835 | - **`path.repo`**
836 | 
837 | 	*Applies to tags: after `623`.*
838 | 
839 | 	Elasticsearch's `path.repo` parameter is predefined as `/var/backups` in `elasticsearch.yml` (see [Snapshot and restore](#snapshot-restore)). 
840 | 
841 | - **Version 6**
842 | 
843 | 	*Applies to tags: `600` and later.*
844 | 
845 | 	Breaking changes are introduced in version 6 of [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/6.x/breaking-changes.html), [Logstash](https://www.elastic.co/guide/en/logstash/6.x/breaking-changes.html), and [Kibana](https://www.elastic.co/guide/en/kibana/6.x/breaking-changes.html).
846 | 
847 | 
848 | - **`ES_HEAP_SIZE` and `LS_HEAP_SIZE`**
849 | 
850 | 	*Applies to tags: `502` to `522`.*
851 | 
852 | 	Overriding the `ES_HEAP_SIZE` and `LS_HEAP_SIZE` environment variables has no effect on the heap size used by Elasticsearch and Logstash (see issue [#129](https://github.com/spujadas/elk-docker/issues/129)).
853 | 
854 | - **Elasticsearch home directory**
855 | 
856 | 	*Applies to tags: `502` and later.*
857 | 
858 | 	Elasticsearch is no longer installed from the `deb` package (which attempts, in version 5.0.2, to modify system files that aren't accessible from a container); instead it is installed from the `tar.gz` package. 
859 | 
860 | 	As a consequence, Elasticsearch's home directory is now `/opt/elasticsearch` (was `/usr/share/elasticsearch`).  
861 | 
862 | - **Version 5**
863 | 
864 | 	*Applies to tags: `es500_l500_k500` and later.*
865 | 
866 | 	Breaking changes are introduced in version 5 of [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/5.0/breaking-changes.html), [Logstash](https://www.elastic.co/guide/en/logstash/5.0/breaking-changes.html), and [Kibana](https://www.elastic.co/guide/en/kibana/5.0/breaking-changes.html).
867 | 
868 | - **Private keys in PKCS#8 format**
869 | 
870 | 	*Applies to tags: `es240_l240_k460` and `es241_l240_k461`.*
871 | 
872 | 	In Logstash version 2.4.x, the private keys used by Logstash with the Beats input [are expected to be in PKCS#8 format](https://github.com/elastic/logstash/issues/5865). To convert the private key (`logstash-beats.key`) from its default PKCS#1 format to PKCS#8, use the following command: 
873 | 
874 | 		$ openssl pkcs8 -in logstash-beats.key -topk8 -nocrypt -out logstash-beats.p8
875 | 
876 | 	and point to the `logstash-beats.p8` file in the `ssl_key` option of Logstash's `02-beats-input.conf` configuration file.
877 | 
878 | - **Logstash forwarder**
879 | 
880 | 	*Applies to tags: `es500_l500_k500` and later.*
881 | 
882 | 	The use of Logstash forwarder is deprecated, its Logstash input plugin configuration has been removed, and port 5000 is no longer exposed.
883 | 
884 | - **UIDs and GIDs**
885 | 
886 | 	*Applies to tags: `es235_l234_k454` and later.*
887 | 
888 | 	Fixed UIDs and GIDs are now assigned to Elasticsearch (both the UID and GID are 991), Logstash (992), and Kibana (993). 
889 | 
890 | - **Java 8**
891 | 
892 | 	*Applies to tags: `es234_l234_k452` and later.*
893 | 
894 | 	This image initially used Oracle JDK 7, which is [no longer updated by Oracle](http://www.oracle.com/technetwork/java/javase/eol-135779.html), and no longer available as a Ubuntu package.
895 | 
896 | 	As from tag `es234_l234_k452`, the image uses Oracle JDK 8. This may have unintended side effects on plugins that rely on Java.
897 | 
898 | - **Logstash configuration auto-reload**
899 | 
900 | 	*Applies to tags: `es231_l231_k450`, `es232_l232_k450`.*
901 | 
902 | 	Logstash's configuration auto-reload option was introduced in Logstash 2.3 and enabled in the images with tags `es231_l231_k450` and `es232_l232_k450`.
903 | 
904 | 	As this feature created a resource leak prior to Logstash 2.3.3 (see [https://github.com/elastic/logstash/issues/5235](https://github.com/elastic/logstash/issues/5235)), the `--auto-reload` option was removed as from the `es233_l232_k451`-tagged image (see [https://github.com/spujadas/elk-docker/issues/41](https://github.com/spujadas/elk-docker/issues/41)).
905 | 
906 | 	Users of images with tags `es231_l231_k450` and `es232_l232_k450` are strongly recommended to override Logstash's options to disable the auto-reload feature by setting the `LS_OPTS` environment variable to `--no-auto-reload` if this feature is not needed.
907 | 
908 | 	To enable auto-reload in later versions of the image:
909 | 
910 | 	- From `es500_l500_k500` onwards: add the `--config.reload.automatic` command-line option to `LS_OPTS`.
911 | 
912 | 	- From `es234_l234_k452` to `es241_l240_k461`: add `--auto-reload` to `LS_OPTS`. 
913 | 
914 | ## Release notes <a name="release-notes"></a>
915 | 
916 | The following information may be helpful when upgrading to later versions of the ELK image:
917 | 
918 | - **Version 8.0**
919 | 
920 |   Elasticsearch security is on by default since version 8.0 of the Elastic stack.
921 | 
922 |   In the opinion of the Docker image’s author, setting up the Elastic stack with security enabled is somewhat fiddly. Security has therefore been disabled in this image to get everything up and running as smoothly as possible out of the box.
923 |   
924 |   See the [Security considerations](#security-considerations) section for information on setting up security.
925 |   
926 |   
927 | 
928 | ## References <a name="references"></a>
929 | 
930 | - [How To Install Elasticsearch, Logstash, and Kibana 4 on Ubuntu 14.04](https://www.digitalocean.com/community/tutorials/how-to-install-elasticsearch-logstash-and-kibana-4-on-ubuntu-14-04)
931 | - [The Docker Book](http://www.dockerbook.com/)
932 | - [The Logstash Book](http://www.logstashbook.com/)
933 | - [Elastic's reference documentation](https://www.elastic.co/guide/index.html):
934 | 	- [Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html)
935 | 	- [Logstash Reference](https://www.elastic.co/guide/en/logstash/current/index.html)
936 | 	- [Kibana Reference](https://www.elastic.co/guide/en/kibana/current/index.html)
937 | 	- [Filebeat Reference](https://www.elastic.co/guide/en/beats/filebeat/current/index.html)
938 | - [gosu, simple Go-based setuid+setgid+setgroups+exec](https://github.com/tianon/gosu), a convenient alternative to `USER` in Dockerfiles (see [Best practices for writing Dockerfiles](https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#user))
939 | 
940 | ## About <a name="about"></a>
941 | 
942 | Written by [Sébastien Pujadas](http://pujadas.net), released under the [Apache 2 license](http://www.apache.org/licenses/LICENSE-2.0).
943 | 


--------------------------------------------------------------------------------
/elasticsearch-default:
--------------------------------------------------------------------------------
 1 | ################################
 2 | # Elasticsearch
 3 | ################################
 4 | 
 5 | # Elasticsearch home directory
 6 | #ES_HOME=/usr/share/elasticsearch
 7 | 
 8 | # Elasticsearch Java path
 9 | #ES_JAVA_HOME=
10 | 
11 | # Elasticsearch configuration directory
12 | #ES_PATH_CONF=/etc/elasticsearch
13 | 
14 | # Elasticsearch PID directory
15 | #PID_DIR=/var/run/elasticsearch
16 | 
17 | # Additional Java OPTS
18 | #ES_JAVA_OPTS=
19 | 
20 | # Configure restart on package upgrade (true, every other setting will lead to not restarting)
21 | #RESTART_ON_UPGRADE=true
22 | 
23 | ################################
24 | # Elasticsearch service
25 | ################################
26 | 
27 | # SysV init.d
28 | #
29 | # The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
30 | ES_STARTUP_SLEEP_TIME=5
31 | 
32 | ################################
33 | # System properties
34 | ################################
35 | 
36 | # Specifies the maximum file descriptor number that can be opened by this process
37 | # When using Systemd, this setting is ignored and the LimitNOFILE defined in
38 | # /usr/lib/systemd/system/elasticsearch.service takes precedence
39 | #MAX_OPEN_FILES=65536
40 | 
41 | # The maximum number of bytes of memory that may be locked into RAM
42 | # Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
43 | # in elasticsearch.yml.
44 | # When using Systemd, the LimitMEMLOCK property must be set
45 | # in /usr/lib/systemd/system/elasticsearch.service
46 | #MAX_LOCKED_MEMORY=unlimited
47 | 
48 | # Maximum number of VMA (Virtual Memory Areas) a process can own
49 | # When using Systemd, this setting is ignored and the 'vm.max_map_count'
50 | # property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
51 | #MAX_MAP_COUNT=262144


--------------------------------------------------------------------------------
/elasticsearch-init:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | #
  3 | # /etc/init.d/elasticsearch -- startup script for Elasticsearch
  4 | #
  5 | ### BEGIN INIT INFO
  6 | # Provides:          elasticsearch
  7 | # Required-Start:    $network $remote_fs $named
  8 | # Required-Stop:     $network $remote_fs $named
  9 | # Default-Start:     2 3 4 5
 10 | # Default-Stop:      0 1 6
 11 | # Short-Description: Starts elasticsearch
 12 | # Description:       Starts elasticsearch using start-stop-daemon
 13 | ### END INIT INFO
 14 | 
 15 | PATH=/bin:/usr/bin:/sbin:/usr/sbin
 16 | NAME=elasticsearch
 17 | DESC="Elasticsearch Server"
 18 | DEFAULT=/etc/default/$NAME
 19 | 
 20 | if [ `id -u` -ne 0 ]; then
 21 |     echo "You need root privileges to run this script"
 22 |     exit 1
 23 | fi
 24 | 
 25 | 
 26 | . /lib/lsb/init-functions
 27 | 
 28 | if [ -r /etc/default/rcS ]; then
 29 |         . /etc/default/rcS
 30 | fi
 31 | 
 32 | 
 33 | # The following variables can be overwritten in $DEFAULT
 34 | 
 35 | # Run Elasticsearch as this user ID and group ID
 36 | ES_USER=elasticsearch
 37 | ES_GROUP=elasticsearch
 38 | 
 39 | # Directory where the Elasticsearch binary distribution resides
 40 | ES_HOME=
 41 | 
 42 | # Directory containing Java
 43 | ES_JAVA_HOME=$ES_HOME/jdk
 44 | 
 45 | # Additional Java OPTS
 46 | #ES_JAVA_OPTS=
 47 | 
 48 | # Maximum number of open files
 49 | #MAX_OPEN_FILES=65536
 50 | 
 51 | # Maximum amount of locked memory
 52 | #MAX_LOCKED_MEMORY=
 53 | 
 54 | # Elasticsearch log directory
 55 | LOG_DIR=/var/log/$NAME
 56 | 
 57 | # Elasticsearch data directory
 58 | DATA_DIR=/var/lib/$NAME
 59 | 
 60 | # Elasticsearch configuration directory
 61 | ES_PATH_CONF=/etc/$NAME
 62 | 
 63 | # Maximum number of VMA (Virtual Memory Areas) a process can own
 64 | #MAX_MAP_COUNT=262144
 65 | 
 66 | # Elasticsearch PID file directory
 67 | PID_DIR="/var/run/elasticsearch"
 68 | 
 69 | # End of variables that can be overwritten in $DEFAULT
 70 | 
 71 | # overwrite settings from default file
 72 | if [ -f "$DEFAULT" ]; then
 73 |     . "$DEFAULT"
 74 | fi
 75 | 
 76 | # Define other required variables
 77 | PID_FILE="$PID_DIR/$NAME.pid"
 78 | DAEMON=$ES_HOME/bin/elasticsearch
 79 | DAEMON_OPTS="-d -p $PID_FILE -Epath.logs=$LOG_DIR -Epath.data=$DATA_DIR"
 80 | DAEMON_ENV_VARS="ES_PATH_CONF=$ES_PATH_CONF"
 81 | 
 82 | export ES_JAVA_OPTS
 83 | export JAVA_HOME
 84 | export ES_INCLUDE
 85 | export ES_JVM_OPTIONS
 86 | 
 87 | if [ ! -x "$DAEMON" ]; then
 88 |     echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON"
 89 |     exit 1
 90 | fi
 91 | 
 92 | checkJava() {
 93 |     if [ -x "$JAVA_HOME/bin/java" ]; then
 94 |         JAVA="$JAVA_HOME/bin/java"
 95 |     else
 96 |         JAVA=`which java`
 97 |     fi
 98 | 
 99 |     if [ ! -x "$JAVA" ]; then
100 |         echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
101 |         exit 1
102 |     fi
103 | }
104 | 
105 | case "$1" in
106 |   start)
107 |         checkJava
108 | 
109 |         log_daemon_msg "Starting $DESC"
110 | 
111 |         pid=`pidofproc -p $PID_FILE elasticsearch`
112 |         if [ -n "$pid" ] ; then
113 |             log_begin_msg "Already running."
114 |             log_end_msg 0
115 |             exit 0
116 |         fi
117 | 
118 |         # Ensure that the PID_DIR exists (it is cleaned at OS startup time)
119 |         if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
120 |             mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
121 |         fi
122 |         if [ -n "$PID_FILE" ] && [ ! -e "$PID_FILE" ]; then
123 |             touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE"
124 |         fi
125 | 
126 |         if [ -n "$MAX_OPEN_FILES" ]; then
127 |             ulimit -n $MAX_OPEN_FILES
128 |         fi
129 | 
130 |         if [ -n "$MAX_LOCKED_MEMORY" ]; then
131 |             ulimit -l $MAX_LOCKED_MEMORY
132 |         fi
133 | 
134 |         if [ -n "$MAX_MAP_COUNT" ]; then
135 |             sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
136 |         fi
137 | 
138 |         # Start Daemon
139 |         start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" \
140 |             --exec /usr/bin/env $DAEMON_ENV_VARS $DAEMON -- $DAEMON_OPTS
141 |         return=$?
142 |         if [ $return -eq 0 ]; then
143 |             i=0
144 |             timeout=10
145 |             # Wait for the process to be properly started before exiting
146 |             until { kill -0 `cat "$PID_FILE"`; } >/dev/null 2>&1
147 |             do
148 |                 sleep 1
149 |                 i=$(($i + 1))
150 |                 if [ $i -gt $timeout ]; then
151 |                     log_end_msg 1
152 |                     exit 1
153 |                 fi
154 |                 done
155 |         fi
156 |         log_end_msg $return
157 |         exit $return
158 |         ;;
159 |   stop)
160 |         log_daemon_msg "Stopping $DESC"
161 | 
162 |         if [ -f "$PID_FILE" ]; then
163 |             start-stop-daemon --stop --pidfile "$PID_FILE" \
164 |                 --user "$ES_USER" \
165 |                 --quiet \
166 |                 --retry TERM/60/KILL/5 > /dev/null
167 |             if [ $? -eq 1 ]; then
168 |                 log_progress_msg "$DESC is not running but pid file exists, cleaning up"
169 |             elif [ $? -eq 3 ]; then
170 |                 PID="`cat $PID_FILE`"
171 |                 log_failure_msg "Failed to stop $DESC (pid $PID)"
172 |                 exit 1
173 |             fi
174 |             rm -f "$PID_FILE"
175 |         else
176 |             log_progress_msg "(not running)"
177 |         fi
178 |         log_end_msg 0
179 |         ;;
180 |   status)
181 |         status_of_proc -p $PID_FILE elasticsearch elasticsearch && exit 0 || exit $?
182 |         ;;
183 |   restart|force-reload)
184 |         if [ -f "$PID_FILE" ]; then
185 |             $0 stop
186 |         fi
187 |         $0 start
188 |         ;;
189 |   *)
190 |         log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
191 |         exit 1
192 |         ;;
193 | esac
194 | 
195 | exit 0
196 | 


--------------------------------------------------------------------------------
/elasticsearch-logrotate:
--------------------------------------------------------------------------------
 1 | /var/log/elasticsearch/*.log {
 2 |     daily
 3 |     rotate 7
 4 |     copytruncate
 5 |     compress
 6 |     delaycompress
 7 |     missingok
 8 |     notifempty
 9 | }
10 | 


--------------------------------------------------------------------------------
/elasticsearch.yml:
--------------------------------------------------------------------------------
  1 | # ======================== Elasticsearch Configuration =========================
  2 | #
  3 | # NOTE: Elasticsearch comes with reasonable defaults for most settings.
  4 | #       Before you set out to tweak and tune the configuration, make sure you
  5 | #       understand what are you trying to accomplish and the consequences.
  6 | #
  7 | # The primary way of configuring a node is via this file. This template lists
  8 | # the most important settings you may want to configure for a production cluster.
  9 | #
 10 | # Please consult the documentation for further information on configuration options:
 11 | # https://www.elastic.co/guide/en/elasticsearch/reference/index.html
 12 | #
 13 | # ---------------------------------- Cluster -----------------------------------
 14 | #
 15 | # Use a descriptive name for your cluster:
 16 | #
 17 | #cluster.name: my-application
 18 | #
 19 | # ------------------------------------ Node ------------------------------------
 20 | #
 21 | # Use a descriptive name for the node:
 22 | #
 23 | node.name: elk
 24 | #
 25 | # Add custom attributes to the node:
 26 | #
 27 | #node.attr.rack: r1
 28 | #
 29 | # ----------------------------------- Paths ------------------------------------
 30 | #
 31 | # Path to directory where to store the data (separate multiple locations by comma):
 32 | #
 33 | #path.data: /path/to/data
 34 | #
 35 | # Path to log files:
 36 | #
 37 | #path.logs: /path/to/logs
 38 | #
 39 | # Path to snapshots for backups:
 40 | #
 41 | path.repo: /var/backups
 42 | #
 43 | # ----------------------------------- Memory -----------------------------------
 44 | #
 45 | # Lock the memory on startup:
 46 | #
 47 | #bootstrap.memory_lock: true
 48 | #
 49 | # Make sure that the heap size is set to about half the memory available
 50 | # on the system and that the owner of the process is allowed to use this
 51 | # limit.
 52 | #
 53 | # Elasticsearch performs poorly when the system is swapping the memory.
 54 | #
 55 | # ---------------------------------- Network -----------------------------------
 56 | #
 57 | # Set the bind address to a specific IP (IPv4 or IPv6):
 58 | #
 59 | #network.host: 192.168.0.1
 60 | network.host: 0.0.0.0
 61 | #
 62 | # Set a custom port for HTTP:
 63 | #
 64 | #http.port: 9200
 65 | #
 66 | # For more information, consult the network module documentation.
 67 | #
 68 | # --------------------------------- Discovery ----------------------------------
 69 | #
 70 | # Pass an initial list of hosts to perform discovery when this node is started:
 71 | # The default list of hosts is ["127.0.0.1", "[::1]"]
 72 | #
 73 | #discovery.seed_hosts: ["127.0.0.1", "[::1]"]
 74 | #
 75 | # Bootstrap the cluster using an initial set of master-eligible nodes:
 76 | #
 77 | cluster.initial_master_nodes: ["elk"]
 78 | #
 79 | # For more information, consult the discovery and cluster formation module documentation.
 80 | #
 81 | # ---------------------------------- Gateway -----------------------------------
 82 | #
 83 | # Block initial recovery after a full cluster restart until N nodes are started:
 84 | #
 85 | #gateway.recover_after_nodes: 3
 86 | #
 87 | # For more information, consult the gateway module documentation.
 88 | #
 89 | # ---------------------------------- Various -----------------------------------
 90 | #
 91 | # Require explicit names when deleting indices:
 92 | #
 93 | #action.destructive_requires_name: true
 94 | #
 95 | # ---------------------------------- Security ----------------------------------
 96 | #
 97 | # Enable/disable security (enabled by default since version 8.0)
 98 | #
 99 | xpack.security.enabled: false
100 | 


--------------------------------------------------------------------------------
/kibana-init:
--------------------------------------------------------------------------------
  1 | #!/bin/sh
  2 | #
  3 | # /etc/init.d/kibana5_init -- startup script for kibana5
  4 | # bsmith@the408.com 2015-02-20; used elasticsearch init script as template
  5 | # https://github.com/akabdog/scripts/edit/master/kibana4_init
  6 | # spujadas 2015-04-09; updated to run as non-root user
  7 | #
  8 | ### BEGIN INIT INFO
  9 | # Provides:          kibana5_init
 10 | # Required-Start:    $network $remote_fs $named
 11 | # Required-Stop:     $network $remote_fs $named
 12 | # Default-Start:     2 3 4 5
 13 | # Default-Stop:      0 1 6
 14 | # Short-Description: Starts kibana5_init
 15 | # Description:       Starts kibana5_init using start-stop-daemon
 16 | ### END INIT INFO
 17 | 
 18 | KIBANA_HOME=
 19 | KIBANA_BIN=${KIBANA_HOME}/bin
 20 | 
 21 | NAME=kibana5
 22 | PID_FILE=/var/run/$NAME.pid
 23 | PATH=/bin:/usr/bin:/sbin:/usr/sbin:$KIBANA_BIN
 24 | DAEMON=$KIBANA_BIN/kibana
 25 | KIBANA_USER=kibana
 26 | KIBANA_GROUP=kibana
 27 | KIBANA_LOG_DIR=/var/log/kibana
 28 | KIBANA_LOG_FILE="${KIBANA_LOG_DIR}/$NAME.log"
 29 | DAEMON_OPTS="-l ${KIBANA_LOG_FILE}"
 30 | DESC="Kibana5"
 31 | NODE_OPTIONS="--max-old-space-size=4096"
 32 | 
 33 | if [ $(id -u) -ne 0 ]; then
 34 |   echo "You need root privileges to run this script"
 35 |   exit 1
 36 | fi
 37 | 
 38 | . /lib/lsb/init-functions
 39 | 
 40 | if [ -r /etc/default/rcS ]; then
 41 |   . /etc/default/rcS
 42 | fi
 43 | 
 44 | case "$1" in
 45 |   start)
 46 |     log_daemon_msg "Starting $DESC"
 47 | 
 48 |     pid=$(pidofproc -p $PID_FILE kibana)
 49 |     if [ -n "$pid" ]; then
 50 |       log_begin_msg "Already running."
 51 |       log_end_msg 0
 52 |       exit 0
 53 |     fi
 54 | 
 55 |     touch ${KIBANA_LOG_FILE}
 56 |     chown ${KIBANA_USER}:${KIBANA_GROUP} ${KIBANA_LOG_FILE}
 57 | 
 58 |     # Start Daemon
 59 |     NODE_OPTIONS="$NODE_OPTIONS" start-stop-daemon --start --user $KIBANA_USER -c $KIBANA_USER \
 60 |       --group $KIBANA_GROUP --pidfile "$PID_FILE" --make-pidfile \
 61 |       --background --exec $DAEMON -- $DAEMON_OPTS
 62 |     log_end_msg $?
 63 |     ;;
 64 | 
 65 |   stop)
 66 |     log_daemon_msg "Stopping $DESC"
 67 | 
 68 |     if [ -f "$PID_FILE" ]; then
 69 |       start-stop-daemon --stop --pidfile "$PID_FILE" \
 70 |         --retry TERM/60/KILL/5 >/dev/null
 71 |       if [ $? -eq 1 ]; then
 72 |         log_progress_msg "$DESC is not running but pid file exists, cleaning up"
 73 |       elif [ $? -eq 3 ]; then
 74 |         PID="$(cat $PID_FILE)"
 75 |         log_failure_msg "Failed to stop $DESC (pid $PID)"
 76 |         exit 1
 77 |       fi
 78 |       rm -f "$PID_FILE"
 79 |     else
 80 |       log_progress_msg "(not running)"
 81 |     fi
 82 |     log_end_msg 0
 83 |     ;;
 84 | 
 85 |   status)
 86 |     status_of_proc -p $PID_FILE kibana kibana && exit 0 || exit $?
 87 |     ;;
 88 | 
 89 |   restart|force-reload)
 90 |     if [ -f "$PID_FILE" ]; then
 91 |       $0 stop
 92 |       sleep 1
 93 |     fi
 94 |     $0 start
 95 |     ;;
 96 | 
 97 |   *)
 98 |     log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
 99 |     exit 1
100 |     ;;
101 | esac
102 | 
103 | exit 0
104 | 


--------------------------------------------------------------------------------
/kibana-logrotate:
--------------------------------------------------------------------------------
 1 | /var/log/kibana/kibana5.log {
 2 |     daily
 3 |     rotate 7
 4 |     copytruncate
 5 |     compress
 6 |     delaycompress
 7 |     missingok
 8 |     notifempty
 9 | }
10 | 


--------------------------------------------------------------------------------
/kibana.yml:
--------------------------------------------------------------------------------
  1 | # For more configuration options see the configuration guide for Kibana in
  2 | # https://www.elastic.co/guide/index.html
  3 | 
  4 | # =================== System: Kibana Server ===================
  5 | # Kibana is served by a back end server. This setting specifies the port to use.
  6 | #server.port: 5601
  7 | 
  8 | # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
  9 | # The default is 'localhost', which usually means remote machines will not be able to connect.
 10 | # To allow connections from remote users, set this parameter to a non-loopback address.
 11 | server.host: "0.0.0.0"
 12 | 
 13 | # Enables you to specify a path to mount Kibana at if you are running behind a proxy.
 14 | # Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath
 15 | # from requests it receives, and to prevent a deprecation warning at startup.
 16 | # This setting cannot end in a slash.
 17 | #server.basePath: ""
 18 | 
 19 | # Specifies whether Kibana should rewrite requests that are prefixed with
 20 | # `server.basePath` or require that they are rewritten by your reverse proxy.
 21 | # Defaults to `false`.
 22 | #server.rewriteBasePath: false
 23 | 
 24 | # Specifies the public URL at which Kibana is available for end users. If
 25 | # `server.basePath` is configured this URL should end with the same basePath.
 26 | #server.publicBaseUrl: ""
 27 | 
 28 | # The maximum payload size in bytes for incoming server requests.
 29 | #server.maxPayload: 1048576
 30 | 
 31 | # The Kibana server's name. This is used for display purposes.
 32 | #server.name: "your-hostname"
 33 | 
 34 | # =================== System: Kibana Server (Optional) ===================
 35 | # Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively.
 36 | # These settings enable SSL for outgoing requests from the Kibana server to the browser.
 37 | #server.ssl.enabled: false
 38 | #server.ssl.certificate: /path/to/your/server.crt
 39 | #server.ssl.key: /path/to/your/server.key
 40 | 
 41 | # =================== System: Elasticsearch ===================
 42 | # The URLs of the Elasticsearch instances to use for all your queries.
 43 | #elasticsearch.hosts: ["http://localhost:9200"]
 44 | 
 45 | # If your Elasticsearch is protected with basic authentication, these settings provide
 46 | # the username and password that the Kibana server uses to perform maintenance on the Kibana
 47 | # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
 48 | # is proxied through the Kibana server.
 49 | #elasticsearch.username: "kibana_system"
 50 | #elasticsearch.password: "pass"
 51 | 
 52 | # Kibana can also authenticate to Elasticsearch via "service account tokens".
 53 | # Service account tokens are Bearer style tokens that replace the traditional username/password based configuration.
 54 | # Use this token instead of a username/password.
 55 | # elasticsearch.serviceAccountToken: "my_token"
 56 | 
 57 | # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
 58 | # the elasticsearch.requestTimeout setting.
 59 | #elasticsearch.pingTimeout: 1500
 60 | 
 61 | # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
 62 | # must be a positive integer.
 63 | #elasticsearch.requestTimeout: 30000
 64 | 
 65 | # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
 66 | # headers, set this value to [] (an empty list).
 67 | #elasticsearch.requestHeadersWhitelist: [ authorization ]
 68 | 
 69 | # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
 70 | # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
 71 | #elasticsearch.customHeaders: {}
 72 | 
 73 | # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
 74 | #elasticsearch.shardTimeout: 30000
 75 | 
 76 | # =================== System: Elasticsearch (Optional) ===================
 77 | # These files are used to verify the identity of Kibana to Elasticsearch and are required when
 78 | # xpack.security.http.ssl.client_authentication in Elasticsearch is set to required.
 79 | #elasticsearch.ssl.certificate: /path/to/your/client.crt
 80 | #elasticsearch.ssl.key: /path/to/your/client.key
 81 | 
 82 | # Enables you to specify a path to the PEM file for the certificate
 83 | # authority for your Elasticsearch instance.
 84 | #elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
 85 | 
 86 | # To disregard the validity of SSL certificates, change this setting's value to 'none'.
 87 | #elasticsearch.ssl.verificationMode: full
 88 | 
 89 | # =================== System: Logging ===================
 90 | # Set the value of this setting to off to suppress all logging output, or to debug to log everything. Defaults to 'error'
 91 | #logging.root.level: debug
 92 | 
 93 | # Enables you to specify a file where Kibana stores log output.
 94 | #logging.appenders.default:
 95 | #  type: file
 96 | #  fileName: /var/logs/kibana.log
 97 | #  layout:
 98 | #    type: json
 99 | 
100 | # Logs queries sent to Elasticsearch.
101 | #logging.loggers:
102 | #  - name: elasticsearch.query
103 | #    level: debug
104 | 
105 | # Logs http responses.
106 | #logging.loggers:
107 | #  - name: http.server.response
108 | #    level: debug
109 | 
110 | # Logs system usage information.
111 | #logging.loggers:
112 | #  - name: metrics.ops
113 | #    level: debug
114 | 
115 | # =================== System: Other ===================
116 | # The path where Kibana stores persistent data not saved in Elasticsearch. Defaults to data
117 | #path.data: data
118 | 
119 | # Specifies the path where Kibana creates the process ID file.
120 | #pid.file: /run/kibana/kibana.pid
121 | 
122 | # Set the interval in milliseconds to sample system and process performance
123 | # metrics. Minimum is 100ms. Defaults to 5000.
124 | #ops.interval: 5000
125 | 
126 | # Specifies locale to be used for all localizable strings, dates and number formats.
127 | # Supported languages are the following: English - en , by default , Chinese - zh-CN .
128 | #i18n.locale: "en"
129 | 
130 | # =================== Frequently used (Optional)===================
131 | 
132 | # =================== Saved Objects: Migrations ===================
133 | # Saved object migrations run at startup. If you run into migration-related issues, you might need to adjust these settings.
134 | 
135 | # The number of documents migrated at a time.
136 | # If Kibana can't start up or upgrade due to an Elasticsearch `circuit_breaking_exception`,
137 | # use a smaller batchSize value to reduce the memory pressure. Defaults to 1000
138 | # migrations.batchSize: 1000
139 | 
140 | # The maximum payload size for indexing batches of upgraded saved objects.
141 | # To avoid migrations failing due to a 413 Request Entity Too Large response from Elasticsearch.
142 | # This value should be lower than or equal to your Elasticsearch cluster’s `http.max_content_length`
143 | # configuration option. Default: 100mb
144 | # migrations.maxBatchSizeBytes: 100mb
145 | 
146 | # The number of times to retry temporary migration failures. Increase the setting
147 | # if migrations fail frequently with a message such as `Unable to complete the [...] step after
148 | # 15 attempts, terminating`. Defaults to 15
149 | # migrations.retryAttempts: 15
150 | 
151 | # =================== Search Autocomplete ===================
152 | # Time in milliseconds to wait for autocomplete suggestions from Elasticsearch.
153 | # This value must be a whole number greater than zero. Defaults to 1000
154 | # data.autocomplete.valueSuggestions.timeout: 1000
155 | 
156 | # Maximum number of documents loaded by each shard to generate autocomplete suggestions.
157 | # This value must be a whole number greater than zero. Defaults to 100000
158 | # data.autocomplete.valueSuggestions.terminateAfter: 100000
159 | 


--------------------------------------------------------------------------------
/logstash-beats.crt:
--------------------------------------------------------------------------------
 1 | -----BEGIN CERTIFICATE-----
 2 | MIIC6zCCAdOgAwIBAgIJANPZwuf+5wTLMA0GCSqGSIb3DQEBCwUAMAwxCjAIBgNV
 3 | BAMMASowHhcNMTUxMjI4MTA0NTMyWhcNMjUxMjI1MTA0NTMyWjAMMQowCAYDVQQD
 4 | DAEqMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp+jHFvhyYKiPXc7k
 5 | 0c33f2QV+1hHNyW/uwcJbp5jG82cuQ41v70Z1+b2veBW4sUlDY3yAIEOPSUD8ASt
 6 | 9m72CAo4xlwYKDvm/Sa3KJtDk0NrQiz6PPyBUFsY+Bj3xn6Nz1RW5YaP+Q1Hjnks
 7 | PEyQu4vLgfTSGYBHLD4gvs8wDWY7aaKf8DfuP7Ov74Qlj2GOxnmiDEF4tirlko0r
 8 | qQcvBgujCqA7rNoG+QDmkn3VrxtX8mKF72bxQ7USCyoxD4cWV2mU2HD2Maed3KHj
 9 | KAvDAzSyBMjI+qi9IlPN5MR7rVqUV0VlSKXBVPct6NG7x4WRwnoKjTXnr3CRADD0
10 | 4uvbQQIDAQABo1AwTjAdBgNVHQ4EFgQUVFurgDwdcgnCYxszc0dWMWhB3DswHwYD
11 | VR0jBBgwFoAUVFurgDwdcgnCYxszc0dWMWhB3DswDAYDVR0TBAUwAwEB/zANBgkq
12 | hkiG9w0BAQsFAAOCAQEAaLSytepMb5LXzOPr9OiuZjTk21a2C84k96f4uqGqKV/s
13 | okTTKD0NdeY/IUIINMq4/ERiqn6YDgPgHIYvQheWqnJ8ir69ODcYCpsMXIPau1ow
14 | T8c108BEHqBMEjkOQ5LrEjyvLa/29qJ5JsSSiULHvS917nVgY6xhcnRZ0AhuJkiI
15 | ARKXwpO5tqJi6BtgzX/3VDSOgVZbvX1uX51Fe9gWwPDgipnYaE/t9TGzJEhKwSah
16 | kNr+7RM+Glsv9rx1KcWcx4xxY3basG3/KwvsGAFPvk5tXbZ780VuNFTTZw7q3p8O
17 | Gk1zQUBOie0naS0afype5qFMPp586SF/2xAeb68gLg==
18 | -----END CERTIFICATE-----
19 | 


--------------------------------------------------------------------------------
/logstash-beats.key:
--------------------------------------------------------------------------------
 1 | -----BEGIN PRIVATE KEY-----
 2 | MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCn6McW+HJgqI9d
 3 | zuTRzfd/ZBX7WEc3Jb+7BwlunmMbzZy5DjW/vRnX5va94FbixSUNjfIAgQ49JQPw
 4 | BK32bvYICjjGXBgoO+b9Jrcom0OTQ2tCLPo8/IFQWxj4GPfGfo3PVFblho/5DUeO
 5 | eSw8TJC7i8uB9NIZgEcsPiC+zzANZjtpop/wN+4/s6/vhCWPYY7GeaIMQXi2KuWS
 6 | jSupBy8GC6MKoDus2gb5AOaSfdWvG1fyYoXvZvFDtRILKjEPhxZXaZTYcPYxp53c
 7 | oeMoC8MDNLIEyMj6qL0iU83kxHutWpRXRWVIpcFU9y3o0bvHhZHCegqNNeevcJEA
 8 | MPTi69tBAgMBAAECggEAGh1xQZhYqcHtsmhoXF1NfinB5XrAcMpVPLCGfgbyYTOk
 9 | iX+1SmIN7++DNtr6iICjF62ZEwz/evET4LPJnsd5SpzUYb2XIELY1Uy9NfqYEwJs
10 | XzmBnhSjxCy3AHdZqiyqv7FdZot8Pv8avwUHpUU/SXwfpdG/D6pM54uuKh8tWRfp
11 | 6Fun0x3tFLhr1iY/jwxXx+V5zZ1A7AyHnelSv1u7gnxd2WPNJsyoYp7iWkSbdmjr
12 | fThE8CsTSgL/ndKOmxnPLs7l7ZUipyBgwjmUoZxCZk1I4w9njXY9mti67+6/SAj7
13 | i26/c7p6H31C+FAqksdKOHWh+zCg8zf1kMW3x3P3MQKBgQDUncHjIZ2rDuoWrOIc
14 | ng6IbuyuSHjUDCs59Z2HQbAVe/0IgDKNspdfddgC5XSN8q5GIWvNQtJjQzuCBRJC
15 | SxKkncOcTH6J3eQ4e4+sqDbHIagHQwuSBQRYkjd/KN63HEEJoh5UB/r/77CRdOhT
16 | m6dBm7QvrlXUpcm+z2V3TwvahQKBgQDKK7Qg0mBKm8QVneZUqGRbzaLcOsDefzdP
17 | IKRhWphAia70z6eoPHgR5MaqCFTPAajFaX6xBzIkPT8g8YUMPHhnw5eJRh/58hbU
18 | 84KBw9jNGjE+H+OTT8+qLicP9EoMOeSVknYIX/zPj7xww0w2mF6tYroKKsiQHZhv
19 | eB16YjqAjQKBgD1BCffm2mbK0DQiMK5v9t3lnziC1pS4wMdc9Lpf+VvnMbn+PRJH
20 | roapC8eh1ZeDoCPCQy2Kn9RLLVzDG0SQHlngvddMznPnwnVnW7gxaj6qep9E+JNj
21 | 8KGX1ndDDg8RC8e7tiMdfXm401TEqp5TzLcBJcNK5Z1y+hGH7MKXumGFAoGATmkI
22 | 4bn2Url7IY8uKCNvWRO2WIgJCcJ5Zx0X5BJI/q7nxldLhTp+ryH10ziL/AV+uaIi
23 | 2vIZhmiitVo26foCEOyRN1KVUFGOfWU8dqvIyDOiaZ/gmd/YgP6Jc+yhU4CYoVI+
24 | qRzhZncu9OUqB/qsrb6evRa+1vZDiughNrgmTHkCgYBggzulPfHEBqhlcg6PCsUj
25 | W6YcxEEPojkPPBsG/aMkGOmCr75I2w81lcjyUv54AiVXpSKqgvs+Zg3nF1WRxMVG
26 | vevXsCc4wdJPn669J68uh34eHvMBJQ8I00P7tBcW1RjXpvaH/HLUMO51vnfMwBtR
27 | OrW1ssSF9AvUG1VUmnMSgA==
28 | -----END PRIVATE KEY-----
29 | 


--------------------------------------------------------------------------------
/logstash-conf/02-beats-input.conf:
--------------------------------------------------------------------------------
1 | input {
2 |   beats {
3 |     port => 5044
4 |     ssl => true
5 |     ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
6 |     ssl_key => "/etc/pki/tls/private/logstash-beats.key"
7 |   }
8 | }
9 | 


--------------------------------------------------------------------------------
/logstash-conf/10-syslog.conf:
--------------------------------------------------------------------------------
 1 | filter {
 2 |   if [type] == "syslog" {
 3 |     grok {
 4 |       match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
 5 |       add_field => [ "received_at", "%{@timestamp}" ]
 6 |       add_field => [ "received_from", "%{host}" ]
 7 |     }
 8 |     syslog_pri { }
 9 |     date {
10 |       match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
11 |     }
12 |   }
13 | }
14 | 


--------------------------------------------------------------------------------
/logstash-conf/11-nginx.conf:
--------------------------------------------------------------------------------
1 | filter {
2 |   if [type] == "nginx-access" {
3 |     grok {
4 |       match => { "message" => "%{NGINXACCESS}" }
5 |     }
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/logstash-conf/30-output.conf:
--------------------------------------------------------------------------------
1 | output {
2 |   elasticsearch {
3 |     hosts => ["localhost"]
4 |     manage_template => false
5 |     index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/logstash-init:
--------------------------------------------------------------------------------
  1 | #!/bin/sh
  2 | # Init script for logstash
  3 | # Maintained by Elasticsearch
  4 | # Generated by pleaserun.
  5 | # Implemented based on LSB Core 3.1:
  6 | #   * Sections: 20.2, 20.3
  7 | # spujadas 2015-05-21; emptied LS_HOME (updated by Dockerfile)
  8 | #
  9 | ### BEGIN INIT INFO
 10 | # Provides:          logstash
 11 | # Required-Start:    $remote_fs $syslog
 12 | # Required-Stop:     $remote_fs $syslog
 13 | # Default-Start:     2 3 4 5
 14 | # Default-Stop:      0 1 6
 15 | # Short-Description:
 16 | # Description:        Starts Logstash as a daemon.
 17 | ### END INIT INFO
 18 | 
 19 | PATH=/sbin:/usr/sbin:/bin:/usr/bin
 20 | export PATH
 21 | 
 22 | if [ $(id -u) -ne 0 ]; then
 23 |    echo "You need root privileges to run this script"
 24 |    exit 1
 25 | fi
 26 | 
 27 | name=logstash
 28 | pidfile="/var/run/$name.pid"
 29 | 
 30 | LS_USER=logstash
 31 | LS_GROUP=logstash
 32 | LS_HOME=
 33 | LS_HEAP_SIZE="500m"
 34 | LS_JAVA_OPTS="-Djava.io.tmpdir=${LS_HOME}"
 35 | LS_LOG_DIR=/var/log/logstash
 36 | LS_LOG_FILE="${LS_LOG_DIR}/${name}-plain.log"
 37 | LS_OPEN_FILES=16384
 38 | LS_NICE=19
 39 | LS_OPTS=
 40 | 
 41 | [ -r /etc/default/$name ] && . /etc/default/$name
 42 | [ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name
 43 | 
 44 | program=/opt/logstash/bin/logstash
 45 | args="--path.logs ${LS_LOG_DIR} ${LS_OPTS}"
 46 | 
 47 | start() {
 48 |   HOME=${LS_HOME}
 49 |   
 50 |   ## removing/updating next lines as overriding JAVA_OPTS prevents Logstash
 51 |   ## from starting
 52 |   #JAVA_OPTS=${LS_JAVA_OPTS}
 53 |   #export PATH HOME JAVA_OPTS LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING
 54 |   
 55 |   export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING
 56 | 
 57 |   touch ${LS_LOG_FILE}
 58 |   chown ${LS_USER}:${LS_GROUP} ${LS_LOG_FILE}
 59 | 
 60 |   # set ulimit as (root, presumably) first, before we drop privileges
 61 |   ulimit -n ${LS_OPEN_FILES}
 62 | 
 63 |   # Run the program!
 64 |   nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP / sh -c "
 65 |     cd $LS_HOME
 66 |     ulimit -n ${LS_OPEN_FILES}
 67 |     exec \"$program\" $args
 68 |   " > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &
 69 | 
 70 |   # Generate the pidfile from here. If we instead made the forked process
 71 |   # generate it there will be a race condition between the pidfile writing
 72 |   # and a process possibly asking for status.
 73 |   echo $! > $pidfile
 74 | 
 75 |   echo "$name started."
 76 |   return 0
 77 | }
 78 | 
 79 | stop() {
 80 |   # Try a few times to kill TERM the program
 81 |   if status; then
 82 |     pid=$(cat "$pidfile")
 83 |     echo "Killing $name (pid $pid) with SIGTERM"
 84 |     kill -TERM $pid
 85 |     # Wait for it to exit.
 86 |     for i in 1 2 3 4 5; do
 87 |       echo "Waiting for $name (pid $pid) to die..."
 88 |       status || break
 89 |       sleep 1
 90 |     done
 91 |     if status; then
 92 |       echo "$name stop failed; still running."
 93 |     else
 94 |       echo "$name stopped."
 95 |       rm -f $pidfile
 96 |     fi
 97 |   fi
 98 | }
 99 | 
100 | status() {
101 |   if [ -f "$pidfile" ] ; then
102 |     pid=$(cat "$pidfile")
103 |     if kill -0 $pid > /dev/null 2> /dev/null; then
104 |       # process by this pid is running.
105 |       # It may not be our pid, but that's what you get with just pidfiles.
106 |       # TODO(sissel): Check if this process seems to be the same as the one we
107 |       # expect. It'd be nice to use flock here, but flock uses fork, not exec,
108 |       # so it makes it quite awkward to use in this case.
109 |       return 0
110 |     else
111 |       return 2 # program is dead but pid file exists
112 |     fi
113 |   else
114 |     return 3 # program is not running
115 |   fi
116 | }
117 | 
118 | force_stop() {
119 |   if status; then
120 |     stop
121 |     status && kill -KILL $(cat "$pidfile")
122 |     rm -f $pidfile
123 |   fi
124 | }
125 | 
126 | 
127 | case "$1" in
128 |   start)
129 |     status
130 |     code=$?
131 |     if [ $code -eq 0 ]; then
132 |       echo "$name is already running"
133 |     else
134 |       start
135 |       code=$?
136 |     fi
137 |     exit $code
138 |     ;;
139 | 
140 |   stop) stop ;;
141 | 
142 |   force-stop) force_stop ;;
143 | 
144 |   status)
145 |     status
146 |     code=$?
147 |     if [ $code -eq 0 ]; then
148 |       echo "$name is running"
149 |     else
150 |       echo "$name is not running"
151 |     fi
152 |     exit $code
153 |     ;;
154 | 
155 |   restart) stop && start ;;
156 | 
157 |   *)
158 |     echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|restart}" >&2
159 |     exit 3
160 |     ;;
161 | esac
162 | 
163 | exit $?
164 | 


--------------------------------------------------------------------------------
/logstash-logrotate:
--------------------------------------------------------------------------------
 1 | /var/log/logstash/*.err /var/log/logstash/*.log /var/log/logstash/*.stdout {
 2 |     daily
 3 |     rotate 7
 4 |     copytruncate
 5 |     compress
 6 |     delaycompress
 7 |     missingok
 8 |     notifempty
 9 | }
10 | 


--------------------------------------------------------------------------------
/nginx-filebeat/Dockerfile:
--------------------------------------------------------------------------------
 1 | # Dockerfile to illustrate how Filebeat can be used with nginx
 2 | # Filebeat OSS 8.14.3
 3 | 
 4 | # Build with:
 5 | # docker build -t filebeat-nginx-example .
 6 | 
 7 | # Run with:
 8 | # docker run -p 80:80 -it --link <elk-container-name>:elk \
 9 | #     --name filebeat-nginx-example filebeat-nginx-example
10 | 
11 | FROM nginx
12 | MAINTAINER Sebastien Pujadas http://pujadas.net
13 | ENV REFRESHED_AT 2020-10-02
14 | 
15 | 
16 | ###############################################################################
17 | #                                INSTALLATION
18 | ###############################################################################
19 | 
20 | ### install Filebeat
21 | 
22 | <<<<<<< HEAD
23 | ENV FILEBEAT_VERSION 8.17.6
24 | ENV FILEBEAT_BASE_VERSION 8.17.6
25 | =======
26 | ENV FILEBEAT_VERSION oss-8.14.3
27 | ENV FILEBEAT_BASE_VERSION 8.17.6
28 | >>>>>>> 7b5a7e1625b123c49d1a817cad15725b10f23144
29 | 
30 | 
31 | RUN apt-get update -qq \
32 |  && apt-get install -qqy curl \
33 |  && apt-get clean
34 | 
35 | RUN curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-amd64.deb \
36 |  && dpkg -i filebeat-${FILEBEAT_VERSION}-amd64.deb \
37 |  && rm filebeat-${FILEBEAT_VERSION}-amd64.deb
38 | 
39 | 
40 | ###############################################################################
41 | #                                CONFIGURATION
42 | ###############################################################################
43 | 
44 | ### tweak nginx image set-up
45 | 
46 | # remove log symlinks
47 | RUN rm /var/log/nginx/access.log /var/log/nginx/error.log
48 | 
49 | 
50 | ### configure Filebeat
51 | 
52 | # config file
53 | ADD filebeat.yml /etc/filebeat/filebeat.yml
54 | RUN chmod 644 /etc/filebeat/filebeat.yml
55 | 
56 | # CA cert
57 | RUN mkdir -p /etc/pki/tls/certs
58 | ADD logstash-beats.crt /etc/pki/tls/certs/logstash-beats.crt
59 | 
60 | # create template based on filebeat version (assumption: it is the same version as elasticsearch version)
61 | RUN filebeat export template --es.version ${FILEBEAT_BASE_VERSION} > /etc/filebeat/filebeat.template.json
62 | 
63 | ###############################################################################
64 | #                                    DATA
65 | ###############################################################################
66 | 
67 | ### add dummy HTML file
68 | 
69 | COPY html /usr/share/nginx/html
70 | 
71 | 
72 | ###############################################################################
73 | #                                    START
74 | ###############################################################################
75 | 
76 | ADD ./start.sh /usr/local/bin/start.sh
77 | RUN chmod +x /usr/local/bin/start.sh
78 | CMD [ "/usr/local/bin/start.sh" ]
79 | 


--------------------------------------------------------------------------------
/nginx-filebeat/filebeat.yml:
--------------------------------------------------------------------------------
 1 | output:
 2 |   logstash:
 3 |     enabled: true
 4 |     hosts:
 5 |       - elk:5044
 6 |     timeout: 15
 7 |     ssl:
 8 |       certificate_authorities:
 9 |       - /etc/pki/tls/certs/logstash-beats.crt
10 | 
11 | filebeat:
12 |   inputs:
13 |     -
14 |       paths:
15 |         - /var/log/syslog
16 |         - /var/log/auth.log
17 |       document_type: syslog
18 |     -
19 |       paths:
20 |         - "/var/log/nginx/*.log"
21 |       fields_under_root: true
22 |       fields:
23 |         type: nginx-access
24 | 


--------------------------------------------------------------------------------
/nginx-filebeat/html/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/spujadas/elk-docker/55770d26c63a38b322aef360c10615c5bb1f4ade/nginx-filebeat/html/favicon.ico


--------------------------------------------------------------------------------
/nginx-filebeat/html/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | 	<head>
 4 | 		<meta charset="utf-8">
 5 | 		<meta name="viewport" content="width=device-width,initial-scale=1">
 6 | 		<title>Filebeats + nginx example</title>
 7 | 	</head>
 8 | 	<body>
 9 | 		<h1>Hello world!</h1>
10 | 	</body>
11 | </html>


--------------------------------------------------------------------------------
/nginx-filebeat/logstash-beats.crt:
--------------------------------------------------------------------------------
 1 | -----BEGIN CERTIFICATE-----
 2 | MIIC6zCCAdOgAwIBAgIJANPZwuf+5wTLMA0GCSqGSIb3DQEBCwUAMAwxCjAIBgNV
 3 | BAMMASowHhcNMTUxMjI4MTA0NTMyWhcNMjUxMjI1MTA0NTMyWjAMMQowCAYDVQQD
 4 | DAEqMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp+jHFvhyYKiPXc7k
 5 | 0c33f2QV+1hHNyW/uwcJbp5jG82cuQ41v70Z1+b2veBW4sUlDY3yAIEOPSUD8ASt
 6 | 9m72CAo4xlwYKDvm/Sa3KJtDk0NrQiz6PPyBUFsY+Bj3xn6Nz1RW5YaP+Q1Hjnks
 7 | PEyQu4vLgfTSGYBHLD4gvs8wDWY7aaKf8DfuP7Ov74Qlj2GOxnmiDEF4tirlko0r
 8 | qQcvBgujCqA7rNoG+QDmkn3VrxtX8mKF72bxQ7USCyoxD4cWV2mU2HD2Maed3KHj
 9 | KAvDAzSyBMjI+qi9IlPN5MR7rVqUV0VlSKXBVPct6NG7x4WRwnoKjTXnr3CRADD0
10 | 4uvbQQIDAQABo1AwTjAdBgNVHQ4EFgQUVFurgDwdcgnCYxszc0dWMWhB3DswHwYD
11 | VR0jBBgwFoAUVFurgDwdcgnCYxszc0dWMWhB3DswDAYDVR0TBAUwAwEB/zANBgkq
12 | hkiG9w0BAQsFAAOCAQEAaLSytepMb5LXzOPr9OiuZjTk21a2C84k96f4uqGqKV/s
13 | okTTKD0NdeY/IUIINMq4/ERiqn6YDgPgHIYvQheWqnJ8ir69ODcYCpsMXIPau1ow
14 | T8c108BEHqBMEjkOQ5LrEjyvLa/29qJ5JsSSiULHvS917nVgY6xhcnRZ0AhuJkiI
15 | ARKXwpO5tqJi6BtgzX/3VDSOgVZbvX1uX51Fe9gWwPDgipnYaE/t9TGzJEhKwSah
16 | kNr+7RM+Glsv9rx1KcWcx4xxY3basG3/KwvsGAFPvk5tXbZ780VuNFTTZw7q3p8O
17 | Gk1zQUBOie0naS0afype5qFMPp586SF/2xAeb68gLg==
18 | -----END CERTIFICATE-----
19 | 


--------------------------------------------------------------------------------
/nginx-filebeat/start.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | curl -XPUT -H "Content-Type: application/json" 'http://elk:9200/_template/filebeat?pretty' -d@/etc/filebeat/filebeat.template.json
4 | /etc/init.d/filebeat start
5 | nginx
6 | tail -f /var/log/nginx/access.log -f /var/log/nginx/error.log
7 | 


--------------------------------------------------------------------------------
/nginx.pattern:
--------------------------------------------------------------------------------
1 | NGUSERNAME [a-zA-Z\.\@\-\+_%]+
2 | NGUSER %{NGUSERNAME}
3 | NGINXACCESS %{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:%{NUMBER:bytes:int}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent}
4 | 


--------------------------------------------------------------------------------
/pipelines.yml:
--------------------------------------------------------------------------------
1 | # This file is where you define your pipelines. You can define multiple.
2 | # For more information on multiple pipelines, see the documentation:
3 | #   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
4 | 
5 | - pipeline.id: main
6 |   path.config: "/etc/logstash/conf.d/*.conf"
7 | 


--------------------------------------------------------------------------------
/start.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | #
  3 | # /usr/local/bin/start.sh
  4 | # Start Elasticsearch, Logstash and Kibana services
  5 | 
  6 | # WARNING - This script assumes that the ELK services are not running, and is
  7 | #   only expected to be run once, when the container is started.
  8 | #   Do not attempt to run this script if the ELK services are running (or be
  9 | #   prepared to reap zombie processes).
 10 | 
 11 | 
 12 | ## handle termination gracefully
 13 | 
 14 | _term() {
 15 |   echo "Terminating ELK"
 16 | 
 17 |   # shut down services
 18 |   timeout 60 service elasticsearch stop &
 19 |   timeout 60 service logstash stop &
 20 |   timeout 60 service kibana stop &
 21 |   trap - SIGTERM SIGINT
 22 | 
 23 |   # wait for all services to stop
 24 |   wait
 25 | 
 26 |   # kill script PGID so all the child processes are killed, to avoid zombies
 27 |   kill -TERM -- -$$ 2>/dev/null
 28 |   exit 0
 29 | }
 30 | 
 31 | trap _term SIGTERM SIGINT
 32 | 
 33 | 
 34 | ## remove pidfiles in case previous graceful termination failed
 35 | # NOTE - This is the reason for the WARNING at the top - it's a bit hackish,
 36 | #   but if it's good enough for Fedora (https://goo.gl/88eyXJ), it's good
 37 | #   enough for me :)
 38 | 
 39 | rm -f /var/run/elasticsearch/elasticsearch.pid /var/run/logstash.pid \
 40 |   /var/run/kibana5.pid
 41 | 
 42 | ## initialise list of log files to stream in console (initially empty)
 43 | OUTPUT_LOGFILES=""
 44 | 
 45 | 
 46 | ## override default time zone (Etc/UTC) if TZ variable is set
 47 | if [ ! -z "$TZ" ]; then
 48 |   ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 49 | fi
 50 | 
 51 | 
 52 | ## run pre-hooks
 53 | if [ -x /usr/local/bin/elk-pre-hooks.sh ]; then
 54 |   . /usr/local/bin/elk-pre-hooks.sh
 55 | fi
 56 | 
 57 | 
 58 | ## start services as needed
 59 | 
 60 | ### crond
 61 | 
 62 | service cron start
 63 | 
 64 | 
 65 | ### Elasticsearch
 66 | 
 67 | if [ -z "$ELASTICSEARCH_START" ]; then
 68 |   ELASTICSEARCH_START=1
 69 | fi
 70 | if [ "$ELASTICSEARCH_START" -ne "1" ]; then
 71 |   echo "ELASTICSEARCH_START is set to something different from 1, not starting..."
 72 | else
 73 |   # update permissions of ES data directory
 74 |   chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
 75 | 
 76 |   # override JVM heap size using custom JVM options file if ES_HEAP_SIZE variable is set
 77 |   if [ ! -z "$ES_HEAP_SIZE" ]; then
 78 |     cat << EOF > ${ES_PATH_CONF}/jvm.options.d/heap_size.options
 79 | -Xmx$ES_HEAP_SIZE
 80 | -Xms$ES_HEAP_SIZE
 81 | EOF
 82 |   fi
 83 | 
 84 |   # disable JVM HeapDumpOnOutOfMemoryError if ES_HEAP_DISABLE variable is set
 85 |   if [ ! -z "$ES_HEAP_DISABLE" ]; then
 86 |     echo "-XX:-HeapDumpOnOutOfMemoryError" > ${ES_PATH_CONF}/jvm.options.d/heap_dump.options
 87 |   fi
 88 | 
 89 |   # override ES_JAVA_OPTS variable if set
 90 |   if [ ! -z "$ES_JAVA_OPTS" ]; then
 91 |     awk -v LINE="ES_JAVA_OPTS=\"$ES_JAVA_OPTS\"" '{ sub(/^#?ES_JAVA_OPTS=.*/, LINE); print; }' /etc/default/elasticsearch \
 92 |         > /etc/default/elasticsearch.new && mv /etc/default/elasticsearch.new /etc/default/elasticsearch
 93 |   fi
 94 | 
 95 |   # override MAX_OPEN_FILES variable if set
 96 |   if [ ! -z "$MAX_OPEN_FILES" ]; then
 97 |     awk -v LINE="MAX_OPEN_FILES=$MAX_OPEN_FILES" '{ sub(/^#?MAX_OPEN_FILES=.*/, LINE); print; }' /etc/init.d/elasticsearch \
 98 |         > /etc/init.d/elasticsearch.new && mv /etc/init.d/elasticsearch.new /etc/init.d/elasticsearch \
 99 |         && chmod +x /etc/init.d/elasticsearch
100 |   fi
101 | 
102 |   # override MAX_MAP_COUNT variable if set
103 |   if [ ! -z "$MAX_MAP_COUNT" ]; then
104 |     awk -v LINE="MAX_MAP_COUNT=$MAX_MAP_COUNT" '{ sub(/^#?MAX_MAP_COUNT=.*/, LINE); print; }' /etc/init.d/elasticsearch \
105 |         > /etc/init.d/elasticsearch.new && mv /etc/init.d/elasticsearch.new /etc/init.d/elasticsearch \
106 |         && chmod +x /etc/init.d/elasticsearch
107 |   fi
108 | 
109 |   service elasticsearch start
110 | 
111 |   # wait for Elasticsearch to start up before either starting Kibana (if enabled)
112 |   # or attempting to stream its log file
113 |   # - https://github.com/elasticsearch/kibana/issues/3077
114 | 
115 |   # set number of retries (default: 30, override using ES_CONNECT_RETRY env var)
116 |   re_is_numeric='^[0-9]+$'
117 |   if ! [[ $ES_CONNECT_RETRY =~ $re_is_numeric ]] ; then
118 |      ES_CONNECT_RETRY=30
119 |   fi
120 | 
121 |   if [ -z "$ELASTICSEARCH_URL" ]; then
122 |     ELASTICSEARCH_URL=${ES_PROTOCOL:-http}://localhost:9200
123 |   fi
124 | 
125 |   counter=0
126 |   while [ ! "$(curl -k ${ELASTICSEARCH_URL} 2> /dev/null)" -a $counter -lt $ES_CONNECT_RETRY  ]; do
127 |     sleep 1
128 |     ((counter++))
129 |     echo "waiting for Elasticsearch to be up ($counter/$ES_CONNECT_RETRY)"
130 |   done
131 |   if [ ! "$(curl -k ${ELASTICSEARCH_URL} 2> /dev/null)" ]; then
132 |     echo "Couldn't start Elasticsearch. Exiting."
133 |     echo "Elasticsearch log follows below."
134 |     cat /var/log/elasticsearch/elasticsearch.log
135 |     exit 1
136 |   fi
137 | 
138 |   # wait for cluster to respond before getting its name
139 |   counter=0
140 |   while [ -z "$CLUSTER_NAME" -a $counter -lt 30 ]; do
141 |     sleep 1
142 |     ((counter++))
143 |     CLUSTER_NAME=$(curl -k ${ELASTICSEARCH_URL}/_cat/health?h=cluster 2> /dev/null | tr -d '[:space:]')
144 |     echo "Waiting for Elasticsearch cluster to respond ($counter/30)"
145 |   done
146 | 
147 |   if [ -z "$CLUSTER_NAME" ]; then
148 |     echo "Couldn't get name of cluster. Exiting."
149 |     echo "Elasticsearch log follows."
150 |     cat /var/log/elasticsearch/elasticsearch.log
151 |     exit 1
152 |   elif [[ "$CLUSTER_NAME" =~ "master_not_discovered_exception" ]]; then
153 |     # If we got a JSON error back, don't treat it like the literal name of the cluster.
154 |     # Example of what this error looks like:
155 |     # [{"error":{"root_cause":[{"type":"master_not_discovered_exception","reason":null}]
156 |     # We don't know the cluster name, so we'll just glob it.
157 |     echo "Failed to contact a healthy master in cluster."
158 |     echo "Elasticsearch logs follow."
159 |     cat /var/log/elasticsearch/*.log
160 |     exit 1
161 |   fi
162 |   OUTPUT_LOGFILES+="/var/log/elasticsearch/${CLUSTER_NAME}.log "
163 | fi
164 | 
165 | 
166 | ### Logstash
167 | 
168 | if [ -z "$LOGSTASH_START" ]; then
169 |   LOGSTASH_START=1
170 | fi
171 | if [ "$LOGSTASH_START" -ne "1" ]; then
172 |   echo "LOGSTASH_START is set to something different from 1, not starting..."
173 | else
174 |   # override LS_HEAP_SIZE variable if set
175 |   if [ ! -z "$LS_HEAP_SIZE" ]; then
176 |     awk -v LINE="-Xmx$LS_HEAP_SIZE" '{ sub(/^.Xmx.*/, LINE); print; }' ${LOGSTASH_PATH_SETTINGS}/jvm.options \
177 |         > ${LOGSTASH_PATH_SETTINGS}/jvm.options.new && mv ${LOGSTASH_PATH_SETTINGS}/jvm.options.new ${LOGSTASH_PATH_SETTINGS}/jvm.options
178 |     awk -v LINE="-Xms$LS_HEAP_SIZE" '{ sub(/^.Xms.*/, LINE); print; }' ${LOGSTASH_PATH_SETTINGS}/jvm.options \
179 |         > ${LOGSTASH_PATH_SETTINGS}/jvm.options.new && mv ${LOGSTASH_PATH_SETTINGS}/jvm.options.new ${LOGSTASH_PATH_SETTINGS}/jvm.options
180 |   fi
181 | 
182 |   if [ ! -z "$LS_HEAP_DISABLE" ]; then
183 |     awk -v LINE="#-XX:+HeapDumpOnOutOfMemoryError" '{ sub(/^-XX:\+HeapDumpOnOutOfMemoryError.*/, LINE); print; }' ${LOGSTASH_PATH_SETTINGS}/jvm.options \
184 |         > ${LOGSTASH_PATH_SETTINGS}/jvm.options.new && mv ${LOGSTASH_PATH_SETTINGS}/jvm.options.new ${LOGSTASH_PATH_SETTINGS}/jvm.options
185 |   fi
186 | 
187 |   # override LS_OPTS variable if set
188 |   if [ ! -z "$LS_OPTS" ]; then
189 |     awk -v LINE="LS_OPTS=\"$LS_OPTS\"" '{ sub(/^LS_OPTS=.*/, LINE); print; }' /etc/init.d/logstash \
190 |         > /etc/init.d/logstash.new && mv /etc/init.d/logstash.new /etc/init.d/logstash && chmod +x /etc/init.d/logstash
191 |   fi
192 | 
193 |   service logstash start
194 |   OUTPUT_LOGFILES+="/var/log/logstash/logstash-plain.log "
195 | fi
196 | 
197 | 
198 | ### Kibana
199 | 
200 | if [ -z "$KIBANA_START" ]; then
201 |   KIBANA_START=1
202 | fi
203 | if [ "$KIBANA_START" -ne "1" ]; then
204 |   echo "KIBANA_START is set to something different from 1, not starting..."
205 | else
206 |   # override NODE_OPTIONS variable if set
207 |   if [ ! -z "$NODE_OPTIONS" ]; then
208 |     awk -v LINE="NODE_OPTIONS=\"$NODE_OPTIONS\"" '{ sub(/^NODE_OPTIONS=.*/, LINE); print; }' /etc/init.d/kibana \
209 |         > /etc/init.d/kibana.new && mv /etc/init.d/kibana.new /etc/init.d/kibana && chmod +x /etc/init.d/kibana
210 |   fi
211 | 
212 |   service kibana start
213 |   OUTPUT_LOGFILES+="/var/log/kibana/kibana5.log "
214 | fi
215 | 
216 | # Exit if nothing has been started
217 | if [ "$ELASTICSEARCH_START" -ne "1" ] && [ "$LOGSTASH_START" -ne "1" ] \
218 |   && [ "$KIBANA_START" -ne "1" ]; then
219 |   >&2 echo "No services started. Exiting."
220 |   exit 1
221 | fi
222 | 
223 | 
224 | ## run post-hooks
225 | if [ -x /usr/local/bin/elk-post-hooks.sh ]; then
226 |   ### if Kibana was started...
227 |   if [ "$KIBANA_START" -eq "1" ]; then
228 | 
229 |   ### ... then wait for Kibana to be up first to ensure that .kibana index is
230 |   ### created before the post-hooks are executed
231 |     # set number of retries (default: 30, override using KIBANA_CONNECT_RETRY env var)
232 |     if ! [[ $KIBANA_CONNECT_RETRY =~ $re_is_numeric ]] ; then
233 |        KIBANA_CONNECT_RETRY=30
234 |     fi
235 | 
236 |     if [ -z "$KIBANA_URL" ]; then
237 |       KIBANA_URL=http://localhost:5601
238 |     fi
239 | 
240 |     counter=0
241 |     while [ ! "$(curl ${KIBANA_URL} 2> /dev/null)" -a $counter -lt $KIBANA_CONNECT_RETRY  ]; do
242 |       sleep 1
243 |       ((counter++))
244 |       echo "waiting for Kibana to be up ($counter/$KIBANA_CONNECT_RETRY)"
245 |     done
246 |     if [ ! "$(curl ${KIBANA_URL} 2> /dev/null)" ]; then
247 |       echo "Couldn't start Kibana. Exiting."
248 |       echo "Kibana log follows below."
249 |       cat /var/log/kibana/kibana5.log
250 |       exit 1
251 |     fi
252 |     # wait for Kibana to not only be up but to return 200 OK
253 |     counter=0
254 |     while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${KIBANA_URL}/api/status)" != "200" && $counter -lt 30 ]]; do
255 |       sleep 1
256 |       ((counter++))
257 |       echo "waiting for Kibana to respond ($counter/30)"
258 |     done
259 |     if [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${KIBANA_URL}/api/status)" != "200" ]]; then
260 |       echo "Timed out waiting for Kibana to respond. Exiting."
261 |       echo "Kibana log follows below."
262 |       cat /var/log/kibana/kibana5.log
263 |       exit 1
264 |     fi
265 |   fi
266 | 
267 |   . /usr/local/bin/elk-post-hooks.sh
268 | fi
269 | 
270 | 
271 | touch $OUTPUT_LOGFILES
272 | tail -f $OUTPUT_LOGFILES &
273 | wait
274 | 


--------------------------------------------------------------------------------