├── .env ├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── app ├── dockerfile ├── main.py └── requirements.txt ├── docker-compose.yml ├── filebeat.yml ├── filebeat_ingest_data └── Air_Quality.log ├── get-cert-details.sh ├── kibana.yml ├── logstash.conf ├── logstash_ingest_data └── Air_Quality.csv └── metricbeat.yml /.env: -------------------------------------------------------------------------------- 1 | # Project namespace (defaults to the current folder name if not set) 2 | COMPOSE_PROJECT_NAME=es 3 | 4 | # Password for the 'elastic' user (at least 6 characters) 5 | ELASTIC_PASSWORD=changeme 6 | 7 | # Password for the 'kibana_system' user (at least 6 characters) 8 | KIBANA_PASSWORD=changeme 9 | 10 | # Version of Elastic products 11 | #https://www.elastic.co/downloads/past-releases#elasticsearch 12 | STACK_VERSION=8.8.2 13 | 14 | # Set the cluster name 15 | CLUSTER_NAME=docker-cluster 16 | 17 | # Set to 'basic' or 'trial' to automatically start the 30-day trial 18 | LICENSE=basic 19 | #LICENSE=trial 20 | 21 | # Port to expose Elasticsearch HTTP API to the host 22 | ES_PORT=9200 23 | 24 | # Port to expose Kibana to the host 25 | KIBANA_PORT=5601 26 | 27 | # Port to expose Fleet to the host 28 | FLEET_PORT=8220 29 | 30 | # Port to expose APM to the host 31 | APMSERVER_PORT=8200 32 | 33 | # APM Secret Token for POC environments only 34 | ELASTIC_APM_SECRET_TOKEN=supersecrettoken 35 | 36 | # Increase or decrease based on the available host memory (in bytes) 37 | ES_MEM_LIMIT=3073741824 38 | KB_MEM_LIMIT=1073741824 39 | LS_MEM_LIMIT=1073741824 40 | 41 | # SAMPLE Predefined Key only to be used in POC environments 42 | ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 43 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | app/.venv 2 | app/__pycache__ 3 | troubleshooting.txt 4 | TODO.md 5 | elastic-stack-docker-part-two.wg 6 | .DS_Store 7 | .gitignore 8 | new-arch.excalidraw 9 | base-arch.excalidraw -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Getting started with the Elastic Stack and Docker Compose: Part 2 2 | ## Elastic Agent, Fleet, and Elastic APM 3 | 4 | This repo is in reference to the blog [Getting Started with the Elastic Stack and Docker Compose: Part 2](https://www.elastic.co/blog/getting-started-with-the-elastic-stack-and-docker-compose-part-2) 5 | 6 | You can read the first blog: [Getting Started with the Elastic Stack and Docker Compose](https://www.elastic.co/blog/getting-started-with-the-elastic-stack-and-docker-compose) or visit it's [GitHub repo](https://github.com/elkninja/elastic-stack-docker-part-one) 7 | 8 | Please feel free to ask any questions via issues [here](https://github.com/elkninja/elastic-stack-docker-part-two/issues), our [Community Slack](https://ela.st/slack), or over in our [Discuss Forums](https://discuss.elastic.co/). 9 | 10 | Pull Requests welcome :) 11 | 12 | 13 | ## Resources: 14 | ### Fleet/Agent 15 | 16 | Overview: https://www.elastic.co/guide/en/fleet/current/fleet-overview.html 17 | 18 | Policy Creation, No UI: https://www.elastic.co/guide/en/fleet/current/create-a-policy-no-ui.html 19 | 20 | Adding Fleet On-Prem: https://www.elastic.co/guide/en/fleet/current/add-fleet-server-on-prem.html 21 | 22 | Agent in a Container: https://www.elastic.co/guide/en/fleet/current/elastic-agent-container.html 23 | 24 | Air Gapped: https://www.elastic.co/guide/en/fleet/current/air-gapped.html 25 | 26 | Secure Fleet: https://www.elastic.co/guide/en/fleet/current/secure-connections.html 27 | 28 | ### APM: 29 | 30 | APM: 31 | https://www.elastic.co/guide/en/apm/guide/current/upgrade-to-apm-integration.html 32 | 33 | On Prem: https://www.elastic.co/guide/en/apm/guide/current/apm-integration-upgrade-steps.html 34 | 35 | Fleet-Managed: https://www.elastic.co/guide/en/fleet/8.8/install-fleet-managed-elastic-agent.html 36 | 37 | Queue Full Error: 38 | https://www.elastic.co/guide/en/apm/server/current/common-problems.html#queue-full 39 | -------------------------------------------------------------------------------- /app/dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM python:3.9-slim-buster 4 | 5 | WORKDIR /app 6 | 7 | COPY requirements.txt requirements.txt 8 | 9 | RUN pip3 install -r requirements.txt 10 | 11 | COPY main.py main.py 12 | 13 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--log-level", "info", "--workers", "1"] -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from elasticapm.contrib.starlette import ElasticAPM, make_apm_client 2 | from fastapi import FastAPI 3 | from nicegui import ui 4 | from typing import Callable 5 | import asyncio 6 | import functools 7 | import httpx as r 8 | #import psutil 9 | 10 | try: 11 | apm = make_apm_client({ 12 | 'SERVICE_NAME': 'my_python_service', 13 | 'SECRET_TOKEN': 'supersecrettoken', 14 | # SERVER_URL must be set to "fleet-server" if running as a docker container. 15 | # if running as a local python script, then set the url to "LOCALHOST" 16 | 'SERVER_URL': 'http://fleet-server:8200', 17 | 'ENVIRONMENT': 'development' 18 | }) 19 | except Exception as e: 20 | print('failed to create client') 21 | 22 | app = FastAPI() 23 | 24 | try: 25 | app.add_middleware(ElasticAPM, client=apm) 26 | except Exception as e: 27 | print('failed to add APM Middleware') 28 | 29 | 30 | @app.get("/custom_message/{message}") 31 | async def custom_message(message: str): 32 | apm.capture_message(f"Custom Message: {message}") 33 | return {"message": f"Custom Message: {message}"} 34 | 35 | 36 | @app.get("/error") 37 | async def throw_error(): 38 | try: 39 | 1 / 0 40 | except Exception as e: 41 | apm.capture_exception() 42 | return {"message": "Failed Successfully :)"} 43 | 44 | 45 | def init(fastapi_app: FastAPI) -> None: 46 | @ui.page('/', title="APM Demo App") 47 | async def show(): 48 | with ui.header(elevated=True).style('background-color: #3874c8').classes('items-center justify-between'): 49 | ui.markdown('### APM DEMO') 50 | ui.button(on_click=lambda: right_drawer.toggle(), icon='menu').props('flat color=white') 51 | with ui.right_drawer(fixed=False).style('background-color: #ebf1fa').props('bordered') as right_drawer: 52 | ui.chat_message('Hello Elastic Stack User!', 53 | name='APM Robot', 54 | stamp='now', 55 | avatar='https://robohash.org/apm_robot') 56 | ui.chat_message('This app is powered by NICEGUI and FastAPI with Elastic APM Instrumentation :)', 57 | name='APM Robot', 58 | stamp='now', 59 | avatar='https://robohash.org/apm_robot') 60 | ui.chat_message('Please click a button to trigger an APM event.', 61 | name='APM Robot', 62 | stamp='now', 63 | avatar='https://robohash.org/apm_robot') 64 | with ui.footer().style('background-color: #3874c8'): 65 | ui.label('APM DEMO PAGE') 66 | 67 | with ui.card(): 68 | ui.label('Generate Error - Python') 69 | ui.button('Generate', on_click=python_error) 70 | 71 | with ui.card(): 72 | ui.label('Generate Error - JS') 73 | ui.button('Generate', on_click=js_error) 74 | 75 | with ui.card(): 76 | ui.label('Generate Custom Message') 77 | custom_message_text = ui.input(placeholder='Message') 78 | ui.button('Generate').on('click', handler=lambda: gen_custom_message(custom_message_text.value)) 79 | 80 | ui.run_with( 81 | fastapi_app, 82 | storage_secret='supersecret', # NOTE setting a secret is optional but allows for persistent storage per user 83 | ) 84 | 85 | 86 | async def io_bound(callback: Callable, *args: any, **kwargs: any): 87 | '''Makes a blocking function awaitable; pass function as first parameter and its arguments as the rest''' 88 | return await asyncio.get_event_loop().run_in_executor(None, functools.partial(callback, *args, **kwargs)) 89 | 90 | 91 | async def python_error(): 92 | try: 93 | res = await io_bound(r.get, 'http://localhost:8000/error') 94 | ui.notify(res.text) 95 | except Exception as e: 96 | apm.capture_exception() 97 | ui.notify(f'{e}') 98 | 99 | 100 | async def js_error(): 101 | try: 102 | res = await ui.run_javascript('fetch("http://localhost:8000/error")') 103 | ui.notify(f'Message: Failed Successfully :)') 104 | except Exception as e: 105 | apm.capture_exception() 106 | ui.notify(f'{e}') 107 | 108 | 109 | async def gen_custom_message(text_message): 110 | try: 111 | res = await io_bound(r.get, 'http://localhost:8000/custom_message/' + str(text_message)) 112 | ui.notify(res.text) 113 | except Exception as e: 114 | apm.capture_exception() 115 | ui.notify(f'{e}') 116 | 117 | init(app) 118 | 119 | try: 120 | apm.capture_message('App Loaded, Hello World!') 121 | except Exception as e: 122 | print('error: ' + e) 123 | 124 | if __name__ == '__main__': 125 | print('Please start the app with the "uvicorn" command as shown in the start.sh script') 126 | -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | elastic-apm==6.17.0 2 | fastapi==0.100.0 3 | httpx==0.24.1 4 | nicegui==1.3.2 5 | starlette==0.27.0 -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | volumes: 3 | certs: 4 | driver: local 5 | esdata01: 6 | driver: local 7 | kibanadata: 8 | driver: local 9 | metricbeatdata01: 10 | driver: local 11 | filebeatdata01: 12 | driver: local 13 | logstashdata01: 14 | driver: local 15 | fleetserverdata: 16 | driver: local 17 | 18 | networks: 19 | default: 20 | name: elastic 21 | external: false 22 | 23 | services: 24 | setup: 25 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 26 | volumes: 27 | - certs:/usr/share/elasticsearch/config/certs 28 | user: "0" 29 | command: > 30 | bash -c ' 31 | if [ x${ELASTIC_PASSWORD} == x ]; then 32 | echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; 33 | exit 1; 34 | elif [ x${KIBANA_PASSWORD} == x ]; then 35 | echo "Set the KIBANA_PASSWORD environment variable in the .env file"; 36 | exit 1; 37 | fi; 38 | if [ ! -f config/certs/ca.zip ]; then 39 | echo "Creating CA"; 40 | bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; 41 | unzip config/certs/ca.zip -d config/certs; 42 | fi; 43 | if [ ! -f config/certs/certs.zip ]; then 44 | echo "Creating certs"; 45 | echo -ne \ 46 | "instances:\n"\ 47 | " - name: es01\n"\ 48 | " dns:\n"\ 49 | " - es01\n"\ 50 | " - localhost\n"\ 51 | " ip:\n"\ 52 | " - 127.0.0.1\n"\ 53 | " - name: kibana\n"\ 54 | " dns:\n"\ 55 | " - kibana\n"\ 56 | " - localhost\n"\ 57 | " ip:\n"\ 58 | " - 127.0.0.1\n"\ 59 | " - name: fleet-server\n"\ 60 | " dns:\n"\ 61 | " - fleet-server\n"\ 62 | " - localhost\n"\ 63 | " ip:\n"\ 64 | " - 127.0.0.1\n"\ 65 | > config/certs/instances.yml; 66 | bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; 67 | unzip config/certs/certs.zip -d config/certs; 68 | fi; 69 | echo "Setting file permissions" 70 | chown -R root:root config/certs; 71 | find . -type d -exec chmod 750 \{\} \;; 72 | find . -type f -exec chmod 640 \{\} \;; 73 | echo "Waiting for Elasticsearch availability"; 74 | until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; 75 | echo "Setting kibana_system password"; 76 | until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; 77 | echo "All done!"; 78 | ' 79 | healthcheck: 80 | test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] 81 | interval: 1s 82 | timeout: 5s 83 | retries: 120 84 | 85 | es01: 86 | depends_on: 87 | setup: 88 | condition: service_healthy 89 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 90 | labels: 91 | co.elastic.logs/module: elasticsearch 92 | volumes: 93 | - certs:/usr/share/elasticsearch/config/certs 94 | - esdata01:/usr/share/elasticsearch/data 95 | ports: 96 | - ${ES_PORT}:9200 97 | environment: 98 | - node.name=es01 99 | - cluster.name=${CLUSTER_NAME} 100 | - discovery.type=single-node 101 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 102 | - bootstrap.memory_lock=true 103 | - xpack.security.enabled=true 104 | - xpack.security.http.ssl.enabled=true 105 | - xpack.security.http.ssl.key=certs/es01/es01.key 106 | - xpack.security.http.ssl.certificate=certs/es01/es01.crt 107 | - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt 108 | - xpack.security.transport.ssl.enabled=true 109 | - xpack.security.transport.ssl.key=certs/es01/es01.key 110 | - xpack.security.transport.ssl.certificate=certs/es01/es01.crt 111 | - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt 112 | - xpack.security.transport.ssl.verification_mode=certificate 113 | - xpack.license.self_generated.type=${LICENSE} 114 | mem_limit: ${ES_MEM_LIMIT} 115 | ulimits: 116 | memlock: 117 | soft: -1 118 | hard: -1 119 | healthcheck: 120 | test: 121 | [ 122 | "CMD-SHELL", 123 | "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", 124 | ] 125 | interval: 10s 126 | timeout: 10s 127 | retries: 120 128 | 129 | kibana: 130 | depends_on: 131 | es01: 132 | condition: service_healthy 133 | image: docker.elastic.co/kibana/kibana:${STACK_VERSION} 134 | labels: 135 | co.elastic.logs/module: kibana 136 | volumes: 137 | - certs:/usr/share/kibana/config/certs 138 | - kibanadata:/usr/share/kibana/data 139 | - ./kibana.yml:/usr/share/kibana/config/kibana.yml:ro 140 | ports: 141 | - ${KIBANA_PORT}:5601 142 | environment: 143 | - SERVERNAME=kibana 144 | - ELASTICSEARCH_HOSTS=https://es01:9200 145 | - ELASTICSEARCH_USERNAME=kibana_system 146 | - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} 147 | - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt 148 | - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY} 149 | - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY} 150 | - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY} 151 | - XPACK_REPORTING_KIBANASERVER_HOSTNAME=localhost 152 | - SERVER_SSL_ENABLED=true 153 | - SERVER_SSL_CERTIFICATE=config/certs/kibana/kibana.crt 154 | - SERVER_SSL_KEY=config/certs/kibana/kibana.key 155 | - SERVER_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt 156 | - ELASTIC_APM_SECRET_TOKEN=${ELASTIC_APM_SECRET_TOKEN} 157 | mem_limit: ${KB_MEM_LIMIT} 158 | healthcheck: 159 | test: 160 | [ 161 | "CMD-SHELL", 162 | "curl -I -s --cacert config/certs/ca/ca.crt https://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", 163 | ] 164 | interval: 10s 165 | timeout: 10s 166 | retries: 120 167 | 168 | metricbeat01: 169 | depends_on: 170 | es01: 171 | condition: service_healthy 172 | kibana: 173 | condition: service_healthy 174 | image: docker.elastic.co/beats/metricbeat:${STACK_VERSION} 175 | user: root 176 | volumes: 177 | - certs:/usr/share/metricbeat/certs 178 | - metricbeatdata01:/usr/share/metricbeat/data 179 | - "./metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro" 180 | - "/var/run/docker.sock:/var/run/docker.sock:ro" 181 | - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro" 182 | - "/proc:/hostfs/proc:ro" 183 | - "/:/hostfs:ro" 184 | environment: 185 | - ELASTIC_USER=elastic 186 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 187 | - ELASTIC_HOSTS=https://es01:9200 188 | - KIBANA_HOSTS=https://kibana:5601 189 | - LOGSTASH_HOSTS=http://logstash01:9600 190 | - CA_CERT=certs/ca/ca.crt 191 | - ES_CERT=certs/es01/es01.crt 192 | - ES_KEY=certs/es01/es01.key 193 | - KB_CERT=certs/kibana/kibana.crt 194 | - KB_KEY=certs/kibana/kibana.key 195 | command: 196 | -strict.perms=false 197 | 198 | filebeat01: 199 | depends_on: 200 | es01: 201 | condition: service_healthy 202 | image: docker.elastic.co/beats/filebeat:${STACK_VERSION} 203 | user: root 204 | volumes: 205 | - certs:/usr/share/filebeat/certs 206 | - filebeatdata01:/usr/share/filebeat/data 207 | - "./filebeat_ingest_data/:/usr/share/filebeat/ingest_data/" 208 | - "./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro" 209 | - "/var/lib/docker/containers:/var/lib/docker/containers:ro" 210 | - "/var/run/docker.sock:/var/run/docker.sock:ro" 211 | environment: 212 | - ELASTIC_USER=elastic 213 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 214 | - ELASTIC_HOSTS=https://es01:9200 215 | - KIBANA_HOSTS=https://kibana:5601 216 | - LOGSTASH_HOSTS=http://logstash01:9600 217 | - CA_CERT=certs/ca/ca.crt 218 | command: 219 | -strict.perms=false 220 | 221 | logstash01: 222 | depends_on: 223 | es01: 224 | condition: service_healthy 225 | kibana: 226 | condition: service_healthy 227 | image: docker.elastic.co/logstash/logstash:${STACK_VERSION} 228 | labels: 229 | co.elastic.logs/module: logstash 230 | user: root 231 | volumes: 232 | - certs:/usr/share/logstash/certs 233 | - logstashdata01:/usr/share/logstash/data 234 | - "./logstash_ingest_data/:/usr/share/logstash/ingest_data/" 235 | - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro" 236 | environment: 237 | - xpack.monitoring.enabled=false 238 | - ELASTIC_USER=elastic 239 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 240 | - ELASTIC_HOSTS=https://es01:9200 241 | 242 | fleet-server: 243 | depends_on: 244 | kibana: 245 | condition: service_healthy 246 | es01: 247 | condition: service_healthy 248 | image: docker.elastic.co/beats/elastic-agent:${STACK_VERSION} 249 | volumes: 250 | - certs:/certs 251 | - fleetserverdata:/usr/share/elastic-agent 252 | - "/var/lib/docker/containers:/var/lib/docker/containers:ro" 253 | - "/var/run/docker.sock:/var/run/docker.sock:ro" 254 | - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro" 255 | - "/proc:/hostfs/proc:ro" 256 | - "/:/hostfs:ro" 257 | ports: 258 | - ${FLEET_PORT}:8220 259 | - ${APMSERVER_PORT}:8200 260 | user: root 261 | environment: 262 | - SSL_CERTIFICATE_AUTHORITIES=/certs/ca/ca.crt 263 | - CERTIFICATE_AUTHORITIES=/certs/ca/ca.crt 264 | - FLEET_CA=/certs/ca/ca.crt 265 | - FLEET_ENROLL=1 266 | - FLEET_INSECURE=true 267 | - FLEET_SERVER_ELASTICSEARCH_CA=/certs/ca/ca.crt 268 | - FLEET_SERVER_ELASTICSEARCH_HOST=https://es01:9200 269 | - FLEET_SERVER_ELASTICSEARCH_INSECURE=true 270 | - FLEET_SERVER_ENABLE=1 271 | - FLEET_SERVER_CERT=/certs/fleet-server/fleet-server.crt 272 | - FLEET_SERVER_CERT_KEY=/certs/fleet-server/fleet-server.key 273 | - FLEET_SERVER_INSECURE_HTTP=true 274 | - FLEET_SERVER_POLICY_ID=fleet-server-policy 275 | - FLEET_URL=https://fleet-server:8220 276 | - KIBANA_FLEET_CA=/certs/ca/ca.crt 277 | - KIBANA_FLEET_SETUP=1 278 | - KIBANA_FLEET_USERNAME=elastic 279 | - KIBANA_FLEET_PASSWORD=${ELASTIC_PASSWORD} 280 | - KIBANA_HOST=https://kibana:5601 281 | 282 | webapp: 283 | build: 284 | context: app 285 | volumes: 286 | - "/var/lib/docker/containers:/var/lib/docker/containers:ro" 287 | - "/var/run/docker.sock:/var/run/docker.sock:ro" 288 | - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro" 289 | - "/proc:/hostfs/proc:ro" 290 | - "/:/hostfs:ro" 291 | ports: 292 | - 8000:8000 -------------------------------------------------------------------------------- /filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.inputs: 2 | - type: filestream 3 | id: default-filestream 4 | paths: 5 | - ingest_data/*.log 6 | 7 | filebeat.autodiscover: 8 | providers: 9 | - type: docker 10 | hints.enabled: true 11 | 12 | processors: 13 | - add_docker_metadata: ~ 14 | 15 | setup.kibana: 16 | host: ${KIBANA_HOSTS} 17 | username: ${ELASTIC_USER} 18 | password: ${ELASTIC_PASSWORD} 19 | 20 | output.elasticsearch: 21 | hosts: ${ELASTIC_HOSTS} 22 | username: ${ELASTIC_USER} 23 | password: ${ELASTIC_PASSWORD} 24 | ssl: 25 | enabled: true 26 | certificate_authorities: ${CA_CERT} 27 | -------------------------------------------------------------------------------- /get-cert-details.sh: -------------------------------------------------------------------------------- 1 | docker cp es-es01-1:/usr/share/elasticsearch/config/certs/ca/ca.crt /tmp/. 2 | cat /tmp/ca.crt 3 | echo 'ssl:'; 4 | echo ' certificate_authorities:'; 5 | echo ' - |'; 6 | cat /tmp/ca.crt | sed 's/^/ /' 7 | openssl x509 -fingerprint -sha256 -noout -in /tmp/ca.crt | awk -F"=" {' print $2 '} | sed s/://g 8 | -------------------------------------------------------------------------------- /kibana.yml: -------------------------------------------------------------------------------- 1 | elastic: 2 | apm: 3 | active: true 4 | serverUrl: "http://fleet-server:8200" 5 | secretToken: ${ELASTIC_APM_SECRET_TOKEN} 6 | server.host: "0.0.0.0" 7 | telemetry.enabled: "true" 8 | xpack.fleet.packages: 9 | - name: fleet_server 10 | version: latest 11 | - name: system 12 | version: latest 13 | - name: elastic_agent 14 | version: latest 15 | - name: apm 16 | version: latest 17 | xpack.fleet.agentPolicies: 18 | - name: Fleet-Server-Policy 19 | id: fleet-server-policy 20 | namespace: default 21 | monitoring_enabled: 22 | - logs 23 | - metrics 24 | package_policies: 25 | - name: fleet_server-1 26 | package: 27 | name: fleet_server 28 | - name: system-1 29 | package: 30 | name: system 31 | - name: elastic_agent-1 32 | package: 33 | name: elastic_agent 34 | - name: apm-1 35 | package: 36 | name: apm 37 | inputs: 38 | - type: apm 39 | enabled: true 40 | vars: 41 | - name: host 42 | value: 0.0.0.0:8200 43 | - name: secret_token 44 | value: ${ELASTIC_APM_SECRET_TOKEN} 45 | -------------------------------------------------------------------------------- /logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | file { 3 | #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html 4 | #default is TAIL which assumes more data will come into the file. 5 | #change to mode => "read" if the file is a compelte file. by default, the file will be removed once reading is complete -- backup your files if you need them. 6 | mode => "tail" 7 | path => "/usr/share/logstash/ingest_data/*" 8 | } 9 | } 10 | 11 | filter { 12 | } 13 | 14 | output { 15 | elasticsearch { 16 | index => "logstash-%{+YYYY.MM.dd}" 17 | hosts=> "${ELASTIC_HOSTS}" 18 | user=> "${ELASTIC_USER}" 19 | password=> "${ELASTIC_PASSWORD}" 20 | cacert=> "certs/ca/ca.crt" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /metricbeat.yml: -------------------------------------------------------------------------------- 1 | metricbeat.config.modules: 2 | path: ${path.config}/modules.d/*.yml 3 | reload.enabled: false 4 | 5 | metricbeat.modules: 6 | - module: elasticsearch 7 | xpack.enabled: true 8 | period: 10s 9 | hosts: ${ELASTIC_HOSTS} 10 | username: ${ELASTIC_USER} 11 | password: ${ELASTIC_PASSWORD} 12 | ssl: 13 | enabled: true 14 | certificate_authorities: ${CA_CERT} 15 | 16 | - module: logstash 17 | xpack.enabled: true 18 | period: 10s 19 | hosts: ${LOGSTASH_HOSTS} 20 | 21 | - module: kibana 22 | metricsets: 23 | - stats 24 | period: 10s 25 | hosts: ${KIBANA_HOSTS} 26 | username: ${ELASTIC_USER} 27 | password: ${ELASTIC_PASSWORD} 28 | xpack.enabled: true 29 | ssl: 30 | enabled: true 31 | certificate_authorities: ${CA_CERT} 32 | 33 | - module: docker 34 | metricsets: 35 | - "container" 36 | - "cpu" 37 | - "diskio" 38 | - "healthcheck" 39 | - "info" 40 | #- "image" 41 | - "memory" 42 | - "network" 43 | hosts: ["unix:///var/run/docker.sock"] 44 | period: 10s 45 | enabled: true 46 | 47 | processors: 48 | - add_host_metadata: ~ 49 | - add_docker_metadata: ~ 50 | 51 | output.elasticsearch: 52 | hosts: ${ELASTIC_HOSTS} 53 | username: ${ELASTIC_USER} 54 | password: ${ELASTIC_PASSWORD} 55 | ssl: 56 | enabled: true 57 | certificate_authorities: ${CA_CERT} 58 | 59 | --------------------------------------------------------------------------------