├── .gitignore ├── README.md ├── docker-compose.yml ├── filebeat ├── Dockerfile └── filebeat.yml └── src ├── Dockerfile ├── logging_setup.py ├── logs └── .gitkeep ├── main.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | __pycache__/ 3 | /src/logs/fastapi-elk-stack.log 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fastapi-elk-stack-logging 2 | 3 | Supporting repository for my blog post [here](https://firasesbai.github.io/articles/2022/01/09/logging-with-elasticsearch.html) 4 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | elasticsearch: 6 | image: "docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3" 7 | container_name: elasticsearch 8 | environment: 9 | #- "ES_JAVA_OPTS=-Xms1g -Xmx1g" 10 | - "discovery.type=single-node" 11 | ports: 12 | - "9200:9200" 13 | volumes: 14 | - elasticsearch_data:/usr/share/elasticsearch/data 15 | 16 | kibana: 17 | image: "docker.elastic.co/kibana/kibana-oss:7.9.3" 18 | container_name: kibana 19 | ports: 20 | - "5601:5601" 21 | environment: 22 | - ELASTICSEARCH_HOSTS:http://elasticsearch:9200 23 | depends_on: 24 | - elasticsearch 25 | 26 | filebeat: 27 | build: 28 | context: ./filebeat 29 | container_name: filebeat 30 | volumes: 31 | - ./fastapi-logs:/src/logs 32 | depends_on: 33 | - elasticsearch 34 | - kibana 35 | 36 | fastapi-app: 37 | build: 38 | context: ./src 39 | container_name: fastapi-app 40 | ports: 41 | - "8000:8000" 42 | volumes: 43 | - ./fastapi-logs:/src/logs 44 | 45 | volumes: 46 | elasticsearch_data: 47 | driver: local -------------------------------------------------------------------------------- /filebeat/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/beats/filebeat-oss:7.9.3 2 | 3 | COPY filebeat.yml /usr/share/filebeat/filebeat.yml 4 | USER root 5 | RUN chown -R root /usr/share/filebeat/ 6 | RUN chmod -R go-w /usr/share/filebeat/ 7 | -------------------------------------------------------------------------------- /filebeat/filebeat.yml: -------------------------------------------------------------------------------- 1 | name: "fastapi-app-filebeat" 2 | logging.metrics.enabled: false 3 | xpack.security.enabled: false 4 | xpack.monitoring.enabled: false 5 | setup.ilm.enabled: false 6 | setup.template.enabled: false 7 | 8 | filebeat.inputs: 9 | - type: log 10 | scan_frequency: 1s 11 | enabled: true 12 | paths: 13 | - /src/logs/fastapi-elk-stack.* 14 | fields: 15 | - service: fastapi-app 16 | fields_under_root: true 17 | 18 | output.elasticsearch: 19 | hosts: ["elasticsearch:9200"] 20 | index: "fastapi-app" -------------------------------------------------------------------------------- /src/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | 3 | # Copy requirements file 4 | COPY requirements.txt . 5 | 6 | # Install dependencies 7 | RUN pip install --no-cache-dir -r requirements.txt 8 | 9 | # Bundle application source 10 | RUN mkdir /src/ 11 | COPY . /src 12 | 13 | # set workdir 14 | WORKDIR /src 15 | 16 | # Start application 17 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] 18 | -------------------------------------------------------------------------------- /src/logging_setup.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | # Central log format for root logger 4 | LOG_FORMAT = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" 5 | 6 | 7 | def setup_root_logger(): 8 | """ Setup configuration of the root logger of the application """ 9 | 10 | # get instance of root logger 11 | logger = logging.getLogger('') 12 | 13 | # configure formatter for logger 14 | formatter = logging.Formatter(LOG_FORMAT) 15 | 16 | # configure console handler 17 | console = logging.StreamHandler() 18 | console.setFormatter(formatter) 19 | 20 | # configure rotating file handler 21 | file = logging.handlers.RotatingFileHandler(filename="logs/fastapi-elk-stack.log", mode='a', 22 | maxBytes=15000000, backupCount=5) 23 | file.setFormatter(formatter) 24 | 25 | # add handlers 26 | logger.addHandler(console) 27 | logger.addHandler(file) 28 | 29 | # configure logger level 30 | logger.setLevel(logging.INFO) 31 | -------------------------------------------------------------------------------- /src/logs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firasesbai/fastapi-elk-stack-logging/0f4b5e735de7af9c471e9eb34fafe0c4cb3a00d0/src/logs/.gitkeep -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import uvicorn 4 | import uuid 5 | 6 | from fastapi import FastAPI 7 | from logging_setup import setup_root_logger 8 | 9 | # setup root logger 10 | setup_root_logger() 11 | 12 | # Get logger for module 13 | LOGGER = logging.getLogger(__name__) 14 | 15 | 16 | LOGGER.info("---Starting App---") 17 | 18 | app = FastAPI() 19 | 20 | 21 | @app.get("/random_uuid") 22 | async def root(): 23 | LOGGER.info(str(uuid.uuid4())) 24 | return "OK" 25 | 26 | if __name__ == '__main__': 27 | uvicorn.run(app, host="0.0.0.0", port=8000, log_config=None) 28 | -------------------------------------------------------------------------------- /src/requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi==0.65.1 2 | uvicorn==0.13.4 --------------------------------------------------------------------------------