├── .github └── PULL_REQUEST_TEMPLATE.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md └── examples ├── fluent-bit ├── add-keys │ ├── README.md │ ├── extra.conf │ ├── permissions.json │ └── task-definition.json ├── adot-resource-monitoring │ ├── README.md │ ├── permissions.json │ └── task-definition.json ├── amazon-managed-service-for-prometheus │ ├── README.md │ ├── config-files │ │ └── fluent-bit.conf │ ├── permissions.json │ └── task-definition.json ├── amazon-opensearch-serverless │ ├── README.md │ ├── aoss-data-access-policy.json │ └── task-definition.json ├── amazon-opensearch │ ├── README.md │ ├── permissions.json │ └── task-definition.json ├── cloudwatchlogs-emf │ ├── README.md │ ├── emf-over-tcp │ │ ├── extra.conf │ │ ├── task-definition-bridge.json │ │ └── task-definition-localhost.json │ ├── permissions.json │ └── task-definition-stdout-emf.json ├── cloudwatchlogs │ ├── README.md │ ├── permissions.json │ ├── task-definition-cloudwatch.json │ └── task-definition-cloudwatch_logs.json ├── config-file-type-file │ ├── Dockerfile │ ├── README.md │ ├── extra.conf │ ├── permissions.json │ ├── runtask_firelens.json │ └── task-definition.json ├── cross-account │ ├── README.md │ ├── cloudwatch-write.json │ └── task-definition.json ├── datadog │ ├── README.md │ └── task-definition.json ├── dynatrace │ ├── README.md │ └── task-definition.json ├── ecs-log-collection │ ├── README.md │ ├── permissions.json │ ├── task-definition-tail.json │ └── task-definition-tcp.json ├── ecs-log-deletion │ ├── Dockerfile │ ├── README.md │ ├── cron │ │ └── delete-log-files │ ├── permissions.json │ └── take-definition-tail.json ├── efs │ ├── README.md │ ├── extra.conf │ └── task-definition.json ├── elastic-cloud │ ├── README.md │ └── task-definition.json ├── enable-debug-logging │ ├── README.md │ ├── extra.conf │ ├── permissions.json │ └── task-definition.json ├── filter-multiline-partial-message-mode │ ├── README.md │ ├── custom-fluent-bit-with-config │ │ ├── Dockerfile │ │ └── extra.conf │ ├── multiline-app │ │ ├── Dockerfile │ │ ├── houndofbaskerville.txt │ │ └── main.py │ ├── permissions.json │ └── task-definition.json ├── filter-multiline │ ├── README.md │ ├── custom-fluent-bit-with-parser │ │ ├── Dockerfile │ │ ├── extra.conf │ │ └── parsers_multiline.conf │ ├── multiline-app │ │ ├── Dockerfile │ │ ├── main.py │ │ └── test.log │ ├── permissions.json │ └── task-definition.json ├── forward-to-aggregator │ ├── README.md │ └── task-definition.json ├── grafana-cloud │ ├── README.md │ └── task-definition.json ├── health-check │ ├── README.md │ ├── monitoring-server-healthcheck.conf │ ├── task-definition-output-metrics-healthcheck.json │ ├── task-definition-tcp-healthcheck.json │ └── task-definition-uptime-healthcheck.json ├── init-metadata │ ├── README.md │ ├── permissions.json │ └── task-definition.json ├── kinesis-firehose │ ├── README.md │ ├── permissions.json │ └── task-definition.json ├── kinesis-stream │ ├── README.md │ ├── permissions.json │ └── task-definition.json ├── log-driver-buffer-limit │ ├── README.md │ ├── permissions.json │ └── task-definition.json ├── logstash │ ├── README.md │ ├── sample_logstash_conf │ └── task-definition.json ├── multi-config-support │ ├── README.md │ ├── config-files │ │ ├── s3-output.conf │ │ ├── tail-input.conf │ │ ├── your-filter.conf │ │ └── your-parser.conf │ ├── permissions.json │ └── task-definition.json ├── newrelic │ ├── README.md │ └── example-task-definition.json ├── oomkill-prevention │ ├── README.md │ ├── filesystem.conf │ ├── firelens-full-filesystem-example │ │ ├── fluent-bit-image │ │ │ ├── Dockerfile │ │ │ └── extra.conf │ │ └── task-definition.json │ ├── firelens-full-memory-example │ │ ├── fluent-bit-image │ │ │ ├── Dockerfile │ │ │ └── extra.conf │ │ └── task-definition.json │ └── memory.conf ├── parse-common-log-formats │ ├── README.md │ ├── parse-apache.conf │ ├── permissions.json │ └── task-definition.json ├── parse-envoy-app-mesh │ ├── README.md │ ├── appmesh-firelens-colorteller-black-ecs-task-def.json │ ├── aws-for-fluent-bit-custom-envoy │ │ ├── Dockerfile │ │ ├── conf │ │ │ └── parse_envoy.conf │ │ └── parsers │ │ │ └── envoy_parser.conf │ └── static │ │ └── images │ │ └── virtual-node-logging.png ├── parse-json │ ├── README.md │ ├── extra.conf │ ├── permissions.json │ └── task-definition.json ├── s3 │ ├── README.md │ ├── permissions.json │ └── task-definition.json ├── sematext │ ├── README.md │ └── task-definition.json ├── send-fb-internal-metrics-to-cw │ ├── Dockerfile │ ├── README.md │ ├── extra.conf │ ├── fb_metrics_parser.conf │ ├── runtask_firelen.json │ ├── screenshot.png │ └── task_definition.json ├── send-to-multiple-destinations │ ├── README.md │ ├── extra.conf │ ├── permissions.json │ └── task-definition.json ├── signalfx │ ├── README.md │ └── task-definition.json ├── solarwinds-loggly │ ├── README.md │ └── task-definition.json └── sumologic │ ├── README.md │ └── task-definition.json ├── fluentd └── multiline-logs │ ├── Dockerfile │ ├── README.md │ ├── extra.conf │ └── task-definition.json └── splitting-log-streams ├── app ├── Dockerfile ├── README.md ├── app.go └── go.mod ├── fluent-bit ├── README.md ├── custom-fluent-bit-image │ ├── Dockerfile │ ├── fluent-bit.conf │ ├── parser.conf │ └── stream_processing.conf └── task_definition.json └── fluentd ├── README.md ├── rewrite-tag.conf └── task_definition.json /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. 7 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/aws-samples/amazon-ecs-firelens-examples/issues), or [recently closed](https://github.com/aws-samples/amazon-ecs-firelens-examples/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws-samples/amazon-ecs-firelens-examples/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/aws-samples/amazon-ecs-firelens-examples/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | -------------------------------------------------------------------------------- /examples/fluent-bit/add-keys/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Adding Keys to the Log Events 2 | 3 | With the custom configuration file in this example, you can add a key to each log message. This is similar to how FireLens adds ECS Metadata to your logs. 4 | 5 | In this example, we add a field called `app-version`- this will allow us to correlate log messages with the version of our app that generated them. The App Version is set via an environment variable, which can be referenced in the Fluent Bit configuration file. 6 | 7 | Assuming ECS Log Metadata is enabled, the final log events in Firehose will look something like the following: 8 | ``` 9 | { 10 | "source": "stdout", 11 | "app-version": "v1.1.14", 12 | "log": "172.17.0.1 - - [03/Oct/2019:00:06:20 +0000] \"GET / HTTP/1.1\" 200 612 \"-\" \"curl/7.54.0\" \"-\"", 13 | "container_id": "e54cccfac2b87417f71877907f67879068420042828067ae0867e60a63529d35", 14 | "container_name": "/ecs-demo-6-container2-a4eafbb3d4c7f1e16e00" 15 | "ecs_cluster": "mycluster", 16 | "ecs_task_arn": "arn:aws:ecs:us-east-2:01234567891011:task/mycluster/3de392df-6bfa-470b-97ed-aa6f482cd7a6", 17 | "ecs_task_definition": "demo:7", 18 | "ec2_instance_id": "i-06bc83dbc2ac2fdf8" 19 | } 20 | ``` 21 | 22 | Keys can be added and removed via the record_modifier filter- for more information see the [Fluent Bit documentation](https://fluentbit.io/documentation/0.12/filter/record_modifier.html). 23 | -------------------------------------------------------------------------------- /examples/fluent-bit/add-keys/extra.conf: -------------------------------------------------------------------------------- 1 | [FILTER] 2 | Name record_modifier 3 | Match * 4 | Record app-version ${APP_VERSION} 5 | -------------------------------------------------------------------------------- /examples/fluent-bit/add-keys/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "firehose:PutRecordBatch" 7 | ], 8 | "Resource": "*" 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/add-keys/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-firehose", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "s3", 14 | "config-file-value": "arn:aws:s3:::yourbucket/yourdirectory/extra.conf" 15 | } 16 | }, 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-west-2", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "environment": [ 27 | { 28 | "name": "APP_VERSION", 29 | "value": "v1.1.14" 30 | } 31 | ], 32 | "memoryReservation": 50 33 | }, 34 | { 35 | "essential": true, 36 | "image": "httpd", 37 | "name": "app", 38 | "logConfiguration": { 39 | "logDriver":"awsfirelens", 40 | "options": { 41 | "Name": "kinesis_firehose", 42 | "region": "us-west-2", 43 | "delivery_stream": "my-stream", 44 | "retry_limit": "2" 45 | } 46 | }, 47 | "memoryReservation": 100 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /examples/fluent-bit/adot-resource-monitoring/README.md: -------------------------------------------------------------------------------- 1 | # FireLens Example: Quickly Enable Resource Monitoring with ADOT 2 | 3 | The [AWS Distro for OpenTelemetry](https://aws-otel.github.io/) has many features for collecting telemetry from your applications. In this tutorial, you will learn how to quickly enable resource utilization monitoring for Fluent Bit using ADOT. OpenTelemetry can do a lot more than is shown in this tutorial. 4 | 5 | This guide is broken down into two tutorials: 6 | - [Tutorial 1: Quickly Enable Task Resource Monitoring](#tutorial-1-quickly-enable-task-resource-monitoring) 7 | - [Tutorial 2: Enable Container Level Metric Collection](#tutorial-2-enable-container-level-metric-collection) 8 | 9 | ### Tutorial 1: Quickly Enable Task Resource Monitoring 10 | 11 | To quickly enable CPU, memory, network and disk usage metrics for your FireLens task, simply add the following container definition to your current task definition: 12 | 13 | ``` 14 | { 15 | "essential": true, 16 | "image": "public.ecr.aws/aws-observability/aws-otel-collector:latest", 17 | "name": "aws-otel-collector", 18 | "logConfiguration": { 19 | "logDriver": "awslogs", 20 | "options": { 21 | "awslogs-group": "aws-otel-collector", 22 | "awslogs-region": "us-west-1", 23 | "awslogs-create-group": "true", 24 | "awslogs-stream-prefix": "ecs-" 25 | } 26 | }, 27 | "memoryReservation": 100, 28 | "command": [ 29 | "--config=/etc/ecs/container-insights/otel-task-metrics-config.yaml" 30 | ] 31 | } 32 | ``` 33 | 34 | This uses the [built-in ADOT configuration](https://github.com/aws-observability/aws-otel-collector/blob/main/config/ecs/container-insights/otel-task-metrics-config.yaml) for ECS Task Level Metrics. Please read the full [documentation](https://aws-otel.github.io/docs/components/ecs-metrics-receiver). 35 | 36 | This configuration will send metrics to CloudWatch with the following sets of dimensions: 37 | - `ClusterName` 38 | - `ClusterName` and `TaskDefinitionFamily` 39 | 40 | This means that you will get CPU, Memory, Disk, and Network metrics for the entire task aggregated across your cluster, and the union of cluster and TaskDefinition Family. 41 | 42 | ### Tutorial 2: Enable Container Level Metric Collection 43 | 44 | For this tutorial, you must do the following: 45 | 46 | 1. Copy the "Full configuration for task- and container-level metrics" example from the [ADOT ECS Documentation](https://aws-otel.github.io/docs/components/ecs-metrics-receiver). 47 | 2. Optionally modify the dimensions and metrics sent to CloudWatch with the `awsemf` exporter. For example, you could add `TaskId` and `ContainerName` as dimensions to monitoring specific Fluent Bit containers. However, please understand that this will lead to a very large number of individual metrics in CloudWatch which will induce cost. 48 | 3. Follow the [ADOT ECS Custom Config tutorial](https://aws-otel.github.io/docs/setup/ecs/config-through-ssm) to run ADOT with your custom configuration. -------------------------------------------------------------------------------- /examples/fluent-bit/adot-resource-monitoring/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:PutLogEvents" 9 | ], 10 | "Resource": "*" 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /examples/fluent-bit/adot-resource-monitoring/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "public.ecr.aws/aws-observability/aws-otel-collector:latest", 27 | "name": "aws-otel-collector", 28 | "logConfiguration": { 29 | "logDriver": "awslogs", 30 | "options": { 31 | "awslogs-group": "aws-otel-collector", 32 | "awslogs-region": "us-west-1", 33 | "awslogs-create-group": "true", 34 | "awslogs-stream-prefix": "ecs-" 35 | } 36 | }, 37 | "memoryReservation": 100, 38 | "command": [ 39 | "--config=/etc/ecs/container-insights/otel-task-metrics-config.yaml" 40 | ] 41 | }, 42 | { 43 | "essential": true, 44 | "image": "nginx", 45 | "name": "app", 46 | "logConfiguration": { 47 | "logDriver":"awsfirelens", 48 | "options": { 49 | "Name": "cloudwatch_logs", 50 | "region": "us-west-2", 51 | "log_key": "log", 52 | "log_group_name": "ecs/application", 53 | "auto_create_group": "true", 54 | "log_stream_prefix": "app-", 55 | "retry_limit": "2" 56 | } 57 | }, 58 | "memoryReservation": 100 59 | } 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-managed-service-for-prometheus/README.md: -------------------------------------------------------------------------------- 1 | ## FireLens Example: Forward metrics to Prometheus - using the Fluent Bit image with init tag 2 | 3 | This example shows how to forward metrics with Fluent Bit's prometheus remote write output plugin to an Amazon Managed Service for Prometheus workspace. 4 | 5 | ### Step 1: Create an Amazon Managed Service for Prometheus Workspace 6 | 7 | * Create an Amazon Managed Service for Prometheus workspace, `my-prometheus`, to store metrics. Refer to the [AMP onboarding docs](https://docs.aws.amazon.com/prometheus/latest/userguide/AMP-onboard-create-workspace.html) for more information. 8 | * Take note of the `prometheus_remote_write` `uri` from the created workspace's remote write url endpoint, used in Fluent Bit's configuration in step 2. 9 | 10 | ### Step 2: Create config file locally 11 | 12 | **fluent-bit.conf** 13 | 14 | ``` 15 | # FireLens Example: Prometheus forward metrics - using the Fluent Bit image with init tag 16 | 17 | # Scrape node metrics every 20 seconds (collect metrics using any metrics input plugin) 18 | # See the docs for more information: https://docs.fluentbit.io/manual/pipeline/inputs/node-exporter-metrics 19 | [INPUT] 20 | Name node_exporter_metrics 21 | Tag node_metrics 22 | Scrape_interval 20 23 | 24 | # Send metrics to AMP via Remote Write 25 | # See the docs for more information: https://docs.fluentbit.io/manual/pipeline/outputs/prometheus-remote-write 26 | [OUTPUT] 27 | Name prometheus_remote_write 28 | Match node_metrics 29 | Host aps-workspaces.< region >.amazonaws.com 30 | Port 443 31 | Uri /workspaces/< my-amp-workspace-id >/api/v1/remote_write 32 | AWS_Auth On 33 | AWS_region us-west-2 34 | Tls On 35 | Tls.verify On 36 | add_label app my-ecs-app 37 | add_label color blue 38 | ``` 39 | 40 | **Note:** You can find this config file in the `config-files` directory of this example. Please modify the `node_exporter_metrics` input configuration options according to the metrics you desire to collect, along with the `prometheus_remote_write` output configuration options for `Host` and `Uri` to match your AMP setup. 41 | 42 | ### Step 3: Upload config file to S3 43 | 44 | * Create the S3 bucket `your-bucket` to store config files 45 | * Upload above config file to this bucket 46 | 47 | ### Step 4: Create the ECS Task 48 | 49 | * Create the ECS Task using provided `task-definition.json`, which uses the Fluent Bit image with init tag 50 | * Change the `taskRoleArn` to an IAM role that has policies listed in the `permissions.json` file found in this example directory. 51 | * Change the `executionRoleArn` to an IAM role that has necessary permissions for launching an ECS task. See [ecs documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_execution_IAM_role.html) for more information. 52 | * Change the `environment` section in the task definition FireLens configuration, copy the S3 ARN of config file and paste it as environment variable's value. The name of environment variable should be `aws_fluent_bit_init_s3_1` 53 | 54 | ### Step 5: View the metrics with Grafana 55 | 56 | To view the metrics sent to Prometheus, consider creating an Amazon Managed Grafana workspace and adding the Amazon Managed Service for Prometheus workspace as it's data source. Refer to following [documentation](https://docs.aws.amazon.com/grafana/latest/userguide/AMP-adding-AWS-config.html) for more information. 57 | -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-managed-service-for-prometheus/config-files/fluent-bit.conf: -------------------------------------------------------------------------------- 1 | # FireLens Example: Prometheus forward metrics - using the Fluent Bit image with init tag 2 | 3 | # Scrape node metrics every 20 seconds (collect metrics using any metrics input plugin) 4 | # See the docs for more information: https://docs.fluentbit.io/manual/pipeline/inputs/node-exporter-metrics 5 | [INPUT] 6 | Name node_exporter_metrics 7 | Tag node_metrics 8 | Scrape_interval 20 9 | 10 | # Send metrics to AMP via Remote Write 11 | # See the docs for more information: https://docs.fluentbit.io/manual/pipeline/outputs/prometheus-remote-write 12 | [OUTPUT] 13 | Name prometheus_remote_write 14 | Match node_metrics 15 | Host aps-workspaces.< region >.amazonaws.com 16 | Port 443 17 | Uri /workspaces/< my-amp-workspace-id >/api/v1/remote_write 18 | AWS_Auth On 19 | AWS_region us-west-2 20 | Tls On 21 | Tls.verify On 22 | add_label app my-ecs-app 23 | add_label color blue -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-managed-service-for-prometheus/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "AllowRemoteWritePrometheus", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "aps:RemoteWrite" 9 | ], 10 | "Resource": "arn:aws:aps:::workspace/ws-" 11 | }, 12 | { 13 | "Sid": "AllowAccessInitTaskConfig", 14 | "Effect": "Allow", 15 | "Action": [ 16 | "s3:PutObject", 17 | "s3:GetObject", 18 | "s3:GetBucketLocation" 19 | ], 20 | "Resource": [ 21 | "arn:aws:s3:::your-bucket", 22 | "arn:aws:s3:::your-bucket/*" 23 | ] 24 | }, 25 | { 26 | "Sid": "AllowWriteCloudWatchLogs", 27 | "Effect": "Allow", 28 | "Action": [ 29 | "logs:CreateLogGroup", 30 | "logs:PutLogEvents", 31 | "logs:CreateLogStream", 32 | "logs:DescribeLogStreams" 33 | ], 34 | "Resource": "*" 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-managed-service-for-prometheus/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-amazon-managed-service-for-prometheus", 3 | "taskRoleArn": "arn:aws:iam::xxxxxxxxxx:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::xxxxxxxxxx:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "public.ecr.aws/aws-observability/aws-for-fluent-bit:init-latest", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens-init" 20 | } 21 | }, 22 | "environment": [ 23 | { 24 | "name": "aws_fluent_bit_init_s3_1", 25 | "value": "arn:aws:s3:::your-bucket/fluent-bit.conf" 26 | } 27 | ], 28 | "memoryReservation": 50 29 | }, 30 | { 31 | "essential": true, 32 | "name": "my-app", 33 | "image": "nginx", 34 | "logConfiguration": { 35 | "logDriver": "awsfirelens", 36 | "options": { 37 | "Name": "cloudwatch", 38 | "region": "us-west-2", 39 | "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", 40 | "auto_create_group": "true", 41 | "log_stream_name": "$(ecs_task_id)", 42 | "retry_limit": "2" 43 | } 44 | }, 45 | "memoryReservation": 100 46 | } 47 | ] 48 | } -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-opensearch-serverless/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Amazon OpenSearch Serverless with Fluent Bit 2 | 3 | Amazon OpenSearch Serverless is new offering that eliminates your need to manage OpenSearch clusters. All existing Fluent Bit OpenSearch output plugin options work with OpenSearch Serverless. The only difference with serverless from a Fluent Bit POV is that you must specify the service name as `aoss` (Amazon OpenSearch Serverless) when you enable `AWS_Auth`: 4 | 5 | ``` 6 | AWS_Auth On 7 | AWS_Region 8 | AWS_Service_Name aoss 9 | ``` 10 | 11 | ### Data Access Permissions 12 | When sending logs to OpenSearch Serverless, your task role (e.g. `arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role`) needs [OpenSearch Serverless Data Access permisions](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-data-access.html). Give your task role the following Data Access permissions to your serverless collection: 13 | 14 | ``` 15 | aoss:CreateIndex 16 | aoss:UpdateIndex 17 | aoss:WriteDocument 18 | ``` 19 | No task role IAM policies are needed to access the collection. 20 | 21 | ### Adding Permissions with AWS CLI 22 | To add Data Access permissions to your task role via AWS CLI, use the following command along with the `aoss-data-access-policy.json` file from the `amazon-opensearch-serverless` example folder. Be sure to update the `aoss-data-access-policy.json` document with your collection name, and task role arn. 23 | ``` 24 | aws opensearchserverless create-access-policy \ 25 | --name log-write-policy \ 26 | --type data \ 27 | --policy file://./aoss-data-access-policy.json 28 | ``` 29 | Please note that the `opensearchserverless` command was introduced in aws cli v1 in [1.27.29](https://github.com/aws/aws-cli/blob/develop/CHANGELOG.rst#L473) and in aws cli v2 in [2.9.2](https://raw.githubusercontent.com/aws/aws-cli/v2/CHANGELOG.rst). If you recieve the error: `argument command: Invalid choice`, please [update your AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). 30 | 31 | ### Additional Information 32 | For more information on Fluent Bit & Amazon OpenSearch Serverless, see the [official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/outputs/opensearch). 33 | 34 | Learn more about Data access control for Amazon OpenSearch Serverless in the [OpenSearch Service Developer Guide](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-clients.html#serverless-ingestion-permissions). -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-opensearch-serverless/aoss-data-access-policy.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Rules":[ 4 | { 5 | "ResourceType":"index", 6 | "Resource":[ 7 | "index/MY_COLLECTION_NAME/*" 8 | ], 9 | "Permission":[ 10 | "aoss:CreateIndex", 11 | "aoss:UpdateIndex", 12 | "aoss:WriteDocument" 13 | ] 14 | } 15 | ], 16 | "Principal":[ 17 | "arn:aws:iam::123456789012:role/ecs_task_iam_role" 18 | ] 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-opensearch-serverless/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-opensearch-serverless", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:2.30.0", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "opensearch", 32 | "Host": "vpc-fake-domain-ke7thhzo07jawrhmz6mb7ite7y.us-west-2.es.amazonaws.com", 33 | "Port": "443", 34 | "Index": "my_index", 35 | "Aws_Auth": "On", 36 | "Aws_Region": "us-west-2", 37 | "Aws_Service_Name": "aoss", 38 | "Suppress_Type_Name": "On", 39 | "tls": "On", 40 | "retry_limit": "2" 41 | } 42 | }, 43 | "memoryReservation": 100 44 | } 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-opensearch/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Amazon OpenSearch Service with Fluent Bit 2 | 3 | For more information on Fluent Bit & Amazon OpenSearch, see the [official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/outputs/opensearch). 4 | -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-opensearch/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": "es:*", 7 | "Resource": "*" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/amazon-opensearch/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-opensearch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "opensearch", 32 | "Host": "vpc-fake-domain-ke7thhzo07jawrhmz6mb7ite7y.us-west-2.es.amazonaws.com", 33 | "Port": "443", 34 | "Index": "my_index", 35 | "Aws_Auth": "On", 36 | "Aws_Region": "us-west-2", 37 | "Suppress_Type_Name": "On", 38 | "tls": "On", 39 | "retry_limit": "2" 40 | } 41 | }, 42 | "memoryReservation": 100 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs-emf/emf-over-tcp/extra.conf: -------------------------------------------------------------------------------- 1 | # TCP input used for EMF payloads 2 | [INPUT] 3 | Name tcp 4 | Listen 0.0.0.0 5 | Port 25888 6 | Chunk_Size 32 7 | Buffer_Size 64 8 | Format none 9 | Tag emf-${HOSTNAME} 10 | # This tag is used by the output plugin to determine the LogStream 11 | # including the HOSTNAME is a way to increase the number of LogStreams. 12 | # The maximum throughput on a 13 | # single LogStream is 5 MB/s (max 1 MB at max 5 TPS). 14 | # In AWSVPC mode, the HOSTNAME is the ENI private IP 15 | # in bridge mode, the HOSTNAME is the Docker container ID 16 | 17 | # Output for EMF over TCP -> CloudWatch 18 | [OUTPUT] 19 | Name cloudwatch_logs 20 | Match emf-* 21 | region us-west-2 22 | log_key log 23 | log_group_name aws-emf-ecs-firelens-example-metrics 24 | log_stream_prefix from-fluent-bit- 25 | auto_create_group true 26 | log_format json/emf 27 | # consider configuring more than 1 worker, up to the numer of cores on your host 28 | # to accommodate higher throughput logging. Only 2.32+ supports multi-worker 29 | workers 1 30 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs-emf/emf-over-tcp/task-definition-bridge.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "fluentbit", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "portMappings": [{ 14 | "protocol": "tcp", 15 | "containerPort": 25888 16 | }], 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-west-2", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "memoryReservation": 50 27 | }, 28 | { 29 | "essential": true, 30 | "image": "nginx", 31 | "name": "app", 32 | "links": [ "fluentbit" ], 33 | "environment": [ 34 | { 35 | "name": "AWS_EMF_AGENT_ENDPOINT", 36 | "value": "tcp://fluentbit:25888" 37 | } 38 | ], 39 | "logConfiguration": { 40 | "logDriver":"awsfirelens", 41 | "options": { 42 | "Name": "cloudwatch_logs", 43 | "region": "us-west-2", 44 | "log_key": "log", 45 | "log_group_name": "stdout-stderr-log-group/application", 46 | "auto_create_group": "true", 47 | "log_stream_prefix": "app-", 48 | "retry_limit": "2", 49 | "workers": "1" 50 | } 51 | }, 52 | "memoryReservation": 100 53 | } 54 | ] 55 | } 56 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs-emf/emf-over-tcp/task-definition-localhost.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "environment": [{ 29 | "name": "AWS_EMF_AGENT_ENDPOINT", 30 | "value": "tcp://127.0.0.1:25888" 31 | }], 32 | "logConfiguration": { 33 | "logDriver":"awsfirelens", 34 | "options": { 35 | "Name": "cloudwatch_logs", 36 | "region": "us-west-2", 37 | "log_key": "log", 38 | "log_group_name": "stdout-stderr-log-group/application", 39 | "auto_create_group": "true", 40 | "log_stream_prefix": "app-", 41 | "retry_limit": "2" 42 | } 43 | }, 44 | "memoryReservation": 100 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs-emf/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:PutLogEvents" 9 | ], 10 | "Resource": "*" 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs-emf/task-definition-stdout-emf.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "cloudwatch_logs", 32 | "region": "us-west-2", 33 | "log_key": "log", 34 | "log_group_name": "emf-log-group/application", 35 | "auto_create_group": "true", 36 | "log_stream_prefix": "emf-", 37 | "log_format": "json/emf", 38 | "retry_limit": "2" 39 | } 40 | }, 41 | "memoryReservation": 100 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs/README.md: -------------------------------------------------------------------------------- 1 | ## FireLens Example: Logging to CloudWatch Logs with Fluent Bit 2 | 3 | ### CloudWatch Golang Plugin vs CloudWatch C Plugin 4 | 5 | There are two Fluent Bit output plugins for sending to Amazon CloudWatch Logs: 6 | * Plugin name `cloudwatch`: the original golang plugin with extensive templating support, including injecting ECS metadata into log stream and group name templates. See its [documentation](https://github.com/aws/amazon-cloudwatch-logs-for-fluent-bit). This directory contains an example task definition thats demonstrates this plugin's ability to inject ECS task metadata into log group and stream names. 7 | * Plugin name `cloudwatch_logs`: the newer and higher performance cloudwatch plugin built in C in the Fluent Bit upstream code base. It has more limited log group and stream name templating support. See its [documentation](https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch). This directory contains an example task definition for the high performance plugin without templating. The log stream name will be set to be `{log_stream_prefix}{log tag}` and [FireLens sets the log tag](https://github.com/aws/aws-for-fluent-bit/blob/mainline/troubleshooting/debugging.md#firelens-tag-and-match-pattern-and-generated-config) to be `{container name in task definition}-firelens-{task ID}`. So the log stream name for this example will be `stdout-stderr-app-firelens-{task ID}`. 8 | 9 | 10 | For more on the AWS Go outputs vs AWS C outputs, check out the [FAQ entry in our debugging guide](https://github.com/aws/aws-for-fluent-bit/blob/mainline/troubleshooting/debugging.md#aws-go-plugins-vs-aws-core-c-plugins). 11 | 12 | #### Recommended cloudwatch_logs configuration options 13 | 14 | To minimize the possibility of log loss when sending to CloudWatch, consider using the recommended configuration outlined in the [CloudWatch Recommendations](https://github.com/aws/aws-for-fluent-bit/issues/340) issue. 15 | 16 | #### High throughput logging to CloudWatch Logs via multiple workers 17 | 18 | As of AWS For Fluent Bit `2.32.0`, the `cloudwatch_logs` plugin supports high throughput logging via multiple workers. Set the `workers` option to an integer value, such as `5`, indicating the number of worker threads dedicated to processing output data to CloudWatch Logs concurrently. To optimize for high throughput logging, consider setting the number of workers the number of cores on the host. AWS For Fluent Bit `2.31.12` and prior does not support multiple workers, and `workers` should be set to `1`. 19 | 20 | ### What if I just want the raw log line from the container to appear in CloudWatch? 21 | 22 | The example shown here is for the `cloudwatch` plugin, however, the `cloudwatch_logs` plugin has the same `log_key` option. 23 | 24 | By default, FireLens will send a JSON event with the raw log line encapsulated in a `log` field. ECS Metadata will also be added. If you just want the raw log line, add the `log_key` option to your log configuration: 25 | 26 | ``` 27 | "logConfiguration": { 28 | "logDriver":"awsfirelens", 29 | "options": { 30 | "Name": "cloudwatch", 31 | "region": "us-west-2", 32 | "log_key": "log", 33 | "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", 34 | "auto_create_group": "true", 35 | "log_stream_name": "$(ecs_task_id)", 36 | "retry_limit": "2" 37 | } 38 | }, 39 | ``` 40 | 41 | This field instructs the CloudWatch plugin to only send the value of the log key. You should additionally disable ECS Metadata to prevent Fluent Bit from performing unnecessary data processing: 42 | 43 | ``` 44 | "firelensConfiguration": { 45 | "type": "fluentbit", 46 | "options": { 47 | "enable-ecs-log-metadata": "false", 48 | } 49 | }, 50 | ``` 51 | 52 | "log_key": "log", 53 | The following table shows how your logs will appear in CloudWatch with and without `log_key` and `enable-ecs-log-metadata` if "`my_sample log`" is printed to stdout: 54 | 55 | |logConfiguration.options contains|firelensConfiguration.options contains|received by CloudWatch|description|comments| 56 | |-|-|-|-|-| 57 | |||{
"container_id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXXXXXX",
"container_name": "my_container_name",
"ecs_cluster": "my_cluster_name",
"ecs_task_arn": "arn:aws:ecs:region:9876543210:task/my_task_arn",
"ecs_task_definition": "my_task_definition:revision_number",
"log": "my_sample log",
"source": "stdout"
}|no log key is set and ecs-log-metadata is enabled by default|| 58 | ||"enable-ecs-log-metadata":"false"|{
"container_id": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXXXXXXXXX",
"container_name": "my_container_name",
"log": "my_sample log",
"source": "stdout"
}|no log key is set and ecs-log-metadata is disabled|| 59 | |"log_key":"log"||"my_sample log"|log key set to "log" and ecs-log-metadata is enabled by default|less efficient| 60 | |"log_key":"log"|"enable-ecs-log-metadata":"false"|"my_sample log"|log key set to "log" and ecs-log-metadata is disabled|more efficient| 61 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs/task-definition-cloudwatch.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "cloudwatch", 32 | "region": "us-west-2", 33 | "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", 34 | "auto_create_group": "true", 35 | "log_stream_name": "$(ecs_task_id)", 36 | "retry_limit": "2" 37 | } 38 | }, 39 | "memoryReservation": 100 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /examples/fluent-bit/cloudwatchlogs/task-definition-cloudwatch_logs.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch_logs", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "cloudwatch_logs", 32 | "region": "us-west-2", 33 | "log_group_name": "/aws/ecs/containerinsights/application", 34 | "auto_create_group": "true", 35 | "log_stream_name": "stdout-stderr-", 36 | "retry_limit": "2", 37 | "workers": "1" 38 | } 39 | }, 40 | "memoryReservation": 100 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /examples/fluent-bit/config-file-type-file/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazon/aws-for-fluent-bit:latest 2 | ADD extra.conf /extra.conf 3 | -------------------------------------------------------------------------------- /examples/fluent-bit/config-file-type-file/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Using the 'file' config type 2 | 3 | **NOTE**: *AWS for Fluent Bit distro now supports an `init` tag for ECS customers which can do the work to import multiple config files from your image or an S3 bucket into the final Fluent Bit configuration*. It can be used as an alternative to the FireLens `config-file-type` and `config-file-value` options. Check out the [init documentation](https://github.com/aws/aws-for-fluent-bit#using-the-init-tag). 4 | 5 | --- 6 | 7 | This is the same config example as the "Adding custom keys to log events" example. In this though, we specifically look at how to use the 'file' 'config-file-type' in FireLens. 8 | 9 | Let's go through it step by step. 10 | 11 | 1. Add your own custom config to extra.conf. Note that the name of that file is not special; you could name it anything. 12 | 2. Build a custom Fluent Bit image using the provided Docker file (which simply copies your extra config file into the AWS for Fluent Bit image). You can place this extra config file anywhere in the Docker image *except* `/fluent-bit/etc/fluent-bit.conf`. That config file path is the path used by FireLens. Push this custom image to Amazon ECR. 13 | 3. Reference your new custom Fluent Bit image ECR repo in the container definition for the FireLens container. 14 | 4. Reference the config file path in the FireLens configuration: 15 | ``` 16 | "firelensConfiguration": { 17 | "type": "fluentbit", 18 | "options": { 19 | "config-file-type": "file", 20 | "config-file-value": "/extra.conf" 21 | } 22 | } 23 | ``` 24 | -------------------------------------------------------------------------------- /examples/fluent-bit/config-file-type-file/extra.conf: -------------------------------------------------------------------------------- 1 | [FILTER] 2 | Name record_modifier 3 | Match * 4 | Record app-version ${APP_VERSION} 5 | -------------------------------------------------------------------------------- /examples/fluent-bit/config-file-type-file/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "firehose:PutRecordBatch" 7 | ], 8 | "Resource": "*" 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/config-file-type-file/runtask_firelens.json: -------------------------------------------------------------------------------- 1 | { 2 | "cluster": "default", 3 | "taskDefinition": "firelens-amazon-es-bug-bash:4", 4 | "launchType": "FARGATE", 5 | "networkConfiguration": { 6 | "awsvpcConfiguration": { 7 | "assignPublicIp": "ENABLED", 8 | "subnets": [ 9 | "subnet-085a02b9ae9f3e86e", 10 | "subnet-0b91aaca96e30c195" 11 | ], 12 | "securityGroups": [ 13 | "sg-030bc708ef4c925a4" 14 | ] 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /examples/fluent-bit/config-file-type-file/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-firehose", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "11111111111.dkr.ecr.us-east-1.amazonaws.com/custom-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "file", 14 | "config-file-value": "/extra.conf" 15 | } 16 | }, 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-west-2", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "environment": [ 27 | { 28 | "name": "APP_VERSION", 29 | "value": "v1.1.14" 30 | } 31 | ], 32 | "memoryReservation": 50 33 | }, 34 | { 35 | "essential": true, 36 | "image": "httpd", 37 | "name": "app", 38 | "logConfiguration": { 39 | "logDriver":"awsfirelens", 40 | "options": { 41 | "Name": "firehose", 42 | "region": "us-west-2", 43 | "delivery_stream": "my-stream", 44 | "retry_limit": "2" 45 | } 46 | }, 47 | "memoryReservation": 100 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /examples/fluent-bit/cross-account/README.md: -------------------------------------------------------------------------------- 1 | # FireLens Cross Account Log Push Example 2 | 3 | This example shows you how to push logs from ECS tasks running in one account, to a CloudWatch Log Group in another account. While the example demonstrates using CloudWatch Logs, all AWS outputs in Fluent Bit support cross account. The steps in this tutorial can be followed for other AWS outputs with only slight modifications. 4 | 5 | This tutorial will use the following terminology: 6 | * Task Account: The AWS Account with the running ECS task containers that produce logs. Let's assume the ID for this account is `111111111111`. 7 | * Log Account: The AWS Account with the CloudWatch Log Group that will have the container logs. Let's assume the ID for this account is `222222222222`. 8 | 9 | 10 | ### Step 1: Configure Base Role 11 | 12 | To push logs from the Task Account to the Log Account, Fluent Bit must use a set of base credentials which it will then use to obtain credentials for the cross account log push. 13 | 14 | Since the Fluent Bit FireLens sidecar runs as a container in the task, it uses the ECS Task Role for its base role/base credentials. This [role is specified in the Task Definition](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) with the `taskRoleArn` field. Let's assume the ARN for the ECS Task Role is `arn:aws:iam::111111111111:role/ecs-task-role`. 15 | 16 | 17 | The role should have the following permissions: 18 | 19 | ``` 20 | { 21 | "Version": "2012-10-17", 22 | "Statement": [ 23 | { 24 | "Effect": "Allow", 25 | "Action": "sts:AssumeRole", 26 | "Resource": "*" 27 | } 28 | ] 29 | } 30 | ``` 31 | 32 | This will allow entities that use the task role (the task containers) to call the STS AssumeRole API. This will allow Fluent Bit to assume the role in the Log Account. 33 | 34 | If you want to limit which roles can be assumed, that can be done with the `Resource` field in the policy above. 35 | 36 | ### Step 2: Set up role in Log Account 37 | 38 | Now in the other account that we will push logs to, we create a role that Fluent Bit can assume. 39 | 40 | The role in the Log Account will be: `arn:aws:iam::222222222222:role/cross-account-log-role`. 41 | 42 | And it will have the following trust relationship: 43 | 44 | ``` 45 | { 46 | "Version": "2012-10-17", 47 | "Statement": [ 48 | { 49 | "Effect": "Allow", 50 | "Principal": { 51 | "arn:aws:iam::111111111111:role/ecs-task-role" 52 | }, 53 | "Action": "sts:AssumeRole", 54 | "Condition": {} 55 | } 56 | ] 57 | } 58 | ``` 59 | 60 | This allows the task role in the Task Account to assume this role. Then add permissions to this role, specifically the `cloudwatch-write.json` policy in this example's directory. If you want to use a different log destination instead of CloudWatch, simply attach the permissions for your chosen destination. 61 | 62 | ### Step 3: Configure Fluent Bit to assume the log push role 63 | 64 | Finally, we just need to configure Fluent Bit to assume the cross account role. To do this, simply specify it with the `role_arn` parameter that all AWS outputs support: 65 | 66 | ``` 67 | role_arn arn:aws:iam::222222222222:role/cross-account-log-role 68 | ``` 69 | 70 | The `task-definition.json` example in this directory demonstrates adding this parameter for a CloudWatch output configured via the task definition. If you have outputs in a custom configuration file, you would add the parameter like this: 71 | 72 | ``` 73 | [OUTPUT] 74 | Name cloudwatch_logs 75 | Match * 76 | Log_Group_Name cross-account-example 77 | Log_Stream_Prefix example- 78 | Auto_Create_Group On 79 | Role_Arn arn:aws:iam::222222222222:role/cross-account-log-role 80 | ``` 81 | 82 | Please note that Fluent Bit will always use regional STS endpoints by default. If you have a VPC endpoint for STS that you want to use, you can specify it with the `sts_endpoint` option supported by all AWS outputs. 83 | 84 | -------------------------------------------------------------------------------- /examples/fluent-bit/cross-account/cloudwatch-write.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /examples/fluent-bit/cross-account/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "cloudwatch", 32 | "region": "us-west-2", 33 | "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", 34 | "auto_create_group": "true", 35 | "log_stream_name": "$(ecs_task_id)", 36 | "retry_limit": "2", 37 | "role_arn": "arn:aws:iam::222222222222:role/cross-account-log-role" 38 | } 39 | }, 40 | "memoryReservation": 100 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /examples/fluent-bit/datadog/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Datadog with Fluent Bit 2 | 3 | For documentation on sending your FireLens monitored log data to Datadog Logs, see: [Fluent Bit and Firelens](https://docs.datadoghq.com/integrations/ecs_fargate/#fluent-bit-and-firelens). 4 | 5 | It should be noted that this example and Datadog docs show `"enable-ecs-log-metadata":"true"` (which is the default). This option tells FireLens to add ECS Task Metadata keys to logs. However, the Datadog output [intentionally](https://github.com/fluent/fluent-bit/blob/v1.9.10/plugins/out_datadog/datadog.c#L254) will not send the metadata in the log and will instead convert the metadata to DataDog tags for easy searching and integration with other DataDog observability features. 6 | 7 | For all configuration parameters for Fluent Bit DataDog output plugin, see [DataDog](https://docs.datadoghq.com/integrations/fluentbit/#configuration-parameters) or [Fluent Bit](https://docs.fluentbit.io/manual/output/datadog) documentation. 8 | 9 | AWS recommends that you store sensitive information, like your Datadog API Key using [secretOptions](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html) as shown in the example Task Definition. This is optional; it is also valid to simply specify the API Key in options map: 10 | 11 | ``` 12 | "logConfiguration": { 13 | "logDriver":"awsfirelens", 14 | "options": { 15 | "Name": "datadog", 16 | "Host": "http-intake.logs.datadoghq.com", 17 | "TLS": "on", 18 | "apikey": "", 19 | "dd_service": "my-httpd-service", 20 | "dd_source": "httpd", 21 | "dd_tags": "project:example", 22 | "provider": "ecs", 23 | "retry_limit": "2" 24 | } 25 | }, 26 | ``` 27 | -------------------------------------------------------------------------------- /examples/fluent-bit/datadog/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-datadog", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration":{ 11 | "type":"fluentbit", 12 | "options":{ 13 | "enable-ecs-log-metadata":"true" 14 | } 15 | }, 16 | "memoryReservation": 50 17 | }, 18 | { 19 | "essential": true, 20 | "image": "httpd", 21 | "name": "app", 22 | "logConfiguration": { 23 | "logDriver":"awsfirelens", 24 | "options": { 25 | "Name": "datadog", 26 | "Host": "http-intake.logs.datadoghq.com", 27 | "TLS": "on", 28 | "dd_service": "my-httpd-service", 29 | "dd_source": "httpd", 30 | "dd_tags": "project:example", 31 | "provider": "ecs", 32 | "retry_limit": "2" 33 | }, 34 | "secretOptions": [{ 35 | "name": "apikey", 36 | "valueFrom": "arn:${Partition}:secretsmanager:${Region}:${Account}:secret:${SecretId}" 37 | }] 38 | }, 39 | "memoryReservation": 100 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /examples/fluent-bit/dynatrace/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Dynatrace with Fluent Bit 2 | 3 | For documentation on sending your FireLens monitored log data to Dynatrace Logs, see: [Fluent Bit with Dynatrace](https://www.dynatrace.com/hub/detail/fluent-bit/). 4 | 5 | AWS recommends that you store sensitive information, like your Dynatrace API Token using [secretOptions](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html) as shown in the example [Task Definition](https://github.com/dynatrace-oss-contrib/amazon-ecs-firelens-examples/blob/mainline/examples/fluent-bit/dynatrace/task-definition.json). This is optional; it is also valid to simply specify the API Token in the URI map: 6 | 7 | ``` 8 | "logConfiguration": { 9 | "logDriver":"awsfirelens", 10 | "options": { 11 | "Name": "http", 12 | "Host": "{your-environment-id}.live.dynatrace.com", 13 | "TLS": "on", 14 | "TLS.verify" : "off", 15 | "Format": "json", 16 | "Json_Date_Format": "iso8601", 17 | "Json_Date_Key": "timestamp", 18 | "Header: "Content-Type application/json; charset=utf-8", 19 | "Port": "443", 20 | "URI": "/api/v2/logs/ingest?api-token={your-API-Token-here}", 21 | "Allow_Duplicated_Headers": "false", 22 | "retry_limit": "2" 23 | } 24 | }, 25 | ``` 26 | -------------------------------------------------------------------------------- /examples/fluent-bit/dynatrace/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-dynatrace", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "public.ecr.aws/aws-observability/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration":{ 11 | "type":"fluentbit", 12 | "options":{ 13 | "enable-ecs-log-metadata":"true" 14 | } 15 | }, 16 | "memoryReservation": 50 17 | }, 18 | { 19 | "essential": true, 20 | "image": "httpd", 21 | "name": "app", 22 | "logConfiguration": { 23 | "logDriver":"awsfirelens", 24 | "options": { 25 | "Name":"http", 26 | "Host":"{your-environment-id}.live.dynatrace.com", 27 | "TLS":"on", 28 | "TLS.verify":"off", 29 | "Format":"json", 30 | "Json_Date_Format":"iso8601", 31 | "Json_Date_Key":"timestamp", 32 | "Header":"Content-Type application/json; charset=utf-8", 33 | "Port":"443" 34 | "Allow_Duplicated_Headers":"false", 35 | "retry_limit":"2" 36 | }, 37 | "secretOptions": [{ 38 | "name": "URI", 39 | "valueFrom": "arn:${Partition}:secretsmanager:${Region}:${Account}:secret:${SecretId}" 40 | }] 41 | }, 42 | "memoryReservation": 100 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-collection/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-collection/task-definition-tail.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "s3", 14 | "config-file-value": "arn:aws:s3:::yourbucket/yourdirectory/tail.conf" 15 | } 16 | }, 17 | "mountPoints" : [ 18 | { 19 | "containerPath" : "/var/log/", 20 | "sourceVolume" : "var-log" 21 | } 22 | ], 23 | "logConfiguration": { 24 | "logDriver": "awslogs", 25 | "options": { 26 | "awslogs-group": "firelens-container", 27 | "awslogs-region": "us-east-1", 28 | "awslogs-create-group": "true", 29 | "awslogs-stream-prefix": "firelens" 30 | } 31 | }, 32 | "memoryReservation": 50 33 | }, 34 | { 35 | "essential": true, 36 | "image": "my-app:latest", 37 | "name": "app", 38 | "mountPoints": [ 39 | { 40 | "containerPath": "/var/log/", 41 | "sourceVolume": "var-log" 42 | } 43 | ], 44 | "logConfiguration": { 45 | "logDriver":"awsfirelens" 46 | }, 47 | "memoryReservation": 100 48 | } 49 | ], 50 | "volumes" : [ 51 | { 52 | "name" : "var-log" 53 | } 54 | ] 55 | } 56 | -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-collection/task-definition-tcp.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "s3", 14 | "config-file-value": "arn:aws:s3:::yourbucket/yourdirectory/tcp.conf" 15 | } 16 | }, 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-east-1", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "memoryReservation": 50 27 | }, 28 | { 29 | "essential": true, 30 | "image": "my-app:latest", 31 | "name": "app", 32 | "logConfiguration": { 33 | "logDriver":"awsfirelens" 34 | }, 35 | "memoryReservation": 100 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-deletion/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the standard Amazon Linux base 2 | FROM amazonlinux 3 | 4 | # Install Cron 5 | RUN yum install -y cronie which findutils perl-core compat-libcap1 lsof tar shadow-utils setuptool procps && yum clean all 6 | 7 | # Copy crontab that deletes logs 8 | COPY cron/delete-log-files /etc/cron.d/delete-log-files 9 | 10 | # Give execution rights on the cron job 11 | RUN chmod 0644 /etc/cron.d/delete-log-files 12 | 13 | -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-deletion/README.md: -------------------------------------------------------------------------------- 1 | # ECS FireLens Log Deletion Example 2 | 3 | This example will walk you through options to set up log file deletion/clean up in your ECS FireLens task. 4 | 5 | This example is based upon the tail example in the [ecs-log-collection](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/ecs-log-collection) FireLens example. It adds log deletion. 6 | 7 | ## Scenario 8 | 9 | For this example, consider that your app produces log files with paths like `/var/log/service-{timestamp}.log`. Each hour, your app write to a log file with a new timestamp. Thus, if your app runs for a very long time, you will quickly have lots of log files sitting around on disk. This is not ideal for two reasons: 10 | 1. Your disk may eventually fill up. 11 | 2. The large number of log files may slow down Fluent Bit. Fluent Bit continuously scans for files matched by your configured `Path` and runs a syscall on them to check if they were modified. If you have tons of log files that it must scan through, this will cause it to slow down, and in extreme cases, you may even lose logs. 12 | 13 | Fluent Bit will use this base configuration (based on the ECS Log Collection example) to tail the log files: 14 | 15 | ``` 16 | [INPUT] 17 | Name tail 18 | Tag service-log 19 | Path /var/log/service*.log 20 | DB /var/log/flb_service.db 21 | DB.locking true 22 | Skip_Long_Lines On 23 | Refresh_Interval 10 24 | Rotate_Wait 30 25 | # Output for stdout logs 26 | # Change the match if your container is not named 'app' 27 | [OUTPUT] 28 | Name cloudwatch 29 | Match app-firelens* 30 | region us-east-1 31 | log_group_name firelens-tutorial-$(ecs_cluster) 32 | log_stream_name /logs/app/$(ec2_instance_id)-$(ecs_task_id) 33 | auto_create_group true 34 | retry_limit 2 35 | # Output for the file logs 36 | [OUTPUT] 37 | Name cloudwatch 38 | Match service-log 39 | region us-east-1 40 | log_group_name firelens-tutorial-$(ecs_cluster) 41 | log_stream_name /logs/service/$(ec2_instance_id)-$(ecs_task_id) 42 | auto_create_group true 43 | retry_limit 2 44 | ``` 45 | 46 | ## Log Deletion Options 47 | 48 | ### Background 49 | 50 | Make sure you are familiar with the [Fluent Bit documentation](https://docs.fluentbit.io/manual/) and with [AWS for Fluent Bit](https://github.com/aws/aws-for-fluent-bit) and its container image distribution. 51 | 52 | ### 1. Use Fluent Bit Exec Input to run deletion command 53 | 54 | ##### Warning: exec input issue in AWS for Fluent Bit <= 2.31.11 55 | 56 | There is a [known issue](https://github.com/aws/aws-for-fluent-bit/issues/661#issuecomment-1569515241) in the exec input in AWS for Fluent Bit <= 2.31.11 which can cause it to crash, generally shortly after startup. 57 | 58 | This issue is resolved in [2.31.12](https://github.com/aws/aws-for-fluent-bit/releases). 59 | 60 | ##### Prerequisites: 61 | * The volume mount for Fluent Bit must not be read-only 62 | * Familiar with [exec input](https://docs.fluentbit.io/manual/pipeline/inputs/exec) of Fluent Bit 63 | 64 | You can run a command that deletes log files by adding an exec input to the configuration. Here is the example: 65 | 66 | ``` 67 | [INPUT] 68 | Name exec 69 | Tag delete-old-files 70 | Command find /var/log/ -type f -mmin +10080 -execdir rm -f -- '{}' ';' 71 | Interval_Sec 600 72 | Interval_NSec 0 73 | Buf_Size 8mb 74 | Oneshot false 75 | 76 | [OUTPUT] 77 | Name null 78 | Match delete-old-files 79 | ``` 80 | Here is a breakdown of the individual components of the command: 81 | * ```Interval_Sec```: This is the polling interval in seconds. 82 | * ```Oneshot```: Only run once at startup. 83 | * ```find /var/log/```: This is the ```find``` command, which searches for files in the directory ```/var/log/```. 84 | * ```-type f```: This option tells find to search only for regular files, not directories or other types of files. 85 | * ```-mmin +10080```: This option tells find to search for files that are older than 7 days (10080 minutes). 86 | * ```-execdir rm -f -- '{}'';'```: This is the action to take on the files that match the search criteria. The ```-execdir``` option runs the ```rm``` command in the directory where each file is found. The ```-f``` option to ```rm``` tells it to force the deletion without prompting for confirmation. The ```--```is a separator that indicates the end of the command options and the start of the filenames, and ```{}``` is a placeholder that gets replaced with the name of each file found by find. Finally, the ```';'``` marks the end of the command that should be executed for each file. 87 | 88 | ### 2. Run deletion command with cron 89 | 90 | ##### Prerequisite: 91 | * Familiar with [cron](https://en.wikipedia.org/wiki/Cron) 92 | * Docker should be installed on your system 93 | 94 | **1. Create a cron job file and write the command below inside the file** 95 | 96 | * Create a file called "delete-log-files" in file ```/cron``` 97 | * Paste the following command into the file 98 | ``` 99 | 0 0 * * * find /var/log/ -type f -mmin +10080 -execdir rm -f -- '{}' ';' 100 | ``` 101 | This is a cron job that runs every day at midnight (```0 0 * * *```). It uses the ```find``` command to locate files in the directory ```/var/log/``` that are older than 7 days (10080 minutes), and then deletes them using the ```rm -f``` command. 102 | * ```0 0 * * *```: This is the cron schedule expression. It specifies that the job should run at 12:00 AM every day. 103 | * Note: In the cron job file called "cron", an empty line is required at the end of the file for a valid cron file. 104 | 105 | **2. Build image with cron** 106 | 107 | * Install Cron 108 | ``` 109 | RUN yum install -y cronie which findutils perl-core compat-libcap1 lsof tar shadow-utils setuptool procps && yum clean all 110 | ``` 111 | 112 | * Copy crontab that deletes logs 113 | ``` 114 | COPY cron/delete-log-files /etc/cron.d/delete-log-files 115 | ``` 116 | 117 | * Give execution rights on the cron job 118 | ``` 119 | RUN chmod 0644 /etc/cron.d/delete-log-files 120 | ``` 121 | 122 | Note: You can view the dockerfile example [here](Dockerfile) and run command ```docker build``` to build the image. 123 | 124 | ### 3. Use log4j delete action 125 | 126 | You can use log4j delete action via public RollingRandomAccessFile appender with the following guidances: 127 | * [delete-logs-on-rollover](https://howtodoinjava.com/java/delete-logs-on-rollover/) 128 | * [Apache log4j docs: Delete on Rollover](https://logging.apache.org/log4j/log4j-2.7/manual/appenders.html#CustomDeleteOnRollover) 129 | 130 | Following is the example of deleting old log files older than 1 day: 131 | ``` 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | ``` -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-deletion/cron/delete-log-files: -------------------------------------------------------------------------------- 1 | 0 0 * * * find /var/log/ -type f -mmin +10080 -execdir rm -f -- '{}' ';' -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-deletion/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } -------------------------------------------------------------------------------- /examples/fluent-bit/ecs-log-deletion/take-definition-tail.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "s3", 14 | "config-file-value": "arn:aws:s3:::yourbucket/yourdirectory/tail.conf" 15 | } 16 | }, 17 | "mountPoints" : [ 18 | { 19 | "containerPath" : "/var/log/", 20 | "sourceVolume" : "var-log" 21 | } 22 | ], 23 | "logConfiguration": { 24 | "logDriver": "awslogs", 25 | "options": { 26 | "awslogs-group": "firelens-container", 27 | "awslogs-region": "us-east-1", 28 | "awslogs-create-group": "true", 29 | "awslogs-stream-prefix": "firelens" 30 | } 31 | }, 32 | "memoryReservation": 50 33 | }, 34 | { 35 | "essential": true, 36 | "image": "my-app:latest", 37 | "name": "app", 38 | "mountPoints": [ 39 | { 40 | "containerPath": "/var/log/", 41 | "sourceVolume": "var-log" 42 | } 43 | ], 44 | "logConfiguration": { 45 | "logDriver":"awsfirelens" 46 | }, 47 | "memoryReservation": 100 48 | } 49 | ], 50 | "volumes" : [ 51 | { 52 | "name" : "var-log" 53 | } 54 | ] 55 | } -------------------------------------------------------------------------------- /examples/fluent-bit/efs/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Storing Configs in EFS 2 | 3 | This example is similar to the 'file' 'config-file-type' in FireLens example. However, in this example, instead of creating a custom Fluent Bit image with the config file, we create an EFS drive with the configuration and then mount it into all of our tasks. One option is to store multiple configs in a single EFS, then mount that into all tasks, and use the FireLens configuration to select the config file for each task. 4 | 5 | Let's go through it step by step. 6 | 7 | 1. Create the EFS drive and add your custom configuration files to it: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/tutorial-efs-volumes.html 8 | 2. Mount the EFS drive in your task definition; here is an example: 9 | ``` 10 | "volumes": [ 11 | { 12 | "name": "firelens-conf", 13 | "efsVolumeConfiguration": { 14 | "fileSystemId": "fs-c3c9bcbb", 15 | "rootDirectory": "/", 16 | "transitEncryption": "DISABLED", 17 | "authorizationConfig": { 18 | "iam": "DISABLED" 19 | } 20 | } 21 | } 22 | ], 23 | ``` 24 | 3. Next, mount the drive into your Fluent Bit container: 25 | ``` 26 | "mountPoints": [ 27 | { 28 | "sourceVolume": "firelens-conf", 29 | "containerPath": "/configs" 30 | } 31 | ], 32 | ``` 33 | 3. Finally, reference the configuration file from the EFS drive in FireLens: 34 | ``` 35 | "firelensConfiguration": { 36 | "type": "fluentbit", 37 | "options": { 38 | "config-file-type": "file", 39 | "config-file-value": "/configs/extra.conf" 40 | } 41 | } 42 | ``` 43 | 44 | 45 | See the task definition file in this folder for a full example. 46 | -------------------------------------------------------------------------------- /examples/fluent-bit/efs/extra.conf: -------------------------------------------------------------------------------- 1 | [OUTPUT] 2 | Name kinesis_firehose 3 | Match * 4 | region us-east-1 5 | delivery_stream my-stream 6 | retry_limit 2 -------------------------------------------------------------------------------- /examples/fluent-bit/efs/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "taskDefinition": { 3 | "family": "firelens-efs-example", 4 | "taskRoleArn": "arn:aws:iam::11111111111:role/ecsTaskExecutionRole", 5 | "executionRoleArn": "arn:aws:iam::11111111111:role/ecsTaskExecutionRole", 6 | "networkMode": "awsvpc", 7 | "containerDefinitions": [ 8 | { 9 | "name": "nginx", 10 | "image": "nginx", 11 | "cpu": 0, 12 | "portMappings": [ 13 | { 14 | "containerPort": 80, 15 | "hostPort": 80, 16 | "protocol": "tcp" 17 | } 18 | ], 19 | "essential": true, 20 | "environment": [], 21 | "mountPoints": [], 22 | "volumesFrom": [], 23 | "logConfiguration": { 24 | "logDriver": "awsfirelens" 25 | } 26 | }, 27 | { 28 | "name": "log_router", 29 | "image": "906394416424.dkr.ecr.us-east-2.amazonaws.com/aws-for-fluent-bit:stable", 30 | "cpu": 0, 31 | "portMappings": [], 32 | "essential": true, 33 | "environment": [], 34 | "mountPoints": [ 35 | { 36 | "sourceVolume": "firelens-conf", 37 | "containerPath": "/configs" 38 | } 39 | ], 40 | "volumesFrom": [], 41 | "user": "0", 42 | "logConfiguration": { 43 | "logDriver": "awslogs", 44 | "options": { 45 | "awslogs-create-group": "true", 46 | "awslogs-group": "firelens-container", 47 | "awslogs-region": "us-east-2", 48 | "awslogs-stream-prefix": "firelens" 49 | } 50 | }, 51 | "firelensConfiguration": { 52 | "type": "fluentbit", 53 | "options": { 54 | "config-file-type": "file", 55 | "config-file-value": "/configs/extra.conf" 56 | } 57 | } 58 | } 59 | ], 60 | "volumes": [ 61 | { 62 | "name": "firelens-conf", 63 | "efsVolumeConfiguration": { 64 | "fileSystemId": "fs-c3c9bcbb", 65 | "rootDirectory": "/", 66 | "transitEncryption": "DISABLED", 67 | "authorizationConfig": { 68 | "iam": "DISABLED" 69 | } 70 | } 71 | } 72 | ], 73 | "cpu": "512", 74 | "memory": "1024" 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /examples/fluent-bit/elastic-cloud/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Elastic Cloud with Fluent Bit 2 | 3 | For all configuration parameters for Fluent Bit Elasticsearch output plugin and Elastic Cloud connection, see the [official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/outputs/elasticsearch). 4 | 5 | AWS recommends that you store sensitive information, like your credentials used to connect to Elastic's Elasticsearch Service using [secretOptions](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html) as shown in the example Task Definition. Please note that the secret must be named Cloud_Auth. Value in this secret should not have any associated key and single/double quotes. 6 | 7 | This is optional; it is also valid to simply specify the Cloud_Auth in options map: 8 | 9 | ``` 10 | "logConfiguration": { 11 | "logDriver": "awsfirelens", 12 | "options": { 13 | "Name": "es", 14 | "Port": "9243", 15 | "Cloud_ID": "", 16 | "Cloud_Auth": ":", 17 | "Index": "elastic_firelens", 18 | "tls": "On", 19 | "tls.verify": "Off", 20 | "retry_limit": "2", 21 | "Suppress_Type_Name": "On" 22 | } 23 | }, 24 | ``` 25 | 26 | This is optional; you can also use regular Elasticsearch credentials to connect to Elastic Cloud in the options map: 27 | 28 | ``` 29 | "logConfiguration": { 30 | "logDriver": "awsfirelens", 31 | "options": { 32 | "Name": "es", 33 | "Port": "9243", 34 | "Host": "", 35 | "HTTP_User": "", 36 | "HTTP_Passwd": "", 37 | "Index": "elastic_firelens", 38 | "tls": "On", 39 | "tls.verify": "Off", 40 | "retry_limit": "2", 41 | "Suppress_Type_Name": "On" 42 | } 43 | }, 44 | ``` 45 | 46 | This is optional; if you would like to collect logs from Fargate, just replace the first three lines of example task definition with the following: 47 | 48 | ``` 49 | "family": "firelens-example-elastic", 50 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 51 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 52 | "cpu": "512", 53 | "memory": "1024", 54 | "requiresCompatibilities": [ 55 | "FARGATE" 56 | ], 57 | ``` 58 | 59 | Note that the task and execution roles are mandatory for both Amazon ECS and AWS Fargate scenarios. 60 | -------------------------------------------------------------------------------- /examples/fluent-bit/elastic-cloud/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-elastic", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "name": "log_router", 8 | "image": "amazon/aws-for-fluent-bit:latest", 9 | "essential": true, 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "enable-ecs-log-metadata": "true" 14 | } 15 | }, 16 | "logConfiguration": { 17 | "logDriver": "awslogs", 18 | "options": { 19 | "awslogs-create-group": "true", 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "", 22 | "awslogs-stream-prefix": "firelens" 23 | } 24 | }, 25 | "memoryReservation": 50 26 | }, 27 | { 28 | "name": "example-app", 29 | "image": "nginx", 30 | "portMappings": [ 31 | { 32 | "containerPort": 80 33 | } 34 | ], 35 | "essential": true, 36 | "environment": [], 37 | "logConfiguration": { 38 | "logDriver": "awsfirelens", 39 | "secretOptions": [ 40 | { 41 | "valueFrom": "arn:${Partition}:secretsmanager:${Region}:${Account}:secret:${SecretId}", 42 | "name": "Cloud_Auth" 43 | } 44 | ], 45 | "options": { 46 | "Name": "es", 47 | "Port": "9243", 48 | "Tag_Key tags": "tags", 49 | "Include_Tag_Key": "true", 50 | "Cloud_ID": "", 51 | "Index": "elastic_firelens", 52 | "tls": "On", 53 | "tls.verify": "Off", 54 | "retry_limit": "2", 55 | "Suppress_Type_Name": "On" 56 | } 57 | }, 58 | "memoryReservation": 100 59 | } 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /examples/fluent-bit/enable-debug-logging/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Enabling Debug Logging for Fluent Bit 2 | 3 | Log level can be set in the [Service](https://docs.fluentbit.io/manual/service) section of the Fluent Bit configuration file. This section is not used by FireLens; you can set it yourself using an external configuration file. 4 | 5 | To enable debug logging in the AWS Fluent Bit plugins; set the environment variable `FLB_LOG_LEVEL`. It can be set to `debug`, `info`, and `error`. 6 | -------------------------------------------------------------------------------- /examples/fluent-bit/enable-debug-logging/extra.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | Log_Level debug 3 | -------------------------------------------------------------------------------- /examples/fluent-bit/enable-debug-logging/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /examples/fluent-bit/enable-debug-logging/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-debug", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "environment": [ 11 | { "name": "FLB_LOG_LEVEL", "value": "debug" } 12 | ], 13 | "firelensConfiguration": { 14 | "type": "fluentbit", 15 | "options": { 16 | "config-file-type": "s3", 17 | "config-file-value": "arn:aws:s3:::yourbucket/yourdirectory/extra.conf" 18 | } 19 | }, 20 | "logConfiguration": { 21 | "logDriver": "awslogs", 22 | "options": { 23 | "awslogs-group": "firelens-container", 24 | "awslogs-region": "us-west-2", 25 | "awslogs-create-group": "true", 26 | "awslogs-stream-prefix": "firelens" 27 | } 28 | }, 29 | "memoryReservation": 50 30 | }, 31 | { 32 | "essential": true, 33 | "image": "nginx", 34 | "name": "app", 35 | "logConfiguration": { 36 | "logDriver":"awsfirelens", 37 | "options": { 38 | "Name": "cloudwatch", 39 | "region": "us-west-2", 40 | "log_group_name": "firelens-fluent-bit", 41 | "auto_create_group": "true", 42 | "log_stream_prefix": "from-fluent-bit", 43 | "retry_limit": "2" 44 | } 45 | }, 46 | "memoryReservation": 100 47 | } 48 | ] 49 | } 50 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline-partial-message-mode/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Concatenate Partial/Split Container Logs 2 | 3 | The `partial_message` `mode` for the Multiline Filter is available on `aws-for-fluent-bit` >= 2.24.0 4 | 5 | FireLens uses the [fluentd log driver](https://docs.docker.com/config/containers/logging/fluentd/) in its [implementation](https://aws.amazon.com/blogs/containers/under-the-hood-firelens-for-amazon-ecs-tasks/). Stdout/stderr logs pass from the container runtime to the log driver, to Fluent Bit. The container runtime denotes the stream of bytes from stdout and stderr as a series of log events by splitting the stream on bytes, or at 16KB. When logs are split at 16KB they become partial messages. For the sake of the example, assume the runtime's max buffer size is 4 bytes instead of 16KB; Fluent Bit would get the following series of messages: 6 | 7 | ``` 8 | {"source": "stdout", "log": "Sher", "partial_message": "true", "partial_id": "dc37eb08b4242c41757d4cd995d983d1cdda4589193755a22fcf47a638317da0", "partial_ordinal": "1", "partial_last": "false", "container_id": "a96998303938eab6087a7f8487ca40350f2c252559bc6047569a0b11b936f0f2", "container_name": "/hopeful_taussig"}] 9 | {"partial_last": "false", "container_id": "a96998303938eab6087a7f8487ca40350f2c252559bc6047569a0b11b936f0f2", "container_name": "/hopeful_taussig", "source": "stdout", "log": "lock", "partial_message": "true", "partial_id": "dc37eb08b4242c41757d4cd995d983d1cdda4589193755a22fcf47a638317da0", "partial_ordinal": "2"}] 10 | {"log": " Hol", "partial_message": "true", "partial_id": "dc37eb08b4242c41757d4cd995d983d1cdda4589193755a22fcf47a638317da0", "partial_ordinal": "3", "partial_last": "false", "container_id": "a96998303938eab6087a7f8487ca40350f2c252559bc6047569a0b11b936f0f2", "container_name": "/hopeful_taussig", "source": "stdout"}] 11 | {"container_id": "a96998303938eab6087a7f8487ca40350f2c252559bc6047569a0b11b936f0f2", "container_name": "/hopeful_taussig", "source": "stdout", "log": "mes", "partial_message": "true", "partial_id": "dc37eb08b4242c41757d4cd995d983d1cdda4589193755a22fcf47a638317da0", "partial_ordinal": "4", "partial_last": "true"}] 12 | ``` 13 | 14 | None of these messages were split on a newline, all were split by the max buffer size. Fluent Bit can re-combine these messages so that you get one message: 15 | ``` 16 | {"container_id": "a96998303938eab6087a7f8487ca40350f2c252559bc6047569a0b11b936f0f2", "container_name": "/hopeful_taussig", "source": "stdout", "log": "Sherlock Holmes"}] 17 | ``` 18 | 19 | This can be accomplished with a simple filter definition: 20 | 21 | ``` 22 | [FILTER] 23 | name multiline 24 | match * 25 | multiline.key_content log 26 | # partial_message mode is incompatible with option multiline.parser 27 | mode partial_message 28 | ``` 29 | 30 | The files included in this example allow you to build a custom Fluent Bit image with this config- please see the base [config-file-type](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/config-file-type-file) example to understand how to build an image with a custom config. 31 | 32 | The included `multiline-app` is just a sample app for demonstration purposes. It prints the entire text of a Sherlock Holmes novel to stdout and stderr, which leads to many partial messages. Please note that CloudWatch Logs has a max event size of 256KB and Fluent Bit will truncate single messages that are over this size. This limit cannot be changed. 33 | 34 | *Note*: 35 | - `permissions.json` shows the permissions we should include in the IAM role to send logs to CloudWatch. 36 | - the container runtime buffer size for logs is fixed at 16KB and can not be changed 37 | 38 | 39 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline-partial-message-mode/custom-fluent-bit-with-config/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/aws-observability/aws-for-fluent-bit:latest 2 | 3 | ADD extra.conf /extra.conf 4 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline-partial-message-mode/custom-fluent-bit-with-config/extra.conf: -------------------------------------------------------------------------------- 1 | [FILTER] 2 | name multiline 3 | match * 4 | multiline.key_content log 5 | # partial_message mode is incompatible with option multiline.parser 6 | mode partial_message 7 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline-partial-message-mode/multiline-app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/amazonlinux/amazonlinux:latest 2 | ADD houndofbaskerville.txt /houndofbaskerville.txt 3 | 4 | RUN yum upgrade -y && yum install -y python3 5 | 6 | WORKDIR /usr/local/bin 7 | 8 | COPY main.py . 9 | 10 | CMD ["python3", "main.py"] 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline-partial-message-mode/multiline-app/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from time import sleep 4 | file1 = open('/houndofbaskerville.txt', 'r') 5 | Lines = file1.readlines() 6 | 7 | INFINITE = False 8 | if 'INFINITE' in os.environ: 9 | INFINITE = True 10 | 11 | iterate = True 12 | 13 | while iterate: 14 | iterate = INFINITE 15 | # print the whole text to stdout 16 | count = 0 17 | for line in Lines: 18 | count += 1 19 | print(line.rstrip(), end='') 20 | print("") 21 | print(count) 22 | sleep(5) 23 | 24 | # print the whole text to stderr 25 | count = 0 26 | for line in Lines: 27 | count += 1 28 | print(line.rstrip(), end='', file=sys.stderr) 29 | print("", file=sys.stderr) 30 | print(count) 31 | sleep(5) -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline-partial-message-mode/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline-partial-message-mode/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-multiline", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "111111111111.dkr.ecr.us-west-2.amazonaws.com/custom-fluent-bit-multiline-example:latest", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "file", 14 | "config-file-value": "/extra.conf" 15 | } 16 | }, 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-west-2", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "memoryReservation": 50 27 | }, 28 | { 29 | "essential": true, 30 | "image": "111111111111.dkr.ecr.us-west-2.amazonaws.com/multiline-example-app:latest", 31 | "name": "app", 32 | "logConfiguration": { 33 | "logDriver":"awsfirelens", 34 | "options": { 35 | "Name": "cloudwatch_logs", 36 | "region": "us-west-2", 37 | "log_group_name": "multiline-test/application", 38 | "auto_create_group": "true", 39 | "log_stream_prefix": "multiline-" 40 | } 41 | }, 42 | "memoryReservation": 100 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Concatenate Multiline or Stack trace log messages 2 | 3 | Multiline Filter is available on `aws-for-fluent-bit` >= v2.22.0. It helps to concatenate messages that originally belong to one context but were split across multiple records or log lines. Common examples are stack traces or applications that print logs in multiple lines. More information could be found in [Fluent Bit official doc](https://docs.fluentbit.io/manual/pipeline/filters/multiline-stacktrace). 4 | 5 | With this filter, you are able to use Fluent Bit built-in parses with auto detection and multi format support on: 6 | - go 7 | - python 8 | - ruby 9 | - java 10 | 11 | As those are built-in, you can directly specify them in a field called `multiline.parser` in `[FILTER]` section. 12 | 13 | Let's go through an example shows how to use the multiline filter: 14 | 15 | 1. In order to parse logs, you can either create a parser file for custom stacktrace or choose the built-in parsers. A parser file `parsers_multiline.conf` could be like as below: 16 | ``` 17 | [MULTILINE_PARSER] 18 | name multiline-regex-test 19 | type regex 20 | flush_timeout 1000 21 | # 22 | # Regex rules for multiline parsing 23 | # --------------------------------- 24 | # 25 | # configuration hints: 26 | # 27 | # - first state always has the name: start_state 28 | # - every field in the rule must be inside double quotes 29 | # 30 | # rules | state name | regex pattern | next state 31 | # ------|---------------|-------------------------------------------- 32 | rule "start_state" "/(Dec \d+ \d+\:\d+\:\d+)(.*)/" "cont" 33 | rule "cont" "/^\s+at.*/" "cont" 34 | ``` 35 | 2. Add your own custom config to `extra.conf`. In this config, you need to specify the above parser file in `[SERVICE]` section and have another `[FILTER]` section to add parsers. Here, You can also directly add a built-in parser like `go`. 36 | 3. Build a custom Fluent Bit image using the provided Docker file (which simply copies these two customized files into the AWS for Fluent Bit image) by `docker build .`. You can place this extra config file anywhere in the Docker image *except* `/fluent-bit/etc/fluent-bit.conf`. That config file path is the path used by FireLens. Push this custom image to Amazon ECR with `ecs-cli push`. 37 | 4. Reference the new custom Fluent Bit image ECR repo in the container definition for the FireLens container. In this example we have also included a sample application container, which works with the example configuration and parser. The logs produced by the app will be concatenated by Fluent Bit. 38 | 5. Reference the config file path in the FireLens configuration: 39 | ``` 40 | "firelensConfiguration": { 41 | "type": "fluentbit", 42 | "options": { 43 | "config-file-type": "file", 44 | "config-file-value": "/extra.conf" 45 | } 46 | } 47 | ``` 48 | *Note*: 49 | - `permissions.json` shows the permissions we should include in our IAM to send logs to CloudWatch. 50 | - the path to the parser file in `extra.conf` should be the absolute path in the image. 51 | - If your logs go to files instead of stdout, it is recommended to use Multiline in [Tail](https://docs.fluentbit.io/manual/pipeline/inputs/tail#multiline-support) direclty instead of the filter. 52 | 53 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/custom-fluent-bit-with-parser/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/aws-observability/aws-for-fluent-bit:latest 2 | 3 | ADD parsers_multiline.conf /parsers_multiline.conf 4 | ADD extra.conf /extra.conf 5 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/custom-fluent-bit-with-parser/extra.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | flush 1 3 | log_level info 4 | parsers_file /parsers_multiline.conf 5 | 6 | [FILTER] 7 | name multiline 8 | match * 9 | multiline.key_content log 10 | multiline.parser go, multiline-regex-test -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/custom-fluent-bit-with-parser/parsers_multiline.conf: -------------------------------------------------------------------------------- 1 | [MULTILINE_PARSER] 2 | name multiline-regex-test 3 | type regex 4 | flush_timeout 1000 5 | # 6 | # Regex rules for multiline parsing 7 | # --------------------------------- 8 | # 9 | # configuration hints: 10 | # 11 | # - first state always has the name: start_state 12 | # - every field in the rule must be inside double quotes 13 | # 14 | # rules | state name | regex pattern | next state 15 | # ------|---------------|-------------------------------------------- 16 | rule "start_state" "/(Dec \d+ \d+\:\d+\:\d+)(.*)/" "cont" 17 | rule "cont" "/^\s+at.*/" "cont" 18 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/multiline-app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/amazonlinux/amazonlinux:latest 2 | ADD test.log /test.log 3 | 4 | RUN yum upgrade -y && yum install -y python3 pip3 5 | 6 | WORKDIR /usr/local/bin 7 | 8 | COPY main.py . 9 | 10 | CMD ["python3", "main.py"] 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/multiline-app/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | file1 = open('/test.log', 'r') 4 | Lines = file1.readlines() 5 | 6 | count = 0 7 | 8 | for i in range(10): 9 | print("app running normally...") 10 | time.sleep(1) 11 | 12 | # Strips the newline character 13 | for line in Lines: 14 | count += 1 15 | print(line.rstrip()) 16 | print(count) 17 | print("app terminated.") -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/multiline-app/test.log: -------------------------------------------------------------------------------- 1 | single line... 2 | Dec 14 06:41:08 Exception in thread "main" java.lang.RuntimeException: Something has gone wrong, aborting! 3 | at com.myproject.module.MyProject.badMethod(MyProject.java:22) 4 | at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:18) 5 | at com.myproject.module.MyProject.anotherMethod(MyProject.java:14) 6 | at com.myproject.module.MyProject.someMethod(MyProject.java:10) 7 | at com.myproject.module.MyProject.main(MyProject.java:6) 8 | another line... 9 | panic: my panic 10 | 11 | goroutine 4 [running]: 12 | panic(0x45cb40, 0x47ad70) 13 | /usr/local/go/src/runtime/panic.go:542 +0x46c fp=0xc42003f7b8 sp=0xc42003f710 pc=0x422f7c 14 | main.main.func1(0xc420024120) 15 | foo.go:6 +0x39 fp=0xc42003f7d8 sp=0xc42003f7b8 pc=0x451339 16 | runtime.goexit() 17 | /usr/local/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42003f7e0 sp=0xc42003f7d8 pc=0x44b4d1 18 | created by main.main 19 | foo.go:5 +0x58 20 | 21 | goroutine 1 [chan receive]: 22 | runtime.gopark(0x4739b8, 0xc420024178, 0x46fcd7, 0xc, 0xc420028e17, 0x3) 23 | /usr/local/go/src/runtime/proc.go:280 +0x12c fp=0xc420053e30 sp=0xc420053e00 pc=0x42503c 24 | runtime.goparkunlock(0xc420024178, 0x46fcd7, 0xc, 0x1000f010040c217, 0x3) 25 | /usr/local/go/src/runtime/proc.go:286 +0x5e fp=0xc420053e70 sp=0xc420053e30 pc=0x42512e 26 | runtime.chanrecv(0xc420024120, 0x0, 0xc420053f01, 0x4512d8) 27 | /usr/local/go/src/runtime/chan.go:506 +0x304 fp=0xc420053f20 sp=0xc420053e70 pc=0x4046b4 28 | runtime.chanrecv1(0xc420024120, 0x0) 29 | /usr/local/go/src/runtime/chan.go:388 +0x2b fp=0xc420053f50 sp=0xc420053f20 pc=0x40439b 30 | main.main() 31 | foo.go:9 +0x6f fp=0xc420053f80 sp=0xc420053f50 pc=0x4512ef 32 | runtime.main() 33 | /usr/local/go/src/runtime/proc.go:185 +0x20d fp=0xc420053fe0 sp=0xc420053f80 pc=0x424bad 34 | runtime.goexit() 35 | /usr/local/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc420053fe8 sp=0xc420053fe0 pc=0x44b4d1 36 | 37 | goroutine 2 [force gc (idle)]: 38 | runtime.gopark(0x4739b8, 0x4ad720, 0x47001e, 0xf, 0x14, 0x1) 39 | /usr/local/go/src/runtime/proc.go:280 +0x12c fp=0xc42003e768 sp=0xc42003e738 pc=0x42503c 40 | runtime.goparkunlock(0x4ad720, 0x47001e, 0xf, 0xc420000114, 0x1) 41 | /usr/local/go/src/runtime/proc.go:286 +0x5e fp=0xc42003e7a8 sp=0xc42003e768 pc=0x42512e 42 | runtime.forcegchelper() 43 | /usr/local/go/src/runtime/proc.go:238 +0xcc fp=0xc42003e7e0 sp=0xc42003e7a8 pc=0x424e5c 44 | runtime.goexit() 45 | /usr/local/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42003e7e8 sp=0xc42003e7e0 pc=0x44b4d1 46 | created by runtime.init.4 47 | /usr/local/go/src/runtime/proc.go:227 +0x35 48 | 49 | goroutine 3 [GC sweep wait]: 50 | runtime.gopark(0x4739b8, 0x4ad7e0, 0x46fdd2, 0xd, 0x419914, 0x1) 51 | /usr/local/go/src/runtime/proc.go:280 +0x12c fp=0xc42003ef60 sp=0xc42003ef30 pc=0x42503c 52 | runtime.goparkunlock(0x4ad7e0, 0x46fdd2, 0xd, 0x14, 0x1) 53 | /usr/local/go/src/runtime/proc.go:286 +0x5e fp=0xc42003efa0 sp=0xc42003ef60 pc=0x42512e 54 | runtime.bgsweep(0xc42001e150) 55 | /usr/local/go/src/runtime/mgcsweep.go:52 +0xa3 fp=0xc42003efd8 sp=0xc42003efa0 pc=0x419973 56 | runtime.goexit() 57 | /usr/local/go/src/runtime/asm_amd64.s:2337 +0x1 fp=0xc42003efe0 sp=0xc42003efd8 pc=0x44b4d1 58 | created by runtime.gcenable 59 | /usr/local/go/src/runtime/mgc.go:216 +0x58 60 | one more line, no multiline -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /examples/fluent-bit/filter-multiline/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-multiline", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "111111111111.dkr.ecr.us-west-2.amazonaws.com/custom-fluent-bit-multiline-example:latest", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "file", 14 | "config-file-value": "/extra.conf" 15 | } 16 | }, 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-west-2", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "memoryReservation": 50 27 | }, 28 | { 29 | "essential": true, 30 | "image": "111111111111.dkr.ecr.us-west-2.amazonaws.com/multiline-example-app:latest", 31 | "name": "app", 32 | "logConfiguration": { 33 | "logDriver":"awsfirelens", 34 | "options": { 35 | "Name": "cloudwatch_logs", 36 | "region": "us-west-2", 37 | "log_group_name": "multiline-test/application", 38 | "auto_create_group": "true", 39 | "log_stream_prefix": "multiline-", 40 | "retry_limit": "2" 41 | } 42 | }, 43 | "memoryReservation": 100 44 | } 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /examples/fluent-bit/forward-to-aggregator/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Forward to an external Fluentd or Fluent Bit Log Aggregator 2 | 3 | You can use FireLens to forward logs to an external Fluentd or Fluent Bit host. For more information on log aggregation see [Building a scalable log aggregator with AWS Fargate, Fluentd, and Amazon Kinesis Data Firehose](https://aws.amazon.com/blogs/compute/building-a-scalable-log-solution-aggregator-with-aws-fargate-fluentd-and-amazon-kinesis-data-firehose/). 4 | -------------------------------------------------------------------------------- /examples/fluent-bit/forward-to-aggregator/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-forward", 3 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 4 | "containerDefinitions": [ 5 | { 6 | "essential": true, 7 | "image": "amazon/aws-for-fluent-bit:stable", 8 | "name": "log_router", 9 | "firelensConfiguration": { 10 | "type": "fluentbit" 11 | }, 12 | "logConfiguration": { 13 | "logDriver": "awslogs", 14 | "options": { 15 | "awslogs-group": "firelens-container", 16 | "awslogs-region": "us-west-2", 17 | "awslogs-create-group": "true", 18 | "awslogs-stream-prefix": "firelens" 19 | } 20 | }, 21 | "memoryReservation": 50 22 | }, 23 | { 24 | "essential": true, 25 | "image": "httpd", 26 | "name": "app", 27 | "logConfiguration": { 28 | "logDriver":"awsfirelens", 29 | "options": { 30 | "Name": "forward", 31 | "Host": "fluentdhost", 32 | "Port": "24224", 33 | "Retry_Limit": "2" 34 | } 35 | }, 36 | "memoryReservation": 100 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /examples/fluent-bit/grafana-cloud/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Grafana Loki with Fluent Bit 2 | 3 | For documentation on sending your FireLens monitored log data to Grafana Loki, see: [Fluent Bit loki Plugin Documentation](https://docs.fluentbit.io/manual/v/1.9-pre/pipeline/outputs/loki). Be aware there is a separate Golang output plugin provided by [Grafana](https://grafana.com/docs/loki/latest/clients/fluentbit/) with different configuration options. Here is a [blog](https://calyptia.com/blog/how-to-send-logs-to-loki-using-fluent-bit) on sending logs to Loki using the loki Fluent Bit plugin. The Loki data source provides access to Loki, Grafana’s log aggregation system. 4 | 5 | It should be noted that this example and Grafana docs show `"enable-ecs-log-metadata":"true"` (which is the default). This option tells FireLens to add ECS Task Metadata keys to logs. 6 | 7 | For all configuration parameters for Fluent Bit Loki output plugin, see [Fluent Bit loki Plugin Documentation](https://docs.fluentbit.io/manual/v/1.9-pre/pipeline/outputs/loki) documentation. 8 | 9 | If you are looking for `"bearer_token"` support, please use the docker hub upstream [image](https://hub.docker.com/r/fluent/fluent-bit) and follow the [Fluent Bit loki latest Documentation](https://docs.fluentbit.io/manual/pipeline/outputs/loki). 10 | 11 | AWS recommends that you store sensitive information, like your Datadog API Key using secretOptions as shown in the example Task Definition. This is optional; it is also valid to simply specify the Https password in options map: 12 | 13 | ``` 14 | "logConfiguration": { 15 | "logDriver":"awsfirelens", 16 | "options": { 17 | "Name": "loki", 18 | "Host": "logs-prod-us-west2.grafana.net", 19 | "port": "443", 20 | "tls": "on", 21 | "tls.verify": "on", 22 | "http_user": "user_id", 23 | "http_passwd": "" 24 | } 25 | }, 26 | ``` 27 | -------------------------------------------------------------------------------- /examples/fluent-bit/grafana-cloud/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-grafana-loki", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration":{ 11 | "type":"fluentbit", 12 | "options":{ 13 | "enable-ecs-log-metadata":"true" 14 | } 15 | }, 16 | "memoryReservation": 50 17 | }, 18 | { 19 | "essential": true, 20 | "image": "httpd", 21 | "name": "app", 22 | "logConfiguration": { 23 | "logDriver":"awsfirelens", 24 | "options": { 25 | "Name": "loki", 26 | "Host": "logs-prod-us-west2.grafana.net", 27 | "port": "443", 28 | "tls": "on", 29 | "tls.verify": "on", 30 | "http_user": "user_id" 31 | }, 32 | "secretOptions": [{ 33 | "name": "http_passwd", 34 | "valueFrom": "arn:${Partition}:secretsmanager:${Region}:${Account}:secret:${SecretId}" 35 | }] 36 | }, 37 | "memoryReservation": 100 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /examples/fluent-bit/health-check/monitoring-server-healthcheck.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | HTTP_Server On 3 | HTTP_Listen 0.0.0.0 4 | HTTP_PORT 2020 5 | # https://docs.fluentbit.io/manual/administration/monitoring#health-check-for-fluent-bit 6 | Health_Check On 7 | # customize error and retry thresholds and evaluation period as desired 8 | HC_Errors_Count 5 9 | HC_Retry_Failure_Count 5 10 | HC_Period 5 -------------------------------------------------------------------------------- /examples/fluent-bit/health-check/task-definition-output-metrics-healthcheck.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "file", 14 | "config-file-value": "/fluent-bit/configs/output-metrics-healthcheck.conf" 15 | } 16 | }, 17 | "healthCheck": { 18 | "retries": 2, 19 | "command": [ 20 | "CMD-SHELL", 21 | "curl -f http://127.0.0.1:2020/api/v1/health || exit 1" 22 | ], 23 | "timeout": 5, 24 | "interval": 10, 25 | "startPeriod": 30 26 | }, 27 | "logConfiguration": { 28 | "logDriver": "awslogs", 29 | "options": { 30 | "awslogs-group": "firelens-container", 31 | "awslogs-region": "us-west-2", 32 | "awslogs-create-group": "true", 33 | "awslogs-stream-prefix": "firelens" 34 | } 35 | }, 36 | "memoryReservation": 50 37 | }, 38 | { 39 | "essential": true, 40 | "image": "nginx", 41 | "name": "app", 42 | "logConfiguration": { 43 | "logDriver":"awsfirelens", 44 | "options": { 45 | "Name": "cloudwatch_logs", 46 | "region": "us-west-2", 47 | "log_group_name": "app-stdout-logs", 48 | "auto_create_group": "true", 49 | "log_stream_name": "ecs-", 50 | "retry_limit": "2" 51 | } 52 | }, 53 | "memoryReservation": 100 54 | } 55 | ] 56 | } 57 | -------------------------------------------------------------------------------- /examples/fluent-bit/health-check/task-definition-tcp-healthcheck.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "healthCheck": { 14 | "retries": 2, 15 | "command": [ 16 | "CMD-SHELL", 17 | "echo '{\"health\": \"check\"}' | nc 127.0.0.1 8877 || exit 1" 18 | ], 19 | "timeout": 5, 20 | "interval": 10, 21 | "startPeriod": 30 22 | }, 23 | "logConfiguration": { 24 | "logDriver": "awslogs", 25 | "options": { 26 | "awslogs-group": "firelens-container", 27 | "awslogs-region": "us-west-2", 28 | "awslogs-create-group": "true", 29 | "awslogs-stream-prefix": "firelens" 30 | } 31 | }, 32 | "memoryReservation": 50 33 | }, 34 | { 35 | "essential": true, 36 | "image": "nginx", 37 | "name": "app", 38 | "logConfiguration": { 39 | "logDriver":"awsfirelens", 40 | "options": { 41 | "Name": "cloudwatch_logs", 42 | "region": "us-west-2", 43 | "log_group_name": "app-stdout-logs", 44 | "auto_create_group": "true", 45 | "log_stream_name": "ecs-", 46 | "retry_limit": "2" 47 | } 48 | }, 49 | "memoryReservation": 100 50 | } 51 | ] 52 | } 53 | -------------------------------------------------------------------------------- /examples/fluent-bit/health-check/task-definition-uptime-healthcheck.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "file", 14 | "config-file-value": "/fluent-bit/configs/output-metrics-healthcheck.conf" 15 | } 16 | }, 17 | "healthCheck": { 18 | "retries": 2, 19 | "command": [ 20 | "CMD-SHELL", 21 | "curl -f http://127.0.0.1:2020/api/v1/uptime || exit 1" 22 | ], 23 | "timeout": 5, 24 | "interval": 10, 25 | "startPeriod": 30 26 | }, 27 | "logConfiguration": { 28 | "logDriver": "awslogs", 29 | "options": { 30 | "awslogs-group": "firelens-container", 31 | "awslogs-region": "us-west-2", 32 | "awslogs-create-group": "true", 33 | "awslogs-stream-prefix": "firelens" 34 | } 35 | }, 36 | "memoryReservation": 50 37 | }, 38 | { 39 | "essential": true, 40 | "image": "nginx", 41 | "name": "app", 42 | "logConfiguration": { 43 | "logDriver":"awsfirelens", 44 | "options": { 45 | "Name": "cloudwatch_logs", 46 | "region": "us-west-2", 47 | "log_group_name": "app-stdout-logs", 48 | "auto_create_group": "true", 49 | "log_stream_name": "ecs-", 50 | "retry_limit": "2" 51 | } 52 | }, 53 | "memoryReservation": 100 54 | } 55 | ] 56 | } 57 | -------------------------------------------------------------------------------- /examples/fluent-bit/init-metadata/README.md: -------------------------------------------------------------------------------- 1 | ## FireLens Example: Using ECS Metadata Provided by Init tag 2 | 3 | This example shows you how to use ECS Metadata in configuration. For more information on how to use the init tag feature, please see our [use case guide](https://github.com/aws/aws-for-fluent-bit/blob/mainline/use_cases/init-process-for-fluent-bit/README.md). 4 | 5 | ### Environment Variables set by init tag 6 | 7 | Init sets the following useful env vars: 8 | 9 | ``` 10 | AWS_REGION / ECS_LAUNCH_TYPE / ECS_CLUSTER / ECS_FAMILY 11 | ECS_TASK_ARN / ECS_TASK_ID / ECS_REVISION / ECS_TASK_DEFINITION 12 | ``` 13 | 14 | You can use these to inject metadata values in your config: 15 | 16 | ``` 17 | [FILTER] 18 | Name record_modifier 19 | Match * 20 | Record ecs_task_id ${ECS_TASK_ID} 21 | ``` 22 | 23 | Remember, if you use FireLens and you did not disable `enable-ecs-log-metadata`, then your logs will [already include cluster name, task ARN, and task definition](https://github.com/aws/aws-for-fluent-bit/blob/mainline/troubleshooting/debugging.md#what-will-the-logs-collected-by-fluent-bit-look-like). 24 | 25 | ### Templating Log Group and Stream Name using init metadata 26 | 27 | Here's an example using logConfiguration options to create an output: 28 | 29 | ``` 30 | "logConfiguration": { 31 | "logDriver": "awsfirelens", 32 | "options": { 33 | "Name": "cloudwatch_logs", 34 | "region": "${AWS_REGION}", 35 | "log_group_name": "${ECS_CLUSTER}/application", 36 | "auto_create_group": "true", 37 | "log_stream_name": "${ECS_TASK_ID}", 38 | "retry_limit": "2" 39 | } 40 | }, 41 | ``` 42 | 43 | Here's the same output created directly using a config file: 44 | 45 | ``` 46 | [OUTPUT] 47 | Name cloudwatch_logs 48 | region ${AWS_REGION} 49 | log_group_name ${ECS_CLUSTER}/application 50 | log_stream_name ${ECS_TASK_ID} 51 | auto_create_group true 52 | retry_limit 2 53 | ``` 54 | 55 | As you can see, the init tag makes it easy to template your configuration with ECS metadata. 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /examples/fluent-bit/init-metadata/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "logs:CreateLogGroup", 8 | "logs:PutLogEvents", 9 | "logs:CreateLogStream" 10 | ], 11 | "Resource": "*" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /examples/fluent-bit/init-metadata/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-init-metadata", 3 | "taskRoleArn": "arn:aws:iam::xxxxxxxxxx:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::xxxxxxxxxx:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "public.ecr.aws/aws-observability/aws-for-fluent-bit:init-latest", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens-init" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "your-app-image", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver": "awsfirelens", 30 | "options": { 31 | "Name": "cloudwatch_logs", 32 | "region": "${AWS_REGION}", 33 | "log_group_name": "${ECS_CLUSTER}/application", 34 | "auto_create_group": "true", 35 | "log_stream_name": "${ECS_TASK_ID}", 36 | "retry_limit": "2" 37 | } 38 | }, 39 | "memoryReservation": 100 40 | } 41 | ] 42 | } -------------------------------------------------------------------------------- /examples/fluent-bit/kinesis-firehose/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Kinesis Data Firehose with Fluent Bit 2 | 3 | For documentation on Fluent Bit & Firehose, see: [amazon-kinesis-firehose-for-fluent-bit](https://github.com/aws/amazon-kinesis-firehose-for-fluent-bit) 4 | -------------------------------------------------------------------------------- /examples/fluent-bit/kinesis-firehose/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "firehose:PutRecordBatch" 7 | ], 8 | "Resource": "*" 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/kinesis-firehose/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-firehose", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "httpd", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "firehose", 32 | "region": "us-west-2", 33 | "delivery_stream": "my-stream", 34 | "retry_limit": "2" 35 | } 36 | }, 37 | "memoryReservation": 100 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /examples/fluent-bit/kinesis-stream/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Kinesis Data Stream with Fluent Bit 2 | 3 | For documentation on Fluent Bit & Kinesis data stream, see: [amazon-kinesis-streams-for-fluent-bit](https://github.com/aws/amazon-kinesis-streams-for-fluent-bit) 4 | -------------------------------------------------------------------------------- /examples/fluent-bit/kinesis-stream/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "kinesis:PutRecords" 7 | ], 8 | "Resource": "*" 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/kinesis-stream/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-kinesis-data-stream", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "httpd", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "kinesis_streams", 32 | "region": "us-west-2", 33 | "stream": "my-data-stream", 34 | "retry_limit": "2" 35 | } 36 | }, 37 | "memoryReservation": 100 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /examples/fluent-bit/log-driver-buffer-limit/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Setting awsfirelens log driver buffer limit size 2 | 3 | The `log-driver-buffer-limit` option is now supported on ECS EC2 and ECS Fargate with PV1.4+. This setting tells FireLens to configure the Fluentd Docker Log Driver field [fluentd-buffer-limit](https://docs.docker.com/config/containers/logging/fluentd/#fluentd-buffer-limit). This setting applies to the log driver that FireLens uses [under the hood](https://aws.amazon.com/blogs/containers/under-the-hood-firelens-for-amazon-ecs-tasks/) to capture stdout & stderr container logs and send them to Fluentd or Fluent Bit. 4 | 5 | If the Fluentd buffer limit is insufficient, log loss can occur. The Fluentd log driver in the container runtime will emit the following [error message](https://github.com/fluent/fluent-logger-golang/blob/v1.9.0/fluent/fluent.go#L422) when logs are lost: 6 | ``` 7 | fluent#appendBuffer: Buffer full 8 | ``` 9 | 10 | On EC2, the Docker container runtime logs can be collected from the system journal with `sudo journalctl -fu docker.service`. On Fargate, container runtime logs are not available to customers. The buffer limit setting controls the [maximum size of the pending log record buffer](https://github.com/fluent/fluent-logger-golang/blob/v1.9.0/fluent/fluent.go#L181) in the log driver; configure a large enough size to ensure there is no log loss for your workload. 11 | 12 | FireLens is a container log router for Amazon ECS and AWS Fargate that gives customers extensibility to use the breadth of services at AWS or partner solutions for log analytics and storage. FireLens works with Fluentd and Fluent Bit and makes it easy to use these two popular open source logging projects. 13 | 14 | ![FireLens](https://d2908q01vomqb2.cloudfront.net/fe2ef495a1152561572949784c16bf23abb28057/2019/11/16/Screen-Shot-2019-09-26-at-5.21.35-PM-1024x572.png) 15 | 16 | The diagram above shows how FireLens works. Container standard out logs are sent to the FireLens container over a Unix socket via the [Fluentd Docker Log Driver](https://docs.docker.com/config/containers/logging/fluentd/). 17 | 18 | To set Fluentd or Fluent Bit output plugin configurations in FireLens, we can configure them in the log configuration section of the Task Definition. We specify `logDriver` as `awsfirelens` to use FireLens and provide config details for the Fluentd or Fluent Bit output plugin in `options` field. The option `log-driver-buffer-limit` will specify the limit size for number of events buffered on the memory. It can help to resolve potential log loss issue because high throughput could result in running out of memory for buffer inside of Docker and the log driver must discard messages from the buffer to add new ones if the buffer is full. This will produce tons of error messages in Docker and the lost logs will make it impossible to troubleshoot problems in the application. Increasing the buffer limit size by configuring the option `log-driver-buffer-limit` could be a good approach to avoid this problem. 19 | 20 | By default, the buffer limit will be set to `1MiB`. In order to increase or decrease the size, you can customize the `log-driver-buffer-limit` option in your log configuration: 21 | 22 | ``` 23 | "logConfiguration": { 24 | "logDriver":"awsfirelens", 25 | "options": { 26 | "Name": "cloudwatch", 27 | "region": "us-west-2", 28 | "log_key": "log", 29 | "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", 30 | "auto_create_group": "true", 31 | "log_stream_name": "$(ecs_task_id)", 32 | "log-driver-buffer-limit": "2097152", 33 | "retry_limit": "2" 34 | } 35 | }, 36 | ``` 37 | 38 | The unit for this field is `byte` so above task definition will set the buffer limit size to `2MiB`. 39 | 40 | *Note*: 41 | - Fargate PV1.3 is on deprecation. The feature is only supported after Fargate PV1.4. 42 | - The value for `log-driver-buffer-limit` should be an integer between 0 and 536870912 (`512MiB`). 43 | - The total amount of memory allocated at the task level must be greater than the amount of memory allocated for all containers in addition to the memory buffer limit for the FireLens log driver. 44 | - The total amount of memory buffer specified must be lesser than 536870912 (`512MiB`) when the container `memory` and `memoryReservertion` values aren't specified. More specifically, you can have an app container with `awsfirelens` log driver and option `log-driver-buffer-limit` set to `300MiB`. However, you won't be allowed to run tasks if you have more than two containers with `log-driver-buffer-limit` as `300MiB`(`300MiB` * 2 > `512MiB`). -------------------------------------------------------------------------------- /examples/fluent-bit/log-driver-buffer-limit/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "logs:CreateLogStream", 7 | "logs:CreateLogGroup", 8 | "logs:DescribeLogStreams", 9 | "logs:PutLogEvents" 10 | ], 11 | "Resource": "*" 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /examples/fluent-bit/log-driver-buffer-limit/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-buffer-limit", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "nginx", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "cloudwatch", 32 | "region": "us-west-2", 33 | "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", 34 | "auto_create_group": "true", 35 | "log_stream_name": "$(ecs_task_id)", 36 | "log-driver-buffer-limit": "2097152", 37 | "retry_limit": "2" 38 | } 39 | }, 40 | "memoryReservation": 100 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /examples/fluent-bit/logstash/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Sending Logs to hosted Logstash with Fluent Bit 2 | 3 | You can use Fluent Bit's [HTTP output plugin](https://docs.fluentbit.io/manual/pipeline/outputs/http) to send your container's log to external or aws hosted logstash. 4 | 5 | To know more about Logstash, please refer [here](https://www.elastic.co/logstash) and [here](https://aws.amazon.com/elasticsearch-service/the-elk-stack/logstash/). 6 | 7 | Logstash works with different plugins, for logs to be processed and transformed, you'll have to enable http input plugins. 8 | 9 | An input plugin enables a specific source of events to be read by Logstash. To read more about input plugin, please refer [here](https://www.elastic.co/guide/en/logstash/current/input-plugins.html). 10 | 11 | Logstash configuration example to enable source of events to be read by Logstash, here we have used port 8090 - 12 | 13 | ``` 14 | input { 15 | beats { 16 | port => 5044 17 | client_inactivity_timeout => 500 18 | } 19 | http { 20 | port => 8080 21 | type => "elb_healthcheck" 22 | } 23 | http { 24 | type => 8090 25 | type => "ecs_fluent_bit_logs" 26 | } 27 | } 28 | ``` 29 | 30 | AWS recommands that you store sensitive information (like the URI containing some sensative token) using [secretOptions](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html), as shown in this example [task definition](task-definition.json). However using URI is optional and it is also valid to use URI as part of `options` map (URI is optional as part of http output plugin of Fluent Bit): 31 | 32 | ``` 33 | "logConfiguration": { 34 | "logDriver": "awsfirelens", 35 | "options": { 36 | "Name": "http", 37 | "Host": "api.logstash.fake.domain", 38 | "URI": "/some//tag//", 39 | "Port": "8090", 40 | "Format": "json", 41 | "Retry_Limit": "2" 42 | } 43 | } 44 | ``` -------------------------------------------------------------------------------- /examples/fluent-bit/logstash/sample_logstash_conf: -------------------------------------------------------------------------------- 1 | input { 2 | beats { 3 | port => 5044 4 | client_inactivity_timeout => 500 5 | } 6 | http { 7 | port => 8080 8 | type => "elb_healthcheck" 9 | } 10 | http { 11 | type => 8090 12 | type => "ecs_fluent_bit_logs" 13 | } 14 | } 15 | 16 | filter { 17 | //Filtering Logs Command Here 18 | } 19 | 20 | output { 21 | //Output Logs to Desire Destination 22 | } -------------------------------------------------------------------------------- /examples/fluent-bit/logstash/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family":"firelens-example-logstash", 3 | "taskRoleArn":"arn:aws:iam::XXXXXXXXXXXX:role/serviceTaskRole", 4 | "executionRoleArn":"arn:aws:iam::XXXXXXXXXXXX:role/ecsTaskExecutionRole", 5 | "networkMode":"awsvpc", 6 | "containerDefinitions":[ 7 | { 8 | "essential":true, 9 | "image":"906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:stable", 10 | "name":"log_router", 11 | "firelensConfiguration":{ 12 | "type":"fluentbit" 13 | }, 14 | "memoryReservation":50 15 | }, 16 | { 17 | "essential":true, 18 | "image":"nginx", 19 | "name":"app", 20 | "portMappings":{ 21 | "ContainerPort":443, 22 | "HostPort":443, 23 | "Protcol":"tcp" 24 | }, 25 | "logConfiguration":{ 26 | "logDriver":"awsfirelens", 27 | "options":{ 28 | "Name":"http", 29 | "Host":"api.logstash.fake.domain", 30 | "Port":"8090", 31 | "Format":"json", 32 | "Retry_Limit": "2" 33 | }, 34 | "secretOptions":[ 35 | { 36 | "name":"URI", 37 | "valueFrom":"arn:${Partition}:secretsmanager:${RegionID}:${AccountID}:secret:${SecretId}" 38 | } 39 | ] 40 | }, 41 | "memoryReservation":100 42 | } 43 | ] 44 | } -------------------------------------------------------------------------------- /examples/fluent-bit/multi-config-support/README.md: -------------------------------------------------------------------------------- 1 | ## FireLens Example: Multiple Config support - using the Fluent Bit image with init tag 2 | 3 | This example shows you how to set multiple config files for Fluent Bit on ECS. For more information on how to use the Multi-config support feature, please see our [use case guide](https://github.com/aws/aws-for-fluent-bit/blob/mainline/use_cases/init-process-for-fluent-bit/README.md). 4 | 5 | This example simulates a situation: A ECS Task include two containers, the app container using your own image, and the log router container using Fluent Bit image, which will forward the logs from the App container to CloudWatch. 6 | 7 | **Based on above situation, suppose your app container also generate a log file which path is `/logs/app.log`. we want to use multiple config feature to forward the content of this log file, parse it first, and forward the parsed logs to S3.** 8 | 9 | 10 | 11 | ### Step 1: Create config files locally 12 | 13 | **tail-input.conf** 14 | 15 | ``` 16 | [INPUT] 17 | Name tail 18 | Tag app 19 | Path /logs/app.log 20 | Read_from_Head True 21 | ``` 22 | 23 | **your-filter.conf** 24 | 25 | ``` 26 | [FILTER] 27 | Name parser 28 | Match app 29 | Key_Name data 30 | Parser app_test 31 | ``` 32 | 33 | **your-parser.conf** 34 | 35 | ``` 36 | [PARSER] 37 | Name app_test 38 | Format regex 39 | Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ 40 | ``` 41 | 42 | **dummy-s3-output.conf** 43 | 44 | ``` 45 | [OUTPUT] 46 | Name s3 47 | Match app 48 | bucket your-bucket 49 | region ${AWS_REGION} 50 | total_file_size 1M 51 | upload_timeout 1m 52 | use_put_object On 53 | ``` 54 | 55 | 56 | 57 | **Note:** you can find these config files in the `config-files` directory of this example, please modify them according to the actual situation to match your needs. 58 | 59 | ### Step 2: Upload config files to S3 60 | 61 | * create the S3 bucket `your-bucket` to store config files 62 | * upload above config files to this bucket 63 | * create the S3 bucket `your-result` to receive the forwarded logs 64 | 65 | 66 | 67 | ### Step 3: Create the ECS Task 68 | 69 | * create the ECS Task using provided `task-definition.json`, which using the Fluent Bit image with init tag 70 | * change the `taskRoleArn` and `executionRoleArn` to your own role ARN 71 | * change the `environment` part in the task definition FireLens configuration, copy the ARN of config files and paste it as environment variable's value. The name of environment variable requires to use the prefix `aws_fluent_bit_init_s3_` 72 | 73 | **Note:** you need mount the log file directory in to the log router container so that Fluent Bit can read it. This is shown in the task definition with the volume configuration. An ephemeral volume is created which is mounted at `/logs` in both containers. 74 | For ECS FireLens Log Collection Tutorial, [please see here.](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/ecs-log-collection) 75 | 76 | 77 | 78 | ### Step 4: Run the Task and check the result 79 | 80 | Run the task then go to S3 to check the result in `your-result` bucket. Content of the log file of your app container should have been forwarded to S3. 81 | -------------------------------------------------------------------------------- /examples/fluent-bit/multi-config-support/config-files/s3-output.conf: -------------------------------------------------------------------------------- 1 | [OUTPUT] 2 | Name s3 3 | Match app 4 | bucket your-bucket 5 | region ${AWS_REGION} 6 | total_file_size 1M 7 | upload_timeout 1m 8 | use_put_object On 9 | -------------------------------------------------------------------------------- /examples/fluent-bit/multi-config-support/config-files/tail-input.conf: -------------------------------------------------------------------------------- 1 | [INPUT] 2 | Name tail 3 | Tag app 4 | Path /logs/app.log 5 | Read_from_Head True 6 | -------------------------------------------------------------------------------- /examples/fluent-bit/multi-config-support/config-files/your-filter.conf: -------------------------------------------------------------------------------- 1 | [FILTER] 2 | Name parser 3 | Match app 4 | Key_Name data 5 | Parser app_test 6 | -------------------------------------------------------------------------------- /examples/fluent-bit/multi-config-support/config-files/your-parser.conf: -------------------------------------------------------------------------------- 1 | [PARSER] 2 | Name app_test 3 | Format regex 4 | Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ 5 | -------------------------------------------------------------------------------- /examples/fluent-bit/multi-config-support/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "logs:CreateLogGroup", 8 | "logs:PutLogEvents", 9 | "logs:CreateLogStream", 10 | "logs:DescribeLogStreams", 11 | "s3:PutObject", 12 | "s3:GetObject", 13 | "s3:GetBucketLocation" 14 | ], 15 | "Resource": "*" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /examples/fluent-bit/multi-config-support/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-multi-config-support", 3 | "taskRoleArn": "arn:aws:iam::xxxxxxxxxx:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::xxxxxxxxxx:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "public.ecr.aws/aws-observability/aws-for-fluent-bit:init-latest", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "mountPoints": [ 14 | { 15 | "containerPath": "/logs/", 16 | "sourceVolume": "log" 17 | } 18 | ], 19 | "logConfiguration": { 20 | "logDriver": "awslogs", 21 | "options": { 22 | "awslogs-group": "firelens-container", 23 | "awslogs-region": "us-west-2", 24 | "awslogs-create-group": "true", 25 | "awslogs-stream-prefix": "firelens-init" 26 | } 27 | }, 28 | "environment": [ 29 | { 30 | "name": "aws_fluent_bit_init_s3_1", 31 | "value": "arn:aws:s3:::your-bucket/tail-input.conf" 32 | }, 33 | { 34 | "name": "aws_fluent_bit_init_s3_2", 35 | "value": "arn:aws:s3:::your-bucket/your-filter.conf" 36 | }, 37 | { 38 | "name": "aws_fluent_bit_init_s3_3", 39 | "value": "arn:aws:s3:::your-bucket/your-parser.conf" 40 | }, 41 | { 42 | "name": "aws_fluent_bit_init_s3_4", 43 | "value": "arn:aws:s3:::your-bucket/s3-output.conf" 44 | } 45 | ], 46 | "memoryReservation": 50 47 | }, 48 | { 49 | "essential": true, 50 | "image": "your-app-image", 51 | "name": "app", 52 | "mountPoints": [ 53 | { 54 | "containerPath": "/logs/", 55 | "sourceVolume": "log" 56 | } 57 | ], 58 | "logConfiguration": { 59 | "logDriver": "awsfirelens", 60 | "options": { 61 | "Name": "cloudwatch", 62 | "region": "us-west-2", 63 | "log_group_name": "/aws/ecs/containerinsights/$(ecs_cluster)/application", 64 | "auto_create_group": "true", 65 | "log_stream_name": "$(ecs_task_id)", 66 | "retry_limit": "2" 67 | } 68 | }, 69 | "memoryReservation": 100 70 | } 71 | ], 72 | "volumes": [ 73 | { 74 | "name": "log" 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /examples/fluent-bit/newrelic/README.md: -------------------------------------------------------------------------------- 1 | # New Relic Logs Firelens Example 2 | 3 | For more detailed documentation about configuring Firelens for New Relic logs please refer to: [Enable New Relic Logs for AWS Firelens](https://docs.newrelic.com/docs/logs/new-relic-logs/enable-logs/enable-new-relic-logs-aws-firelens) 4 | 5 | ## Images for Log Router Container 6 | New Relic uses a custom Fluent Bit plugin and has provided custom images for US/EU regions 7 | 8 | | AWS Region | Full Image Name | 9 | |--------------|-------------------------------------------------------------------------------------| 10 | | us-east-1 | 533243300146.dkr.ecr.us-east-1.amazonaws.com/newrelic/logging-firelens-fluentbit | 11 | | us-east-2 | 533243300146.dkr.ecr.us-east-2.amazonaws.com/newrelic/logging-firelens-fluentbit | 12 | | us-west-1 | 533243300146.dkr.ecr.us-east-1.amazonaws.com/newrelic/logging-firelens-fluentbit | 13 | | us-west-2 | 533243300146.dkr.ecr.us-west-2.amazonaws.com/newrelic/logging-firelens-fluentbit | 14 | | ca-cental-1 | 533243300146.dkr.ecr.ca-central-1.amazonaws.com/newrelic/logging-firelens-fluentbit | 15 | | eu-central-1 | 533243300146.dkr.ecr.eu-central-1.amazonaws.com/newrelic/logging-firelens-fluentbit | 16 | | eu-west-1 | 533243300146.dkr.ecr.eu-west-1.amazonaws.com/newrelic/logging-firelens-fluentbit | 17 | | eu-west-2 | 533243300146.dkr.ecr.eu-west-2.amazonaws.com/newrelic/logging-firelens-fluentbit | 18 | | eu-west-3 | 533243300146.dkr.ecr.eu-west-3.amazonaws.com/newrelic/logging-firelens-fluentbit | 19 | | eu-north-1 | 533243300146.dkr.ecr.eu-north-1.amazonaws.com/newrelic/logging-firelens-fluentbit | 20 | 21 | ## Application Container logConfiguration 22 | AWS recommends that you store your New Relic Insights Insert Key with the [AWS Secrets Manager](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/specifying-sensitive-data.html) as shown below: 23 | 24 | ``` 25 | "logConfiguration": { 26 | "logDriver":"awsfirelens", 27 | "options": { 28 | "Name": "newrelic" 29 | }, 30 | "secretOptions": [{ 31 | "name": "apiKey", 32 | "valueFrom": "arn:aws:secretsmanager:region:aws_account_id:secret:secret_name-AbCdEf" 33 | }] 34 | } 35 | 36 | ``` 37 | 38 | You can also alternatively add your New Relic Insights Insert Key directly into the logConfiguration block as shown below: 39 | ``` 40 | "logConfiguration": { 41 | "logDriver":"awsfirelens", 42 | "options": { 43 | "Name": "newrelic" 44 | “apiKey”: “[YOUR_INSIGHTS_INSERT_KEY]” 45 | } 46 | } 47 | 48 | ``` -------------------------------------------------------------------------------- /examples/fluent-bit/newrelic/example-task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "newrelic-firelens-example", 3 | "networkMode": "awsvpc", 4 | "requiresCompatibilities": [ 5 | "FARGATE" 6 | ], 7 | "containerDefinitions": [ 8 | { 9 | "essential": true, 10 | "image": "533243300146.dkr.ecr.us-east-2.amazonaws.com/newrelic/logging-firelens-fluentbit", 11 | "name": "log_router", 12 | "firelensConfiguration": { 13 | "type": "fluentbit", 14 | "options": { 15 | "enable-ecs-log-metadata": "true" 16 | } 17 | } 18 | }, 19 | { 20 | "essential": true, 21 | "name": "webserver", 22 | "image": "nginx", 23 | "cpu": 512, 24 | "memoryReservation": 1024, 25 | "portMappings": [ 26 | { 27 | "containerPort": 5000 28 | } 29 | ], 30 | "logConfiguration": { 31 | "logDriver": "awsfirelens", 32 | "options": { 33 | "Name": "newrelic", 34 | "Retry_Limit": "2" 35 | }, 36 | "secretOptions": [ 37 | { 38 | "name": "apiKey", 39 | "valueFrom": "arn:aws:secretsmanager:region:aws_account_id:secret:secret_name-AbCdEf" 40 | } 41 | ] 42 | } 43 | } 44 | ], 45 | "executionRoleArn": "arn:aws:iam::aws_account_id:role/ecsTaskExecutionRole", 46 | "taskRoleArn": "arn:aws:iam::aws_account_id:role/ecsTaskExecutionRole", 47 | "cpu": "1 vcpu", 48 | "memory": "2 gb" 49 | } 50 | -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/filesystem.conf: -------------------------------------------------------------------------------- 1 | # Example for storage.type filesystem 2 | 3 | # Example for default storage.type memory 4 | 5 | [SERVICE] 6 | Grace 30 7 | storage.path /var/log/flb-storage 8 | # Total Max Memory Usage <= 2 * # of input definitions * storage.max_chunks_up * 2 MB per chunk 9 | # 2 MB internal chunk size is not configurable 10 | # So container memory reservation should be AT LEAST 100 MB 11 | storage.max_chunks_up 25 12 | # Recommended: enable metrics interface for plugins and storage 13 | # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/send-fb-internal-metrics-to-cw 14 | HTTP_Server On 15 | HTTP_Listen 0.0.0.0 16 | HTTP_PORT 2020 17 | # enable storage metrics 18 | storage.metrics On 19 | 20 | [INPUT] 21 | Name tcp 22 | Listen 0.0.0.0 23 | Port 5170 24 | Chunk_Size 32 25 | # this number of kilobytes is the max size of single log message that can be accepted 26 | Buffer_Size 64 27 | Format none 28 | Tag tcp-logs 29 | # filesystem buffer for logs collected by this input 30 | storage.type filesystem 31 | # pause the input when the [SERVICE] storage.max_chunks_up memory limit is hit 32 | storage.pause_on_chunks_overlimit Off # default/recommended value 33 | 34 | [OUTPUT] 35 | Name cloudwatch_logs 36 | Match * 37 | region us-east-1 38 | log_group_name firelens-tutorial 39 | log_stream_name /logs/${HOSTNAME} 40 | auto_create_group true 41 | retry_limit 2 42 | # limit the filesystem storage that this output can use 43 | storage.total_limit_size 1G 44 | # consider configuring more than 1 worker, up to the numer of cores on your host 45 | # to accommodate higher throughput logging. Only 2.32+ supports multi-worker 46 | workers 1 -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/firelens-full-filesystem-example/fluent-bit-image/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/aws-observability/aws-for-fluent-bit:stable 2 | 3 | ADD extra.conf /fluent-bit/alt/fluent-bit.conf 4 | CMD echo -n "AWS for Fluent Bit Container Image Version " && \ 5 | cat /AWS_FOR_FLUENT_BIT_VERSION && echo "" && \ 6 | exec /fluent-bit/bin/fluent-bit -e /fluent-bit/firehose.so -e /fluent-bit/cloudwatch.so -e /fluent-bit/kinesis.so -c /fluent-bit/alt/fluent-bit.conf -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/firelens-full-filesystem-example/fluent-bit-image/extra.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | Grace 30 3 | Flush 1 4 | # TODO: must be write-able path. 5 | # https://github.com/aws/aws-for-fluent-bit/blob/mainline/troubleshooting/debugging.md#storagepath-cannot-initialize-root-path 6 | storage.path /var/log/flb-storage 7 | # Total Max Memory Usage <= 2 * # of input definitions * storage.max_chunks_up * 2 MB per chunk 8 | # 2 MB internal chunk size is not configurable 9 | # So container memory reservation should be AT LEAST 100 MB 10 | # TODO: Update limit as needed 11 | storage.max_chunks_up 25 12 | # Recommended: enable metrics interface for plugins and storage 13 | # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/send-fb-internal-metrics-to-cw 14 | HTTP_Server On 15 | HTTP_Listen 0.0.0.0 16 | HTTP_PORT 2020 17 | # enable storage metrics 18 | storage.metrics On 19 | 20 | # This is the required input to recieve container stdout & stderr logs 21 | # with FireLens 22 | [INPUT] 23 | Name forward 24 | unix_path /var/run/fluent.sock 25 | # filesystem buffer for logs collected by this input 26 | storage.type filesystem 27 | # pause the input when the [SERVICE] storage.max_chunks_up memory limit is hit 28 | storage.pause_on_chunks_overlimit Off # default/recommended value 29 | 30 | # This input is part of the FireLens generated config but is not used by default 31 | # If your app code does not send logs over fluent forward to 24224, you can remove this 32 | [INPUT] 33 | Name forward 34 | Listen 0.0.0.0 35 | Port 24224 36 | # filesystem buffer for logs collected by this input 37 | storage.type filesystem 38 | # pause the input when the [SERVICE] storage.max_chunks_up memory limit is hit 39 | storage.pause_on_chunks_overlimit Off # default/recommended value 40 | 41 | # TODO: update output(s) with your desired settings 42 | [OUTPUT] 43 | Name cloudwatch_logs 44 | # TODO: https://github.com/aws/aws-for-fluent-bit/blob/mainline/troubleshooting/debugging.md#firelens-tag-and-match-pattern-and-generated-config 45 | Match * 46 | region us-east-1 47 | log_group_name firelens-tutorial 48 | log_stream_name /logs/${HOSTNAME} 49 | auto_create_group true 50 | retry_limit 2 51 | # limit the filesystem storage that this output can use 52 | storage.total_limit_size 1G 53 | # consider configuring more than 1 worker, up to the numer of cores on your host 54 | # to accommodate higher throughput logging. Only 2.32+ supports multi-worker 55 | workers 1 -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/firelens-full-filesystem-example/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch_logs", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "XXXXXXXXXXXX.dkr.ecr.us-east-1.amazonaws.com/customized-flb-with-entrypoint-overridden:latest", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options":{ 13 | "enable-ecs-log-metadata":"false" 14 | } 15 | }, 16 | "logConfiguration": { 17 | "logDriver": "awslogs", 18 | "options": { 19 | "awslogs-group": "firelens-container", 20 | "awslogs-region": "us-west-2", 21 | "awslogs-create-group": "true", 22 | "awslogs-stream-prefix": "firelens" 23 | } 24 | }, 25 | "memoryReservation": 50 26 | }, 27 | { 28 | "essential": true, 29 | "image": "nginx", 30 | "name": "app", 31 | "logConfiguration": { 32 | "logDriver":"awsfirelens" 33 | }, 34 | "memoryReservation": 100 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/firelens-full-memory-example/fluent-bit-image/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/aws-observability/aws-for-fluent-bit:stable 2 | ADD extra.conf /fluent-bit/alt/fluent-bit.conf 3 | CMD echo -n "AWS for Fluent Bit Container Image Version " && \ 4 | cat /AWS_FOR_FLUENT_BIT_VERSION && echo "" && \ 5 | exec /fluent-bit/bin/fluent-bit -e /fluent-bit/firehose.so -e /fluent-bit/cloudwatch.so -e /fluent-bit/kinesis.so -c /fluent-bit/alt/fluent-bit.conf -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/firelens-full-memory-example/fluent-bit-image/extra.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | Grace 30 3 | Flush 1 4 | # Recommended: enable metrics interface for plugins and storage 5 | # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/send-fb-internal-metrics-to-cw 6 | HTTP_Server On 7 | HTTP_Listen 0.0.0.0 8 | HTTP_PORT 2020 9 | # enable storage metrics 10 | storage.metrics On 11 | 12 | # This is the required input to recieve container stdout & stderr logs 13 | # with FireLens 14 | [INPUT] 15 | Name forward 16 | unix_path /var/run/fluent.sock 17 | # default memory buffer only for logs collected by this input 18 | storage.type memory 19 | # input will stop using memory and pause ingestion if buffer reaches 50 MB 20 | # Total Max Memory Usage <= 2 * SUM(Each input Mem_Buf_Limit) 21 | # So for this example container memory reservation must be 22 | # AT LEAST 100MB 23 | Mem_Buf_Limit 50MB 24 | 25 | 26 | # This input is part of the FireLens generated config but is not used by default 27 | # If your app code does not send logs over fluent forward to 24224, you can remove this 28 | [INPUT] 29 | Name forward 30 | Listen 0.0.0.0 31 | Port 24224 32 | # default memory buffer only for logs collected by this input 33 | storage.type memory 34 | # input will stop using memory and pause ingestion if buffer reaches 50 MB 35 | # Total Max Memory Usage <= 2 * SUM(Each input Mem_Buf_Limit) 36 | # So for this example container memory reservation must be 37 | # AT LEAST 100MB 38 | Mem_Buf_Limit 50MB 39 | 40 | # TODO: update output(s) with your desired settings 41 | [OUTPUT] 42 | Name cloudwatch_logs 43 | # TODO: https://github.com/aws/aws-for-fluent-bit/blob/mainline/troubleshooting/debugging.md#firelens-tag-and-match-pattern-and-generated-config 44 | Match * 45 | region us-east-1 46 | log_group_name firelens-tutorial 47 | log_stream_name /logs/${HOSTNAME} 48 | auto_create_group true 49 | retry_limit 2 50 | # consider configuring more than 1 worker, up to the numer of cores on your host 51 | # to accommodate higher throughput logging. Only 2.32+ supports multi-worker 52 | workers 1 -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/firelens-full-memory-example/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-cloudwatch_logs", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "XXXXXXXXXXXX.dkr.ecr.us-east-1.amazonaws.com/customized-flb-with-entrypoint-overridden:latest", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options":{ 13 | "enable-ecs-log-metadata":"false" 14 | } 15 | }, 16 | "logConfiguration": { 17 | "logDriver": "awslogs", 18 | "options": { 19 | "awslogs-group": "firelens-container", 20 | "awslogs-region": "us-west-2", 21 | "awslogs-create-group": "true", 22 | "awslogs-stream-prefix": "firelens" 23 | } 24 | }, 25 | "memoryReservation": 50 26 | }, 27 | { 28 | "essential": true, 29 | "image": "nginx", 30 | "name": "app", 31 | "logConfiguration": { 32 | "logDriver":"awsfirelens" 33 | }, 34 | "memoryReservation": 100 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /examples/fluent-bit/oomkill-prevention/memory.conf: -------------------------------------------------------------------------------- 1 | # Example for default storage.type memory 2 | 3 | [SERVICE] 4 | Grace 30 5 | 6 | [INPUT] 7 | Name tcp 8 | Listen 0.0.0.0 9 | Port 5170 10 | Chunk_Size 32 11 | # this number of kilobytes is the max size of single log message that can be accepted 12 | Buffer_Size 64 13 | Format none 14 | Tag tcp-logs 15 | # input will stop using memory and pause ingestion if buffer reaches 50 MB 16 | # Total Max Memory Usage <= 2 * SUM(Each input Mem_Buf_Limit) 17 | # So for this example container memory reservation must be 18 | # AT LEAST 100MB 19 | Mem_Buf_Limit 50MB 20 | 21 | # from https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/ecs-log-collection 22 | # https://github.com/aws/amazon-cloudwatch-logs-for-fluent-bit#templating-log-group-and-stream-names 23 | [OUTPUT] 24 | Name cloudwatch 25 | Match * 26 | region us-east-1 27 | log_group_name firelens-tutorial-$(ecs_cluster) 28 | log_stream_name /logs/$(ec2_instance_id)-$(ecs_task_id) 29 | auto_create_group true 30 | retry_limit 2 -------------------------------------------------------------------------------- /examples/fluent-bit/parse-common-log-formats/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Parse common log formats 2 | 3 | Often, your application will output logs in a common format (ex: apache, nginx), and you will want to turn them into JSON. 4 | 5 | For example, an nginx log line like the following: 6 | ``` 7 | 172.17.0.1 - - [03/Oct/2019:00:06:20 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.54.0" "-" 8 | ``` 9 | 10 | Could look like the following when it is sent to your log destination: 11 | ``` 12 | { 13 | "remote": "172.17.0.1", 14 | "host": "-", 15 | "user": "-", 16 | "method": "GET", 17 | "path": "/", 18 | "code": "200", 19 | "size": "612", 20 | "referer": "-", 21 | "agent": "curl/7.54.0" 22 | } 23 | ``` 24 | 25 | In order to parse logs, you will need a [parsers file](https://docs.fluentbit.io/manual/parser), which is specified in the [service section](https://docs.fluentbit.io/manual/service): 26 | ``` 27 | [SERVICE] 28 | Parsers_File /fluent-bit/parsers/parsers.conf 29 | ``` 30 | 31 | The `amazon/aws-for-fluent-bit` image contains a number of parsers files under `/fluent-bit/parsers/`. These parsers are copied directly from the official Fluent Bit Docker image. See their [repository for the parsers](https://github.com/fluent/fluent-bit-docker-image/tree/1.3). 32 | 33 | You may need to create your own custom parser. To do this, you will need to create a custom image with the file. 34 | 35 | Example Dockerfile: 36 | ``` 37 | FROM amazon/aws-for-fluent-bit:latest 38 | ADD custom_parser.conf /fluent-bit/parsers/custom_parser.conf 39 | ``` 40 | 41 | Service Section in external Fluent Bit config file: 42 | ``` 43 | [SERVICE] 44 | Parsers_File /fluent-bit/parsers/custom_parser.conf 45 | ``` 46 | 47 | One final note, if you do not specify `Reserve_Data True`, then the ECS Log metadata fields will be removed from your log events. This is because without that field, Fluent Bit will produce a final log event which consists solely of the JSON that resulted from parsing the `log` key (which contains the actual log line from your application container). 48 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-common-log-formats/parse-apache.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | Log_Level info 3 | Parsers_File /fluent-bit/parsers/parsers.conf 4 | 5 | [FILTER] 6 | Name parser 7 | Match ** 8 | Parser apache 9 | Key_Name log 10 | Reserve_Data True 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-common-log-formats/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "firehose:PutRecordBatch" 7 | ], 8 | "Resource": "*" 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-common-log-formats/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-json", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "s3", 14 | "config-file-value": "arn:aws:s3:::yourbucket/yourdirectory/extra.conf" 15 | } 16 | }, 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-west-2", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "memoryReservation": 50 27 | }, 28 | { 29 | "essential": true, 30 | "image": "httpd", 31 | "name": "app", 32 | "logConfiguration": { 33 | "logDriver":"awsfirelens", 34 | "options": { 35 | "Name": "firehose", 36 | "region": "us-west-2", 37 | "delivery_stream": "my-stream", 38 | "retry_limit": "2" 39 | } 40 | }, 41 | "memoryReservation": 100 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-envoy-app-mesh/README.md: -------------------------------------------------------------------------------- 1 | ## FireLens Example: Parse Envoy Access Logs from AWS App Mesh 2 | 3 | In this example, we will showcase how to parse the Envoy access logs for AWS App Mesh with FireLens. Some of the Fluent Bit concepts around custom configuration are covered in the [parse-common-log-formats](/examples/fluent-bit/parse-common-log-formats/README.md) example. This example builds on those concepts to showcase how to wire up a custom parser with FireLens and AWS App Mesh. 4 | 5 | #### Sample application 6 | 7 | In order to demonstrate a microservices application running in a service mesh, this example uses the [Color App](https://github.com/aws/aws-app-mesh-examples/tree/master/examples/apps/colorapp) as the example application. 8 | 9 | #### AWS App Mesh Virtual Node Logging 10 | 11 | Once you’ve created your environment you need to turn on access logging for Envoy in App Mesh. When you create your virtual nodes, you have the option to configure Envoy access logs. Here is an example from the console: 12 | 13 | ![virtual-node-logging](static/images/virtual-node-logging.png) 14 | 15 | Once you’ve done this for one of the virtual nodes like colorteller-black you would see something like this in your access logs. 16 | 17 | ``` 18 | {"log":"[2020-01-23T16:32:40.781Z] \"GET / HTTP/1.1\" 200 - 0 5 0 0 \"-\" \"Go-http-client/1.1\" \"0ed75cb8-a563-9ca3-8ff0-2d8eab307e3e\" \"colorteller.appmesh-demo:9080\" \"127.0.0.1:9080\"\n","stream":"stdout","time":"2020-01-23T16:32:49.400311038Z"} 19 | ``` 20 | 21 | #### Parsing the Envoy Access Logs 22 | 23 | In order to parse the Envoy logs, you need to write a custom parser as shown here: 24 | 25 | ``` 26 | [PARSER] 27 | Name envoy 28 | Format regex 29 | Regex ^\[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)? (?\S+)" (?[^ ]*) (?[^ ]*) (?[^ ]*) (?[^ ]*) (?[^ ]*) (?[^ ]*) "(?[^ ]*)" "(?[^\"]*)" "(?[^\"]*)" "(?[^ ]*)" "(?[^ ]*)" 30 | ``` 31 | 32 | You would then need to create your own custom Docker image of `amazon/aws-for-fluent-bit` using this [Dockerfile](aws-for-fluent-bit-custom-envoy/Dockerfile) and upload it to [Amazon Elastic Container Registry](https://aws.amazon.com/ecr/). Once uploaded, you would reference your custom Fluent Bit image in your task definitions along with the FireLens specific values as highlighted below. 33 | 34 | ### Wire up the Task Definition 35 | 36 | The Color App example creates multiple task definitions for each color in the mesh and the example [appmesh-firelens-colorteller-black-ecs-task-def.json](appmesh-firelens-colorteller-black-ecs-task-def.json) task definition highlights how to wire up your custom Fluent Bit image with Envoy. 37 | 38 | Once you've registered the task definition and updated the service to use the latest version you can then navigate to your CloudWatch log group to view the parsed Envoy logs in JSON like the example below: 39 | 40 | ``` 41 | { 42 | "authority": "colorteller.appmesh-demo:9080", 43 | "bytes_received": "0", 44 | "bytes_sent": "6", 45 | "code": "200", 46 | "container_id": "32561e17b9b943cc6a07d8db68d2d0c921fe0e9daafa9e4c7d402fc36eaf3196", 47 | "container_name": "/ecs-appmesh-firelens-6-envoy-d4b2bcf39bd698b9a101", 48 | "duration": "0", 49 | "ecs_cluster": "arn:aws:ecs:us-east-1:012345678910:cluster/appmesh-firelens", 50 | "ecs_task_arn": "arn:aws:ecs:us-east-1:012345678910:task/b69367b1-d558-4116-9b9f-18dfcae657d1", 51 | "ecs_task_definition": "appmesh-firelens-colorteller-black:6", 52 | "method": "GET", 53 | "path": "/", 54 | "protocol": "HTTP/1.1", 55 | "request_id": "3a1957c3-3d47-9259-bdc6-f88ebc4b3da7", 56 | "response_flags": "-", 57 | "source": "stdout", 58 | "start_time": "2020-02-03T19:03:22.305Z", 59 | "upstream_host": "127.0.0.1:9080", 60 | "user_agent": "Go-http-client/1.1", 61 | "x_envoy_upstream_service_time": "0", 62 | "x_forwarded_for": "-" 63 | } 64 | 65 | ``` 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-envoy-app-mesh/appmesh-firelens-colorteller-black-ecs-task-def.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "appmesh-firelens-colorteller-black", 3 | "proxyConfiguration": { 4 | "type": "APPMESH", 5 | "containerName": "envoy", 6 | "properties": [{ 7 | "name": "IgnoredUID", 8 | "value": "1337" 9 | }, 10 | { 11 | "name": "ProxyIngressPort", 12 | "value": "15000" 13 | }, 14 | { 15 | "name": "ProxyEgressPort", 16 | "value": "15001" 17 | }, 18 | { 19 | "name": "AppPorts", 20 | "value": "9080" 21 | }, 22 | { 23 | "name": "EgressIgnoredIPs", 24 | "value": "169.254.170.2,169.254.169.254" 25 | } 26 | ] 27 | }, 28 | "containerDefinitions": [ 29 | { 30 | "essential": true, 31 | "image": "012345678910.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit-custom-envoy:latest", 32 | "name": "log_router", 33 | "firelensConfiguration": { 34 | "type": "fluentbit", 35 | "options": { 36 | "enable-ecs-log-metadata": "true", 37 | "config-file-type": "file", 38 | "config-file-value": "/fluent-bit/conf/parse_envoy.conf" 39 | } 40 | }, 41 | "logConfiguration": { 42 | "logDriver": "awslogs", 43 | "options": { 44 | "awslogs-group": "appmesh-firelens", 45 | "awslogs-region": "us-east-1", 46 | "awslogs-stream-prefix": "fluentbit" 47 | } 48 | }, 49 | "memoryReservation": 50 50 | }, 51 | { 52 | "name": "app", 53 | "image": "012345678910.dkr.ecr.us-east-1.amazonaws.com/colorteller:latest", 54 | "portMappings": [{ 55 | "containerPort": 9080, 56 | "hostPort": 9080, 57 | "protocol": "tcp" 58 | }], 59 | "environment": [{ 60 | "name": "COLOR", 61 | "value": "black" 62 | }, 63 | { 64 | "name": "SERVER_PORT", 65 | "value": "9080" 66 | } 67 | ], 68 | "essential": true, 69 | "dependsOn": [{ 70 | "containerName": "envoy", 71 | "condition": "HEALTHY" 72 | }] 73 | }, 74 | { 75 | "name": "envoy", 76 | "image": "840364872350.dkr.ecr.us-east-1.amazonaws.com/aws-appmesh-envoy:v1.12.2.1-prod", 77 | "user": "1337", 78 | "essential": true, 79 | "ulimits": [{ 80 | "name": "nofile", 81 | "hardLimit": 15000, 82 | "softLimit": 15000 83 | }], 84 | "portMappings": [{ 85 | "containerPort": 9901, 86 | "hostPort": 9901, 87 | "protocol": "tcp" 88 | }, 89 | { 90 | "containerPort": 15000, 91 | "hostPort": 15000, 92 | "protocol": "tcp" 93 | }, 94 | { 95 | "containerPort": 15001, 96 | "hostPort": 15001, 97 | "protocol": "tcp" 98 | } 99 | ], 100 | "environment": [{ 101 | "name": "APPMESH_VIRTUAL_NODE_NAME", 102 | "value": "mesh/color-mesh/virtualNode/colorteller-black-appmesh-demo" 103 | }, 104 | { 105 | "name": "ENVOY_LOG_LEVEL", 106 | "value": "info" 107 | } 108 | ], 109 | "logConfiguration": { 110 | "logDriver": "awsfirelens", 111 | "options": { 112 | "Name": "cloudwatch", 113 | "region": "us-east-1", 114 | "log_group_name": "appmesh-firelens", 115 | "auto_create_group": "true", 116 | "log_stream_prefix": "envoy-black-", 117 | "retry_limit": "2" 118 | } 119 | }, 120 | "healthCheck": { 121 | "command": [ 122 | "CMD-SHELL", 123 | "curl -s http://localhost:9901/server_info | grep state | grep -q LIVE" 124 | ], 125 | "interval": 5, 126 | "timeout": 2, 127 | "retries": 3 128 | } 129 | } 130 | ], 131 | "taskRoleArn": "arn:aws:iam::012345678910:role/ecs_task_role", 132 | "executionRoleArn": "arn:aws:iam::012345678910:role/ecsExecutionRole", 133 | "requiresCompatibilities": [ 134 | "FARGATE" 135 | ], 136 | "networkMode": "awsvpc", 137 | "cpu": "1024", 138 | "memory": "2048" 139 | } -------------------------------------------------------------------------------- /examples/fluent-bit/parse-envoy-app-mesh/aws-for-fluent-bit-custom-envoy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazon/aws-for-fluent-bit:latest 2 | 3 | ADD conf/* /fluent-bit/conf/ 4 | ADD parsers/* /fluent-bit/parsers/ 5 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-envoy-app-mesh/aws-for-fluent-bit-custom-envoy/conf/parse_envoy.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | log_level info 3 | Parsers_File /fluent-bit/parsers/envoy_parser.conf 4 | 5 | [Filter] 6 | Name parser 7 | Match ** 8 | Key_Name log 9 | Parser envoy 10 | Reserve_Data True 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-envoy-app-mesh/aws-for-fluent-bit-custom-envoy/parsers/envoy_parser.conf: -------------------------------------------------------------------------------- 1 | [PARSER] 2 | Name envoy 3 | Format regex 4 | Regex ^\[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)? (?\S+)" (?[^ ]*) (?[^ ]*) (?[^ ]*) (?[^ ]*) (?[^ ]*) (?[^ ]*) "(?[^ ]*)" "(?[^\"]*)" "(?[^\"]*)" "(?[^ ]*)" "(?[^ ]*)" 5 | 6 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-envoy-app-mesh/static/images/virtual-node-logging.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-ecs-firelens-examples/852c3a7f3dbd0d29ca1faa9a9b9293715cdff905/examples/fluent-bit/parse-envoy-app-mesh/static/images/virtual-node-logging.png -------------------------------------------------------------------------------- /examples/fluent-bit/parse-json/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Parsing container stdout logs that are serialized JSON 2 | 3 | As of [AWS for Fluent Bit](https://hub.docker.com/r/amazon/aws-for-fluent-bit) version 1.3, an external configuration file is not needed to parse JSON. The config shown in [extra.conf](extra.conf) is included in the image. Simply reference it in your FireLens configuration: 4 | 5 | ``` 6 | "firelensConfiguration": { 7 | "type": "fluentbit", 8 | "options": { 9 | "config-file-type": "file", 10 | "config-file-value": "/fluent-bit/configs/parse-json.conf" 11 | } 12 | }, 13 | ``` 14 | 15 | This Fluent Bit config file will parse any logs that are JSON. 16 | For example, if the logs at your destination looked like this without JSON parsing: 17 | 18 | ``` 19 | { 20 | "source": "stdout", 21 | "log": "{\"requestID\": \"b5d716fca19a4252ad90e7b8ec7cc8d2\", \"requestInfo\": {\"ipAddress\": \"204.16.5.19\", \"path\": \"/activate\", \"user\": \"TheDoctor\"}}", 22 | "container_id": "e54cccfac2b87417f71877907f67879068420042828067ae0867e60a63529d35", 23 | "container_name": "/ecs-demo-6-container2-a4eafbb3d4c7f1e16e00" 24 | "ecs_cluster": "mycluster", 25 | "ecs_task_arn": "arn:aws:ecs:us-east-2:01234567891011:task/mycluster/3de392df-6bfa-470b-97ed-aa6f482cd7a6", 26 | "ecs_task_definition": "demo:7" 27 | "ec2_instance_id": "i-06bc83dbc2ac2fdf8" 28 | } 29 | ``` 30 | 31 | Then with JSON parsing they'll look like this: 32 | 33 | ``` 34 | { 35 | "source": "stdout", 36 | "container_id": "e54cccfac2b87417f71877907f67879068420042828067ae0867e60a63529d35", 37 | "container_name": "/ecs-demo-6-container2-a4eafbb3d4c7f1e16e00" 38 | "ecs_cluster": "mycluster", 39 | "ecs_task_arn": "arn:aws:ecs:us-east-2:01234567891011:task/mycluster/3de392df-6bfa-470b-97ed-aa6f482cd7a6", 40 | "ecs_task_definition": "demo:7" 41 | "ec2_instance_id": "i-06bc83dbc2ac2fdf8" 42 | "requestID": "b5d716fca19a4252ad90e7b8ec7cc8d2", 43 | "requestInfo": { 44 | "ipAddress": "204.16.5.19", 45 | "path": "/activate", 46 | "user": "TheDoctor" 47 | } 48 | } 49 | ``` 50 | 51 | As you can see, the serialized JSON is expanded into top level fields in the final JSON. For more information on JSON parsing see, the [Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/parser). 52 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-json/extra.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | Parsers_File /fluent-bit/parsers/parsers.conf 3 | Flush 1 4 | Grace 30 5 | 6 | [FILTER] 7 | Name parser 8 | Match * 9 | Key_Name log 10 | Parser json 11 | Reserve_Data True 12 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-json/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "firehose:PutRecordBatch" 7 | ], 8 | "Resource": "*" 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/parse-json/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-json", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit", 12 | "options": { 13 | "config-file-type": "s3", 14 | "config-file-value": "arn:aws:s3:::yourbucket/yourdirectory/extra.conf" 15 | } 16 | }, 17 | "logConfiguration": { 18 | "logDriver": "awslogs", 19 | "options": { 20 | "awslogs-group": "firelens-container", 21 | "awslogs-region": "us-west-2", 22 | "awslogs-create-group": "true", 23 | "awslogs-stream-prefix": "firelens" 24 | } 25 | }, 26 | "memoryReservation": 50 27 | }, 28 | { 29 | "essential": true, 30 | "image": "httpd", 31 | "name": "app", 32 | "logConfiguration": { 33 | "logDriver":"awsfirelens", 34 | "options": { 35 | "Name": "firehose", 36 | "region": "us-west-2", 37 | "delivery_stream": "my-stream", 38 | "retry_limit": "2" 39 | } 40 | }, 41 | "memoryReservation": 100 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /examples/fluent-bit/s3/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to S3 with Fluent Bit 2 | 3 | For documentation on the Fluent Bit S3 plugin, see: [out_s3 in Fluent Bit upstream](https://docs.fluentbit.io/manual/pipeline/outputs/s3) 4 | 5 | S3 Support was released in Fluent Bit 1.6.0 and AWS for Fluent Bit 2.8.0. 6 | 7 | ### Reliability 8 | 9 | The S3 plugin is meant to have a unique, persistent disk location configured with the `store_dir` parameter, which means: 10 | - Each instance of Fluent Bit should use a unique directory/volume to buffer data 11 | - Data volumes should be persistent. Fluent Bit can be restarted and will recover unsent data in the local buffer directory 12 | 13 | The `store_dir` is used for two purposes: 14 | 1. Storing chunks of data before uploading them. If you enable S3 put object, then the plugin will buffer the entire file on Disk before sending. By default the plugin uses multipart uploads, and will only buffer a single chunk of the upload on disk at any point in time. 15 | 2. Storing metadata about multipart uploads. Multipart uploads will not be visible in S3 until all parts/chunks are sent. Fluent Bit expects a persistent disk to store this data. If Fluent Bit is stopped unexpectedly, it can be restarted with the same disk and will complete any unfinished uploads. 16 | 17 | This task definition example shows a scenario where you do not have persistent disk storage set up. Fluent Bit is therefore configured with a small upload file size and a short upload timeout, so that very little data is buffered disk at any point in time. It uses the PutObject API, which is better for environments without persistent storage (the default is multipart uploads, which are great when you have persistent disk on which Fluent Bit can store its upload state file). 18 | 19 | Please see the section in the Fluent Bit S3 documentation on reliability. 20 | 21 | If reliability is a significant concern, we recommend using Kinesis Data Firehose to as a reliable distributed buffer between Fluent Bit and S3. 22 | -------------------------------------------------------------------------------- /examples/fluent-bit/s3/permissions.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [{ 4 | "Effect": "Allow", 5 | "Action": [ 6 | "s3:PutObject" 7 | ], 8 | "Resource": "*" 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /examples/fluent-bit/s3/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-firehose", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration": { 11 | "type": "fluentbit" 12 | }, 13 | "logConfiguration": { 14 | "logDriver": "awslogs", 15 | "options": { 16 | "awslogs-group": "firelens-container", 17 | "awslogs-region": "us-west-2", 18 | "awslogs-create-group": "true", 19 | "awslogs-stream-prefix": "firelens" 20 | } 21 | }, 22 | "memoryReservation": 50 23 | }, 24 | { 25 | "essential": true, 26 | "image": "httpd", 27 | "name": "app", 28 | "logConfiguration": { 29 | "logDriver":"awsfirelens", 30 | "options": { 31 | "Name": "s3", 32 | "region": "us-west-2", 33 | "bucket": "your-bucket", 34 | "total_file_size": "1M", 35 | "upload_timeout": "1m", 36 | "use_put_object": "On", 37 | "retry_limit": "2" 38 | } 39 | }, 40 | "memoryReservation": 100 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /examples/fluent-bit/sematext/README.md: -------------------------------------------------------------------------------- 1 | ### FireLens Example: Logging to Sematext with Fluent Bit 2 | 3 | For documentation on sending your logs from AWS ECS running on either AWS Fargate or AWS EC2 to Sematext Logs, see: [Elastic Container Service (ECS) Logs Integration](https://sematext.com/docs/integration/ecs-logs/). 4 | 5 | AWS recommends that you store sensitive information, like your Sematext `LOGS_TOKEN` using [secretOptions](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html) as shown in the example Task Definition. This is optional; it is also valid to simply specify the `LOGS_TOKEN` in the `logConfiguration`. Note, the `URI` field needs to have a `/` in front of the token: 6 | 7 | ```json 8 | "logConfiguration": { 9 | "logDriver":"awsfirelens", 10 | "options": { 11 | "Name": "http", 12 | "Match": "*", 13 | "Header": "sourceName nginx", 14 | "Host": "logs-ecs-receiver.sematext.com", 15 | "URI": "/", 16 | "Port": "443", 17 | "TLS": "on", 18 | "Format": "json", 19 | "Compress": "gzip", 20 | "Retry_Limit": "2" 21 | } 22 | }, 23 | ``` 24 | 25 | **Note: If you are using the EU region of Sematext, use this Host value: `"Host": "logs-ecs-receiver.eu.sematext.com"`** 26 | -------------------------------------------------------------------------------- /examples/fluent-bit/sematext/task-definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "family": "firelens-example-sematext", 3 | "taskRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_iam_role", 4 | "executionRoleArn": "arn:aws:iam::XXXXXXXXXXXX:role/ecs_task_execution_role", 5 | "containerDefinitions": [ 6 | { 7 | "essential": true, 8 | "image": "amazon/aws-for-fluent-bit:stable", 9 | "name": "log_router", 10 | "firelensConfiguration":{ 11 | "type":"fluentbit", 12 | "options":{ 13 | "enable-ecs-log-metadata":"true" 14 | } 15 | }, 16 | "memoryReservation": 50 17 | }, 18 | { 19 | "essential": true, 20 | "image": "nginx", 21 | "name": "website", 22 | "logConfiguration": { 23 | "logDriver":"awsfirelens", 24 | "options": { 25 | "Name": "http", 26 | "Match": "*", 27 | "Header": "sourceName nginx", 28 | "Host": "logs-ecs-receiver.sematext.com", 29 | "Port": "443", 30 | "TLS": "on", 31 | "Format": "json", 32 | "compress": "gzip", 33 | "retry_limit": "2" 34 | }, 35 | "secretOptions": [{ 36 | "name": "URI", 37 | "valueFrom": "arn:${Partition}:secretsmanager:${Region}:${Account}:secret:${SecretId}" 38 | }] 39 | }, 40 | "memoryReservation": 100 41 | } 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /examples/fluent-bit/send-fb-internal-metrics-to-cw/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/aws-observability/aws-for-fluent-bit:latest 2 | ADD extra.conf /extra.conf 3 | ADD fb_metrics_parser.conf /fb_metrics_parser.conf -------------------------------------------------------------------------------- /examples/fluent-bit/send-fb-internal-metrics-to-cw/extra.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | HTTP_Server On 3 | HTTP_Listen 0.0.0.0 4 | HTTP_PORT 2020 5 | Parsers_File /fb_metrics_parser.conf 6 | storage.metrics On 7 | 8 | # Configure FB to scrape its own prom metrics 9 | [INPUT] 10 | Name exec 11 | Command curl -s http://127.0.0.1:2020/api/v1/metrics/prometheus 12 | Interval_Sec 30 13 | Tag fb_metrics 14 | 15 | # Filter out everything except output metrics 16 | # TODO: REMOVE this filter if you want INPUT PLUGIN METRICS 17 | [FILTER] 18 | Name grep 19 | Match fb_metrics 20 | Regex exec output 21 | 22 | # Filter out the HELP and TYPE fields which aren't parseable by the cw metric filter 23 | [FILTER] 24 | Name grep 25 | Match fb_metrics 26 | Exclude exec HELP 27 | 28 | [FILTER] 29 | Name grep 30 | Match fb_metrics 31 | Exclude exec TYPE 32 | 33 | # Parse the metrics to json for easy parsing in CW Log Group Metrics filter 34 | [FILTER] 35 | Name parser 36 | Match fb_metrics 37 | Key_Name exec 38 | Parser fluentbit_prom_metrics_to_json 39 | Reserve_Data True 40 | 41 | 42 | # Un-comment this section if you want storage metrics 43 | # Unfortunately, Fluent Bit publishes storage metrics via 44 | # a different HTTP path which vends metrics in JSON instead of Prometheus 45 | # These metrics can be sent by the same output as the prom metrics 46 | #[INPUT] 47 | # Name exec 48 | # Command curl -s http://127.0.0.1:2020/api/v1/storage && echo "" 49 | # Interval_Sec 5 50 | # Tag fb_metrics-storage 51 | 52 | # Parse the JSON metrics 53 | #[FILTER] 54 | # Name parser 55 | # Match fb_metrics-storage 56 | # Key_Name exec 57 | # Parser json 58 | 59 | # Add hostname to metrics to identify them 60 | # [FILTER] 61 | # Name record_modifier 62 | # Match * 63 | # Record hostname ${HOSTNAME} 64 | 65 | # Send the metrics as CW Logs 66 | # The CW Metrics filter on the log group will turn them into metrics 67 | # Use hostname in logs to differentiate log streams per task in Fargate 68 | # Alternative is to use: https://github.com/aws/amazon-cloudwatch-logs-for-fluent-bit#templating-log-group-and-stream-names 69 | [OUTPUT] 70 | Name cloudwatch_logs 71 | Match fb_metrics* 72 | region ${FLUENT_BIT_METRICS_LOG_REGION} 73 | log_group_name ${FLUENT_BIT_METRICS_LOG_GROUP} 74 | log_stream_name ${HOSTNAME}-fb-internal-metrics 75 | auto_create_group On 76 | retry_limit 2 77 | # consider configuring more than 1 worker, up to the numer of cores on your host 78 | # to accommodate higher throughput logging. Only 2.32+ supports multi-worker 79 | workers 1 80 | -------------------------------------------------------------------------------- /examples/fluent-bit/send-fb-internal-metrics-to-cw/fb_metrics_parser.conf: -------------------------------------------------------------------------------- 1 | [PARSER] 2 | Name fluentbit_prom_metrics_to_json 3 | Format regex 4 | Regex ^(?[^ {}]*){name=\"(?[^ {}\"]*)\"} (?[^ ]*) (?