├── .bookignore ├── .codespellrc ├── .gitbook.yaml ├── .gitbook └── assets │ ├── 01 (2) (1).png │ ├── 01 (2) (2).png │ ├── 01 (2).png │ ├── 02 (1) (1).png │ ├── 02 (1).png │ ├── 03 (1) (1).png │ ├── 03 (1).png │ ├── 04 (1).png │ ├── 04.png │ ├── 05 (1).png │ ├── 05.png │ ├── 7.10_efk-kibana-discover.png │ ├── 7.10_efk-kibana-index.png │ ├── 7.10_efk-kibana-timestamp.png │ ├── 7.10_kibana-homepage.png │ ├── 7.2_efk-kibana-discover.png │ ├── 7.2_efk-kibana-index.png │ ├── 7.2_efk-kibana-timestamp.png │ ├── 7.2_kibana-homepage.png │ ├── 8.17_efk-kibana-create-data-view.png │ ├── 8.17_efk-kibana-discover-start-page.png │ ├── 8.17_efk-kibana-discover.png │ ├── Fluentd_horizontal.png │ ├── Fluentd_icon.png │ ├── Fluentd_square.png │ ├── apache-to-mongodb (1).png │ ├── apache-to-mongodb.png │ ├── apache_logs (3) (3) (3) (3) (3) (3) (3) (1).png │ ├── apache_logs (3) (3) (3) (3) (3) (3) (3) (2).png │ ├── apache_logs (3) (3) (3) (3) (3) (3) (3) (3).png │ ├── apache_logs (3) (3) (3) (3) (3) (3) (3).png │ ├── buffer-internal-and-parameters (1).png │ ├── buffer-internal-and-parameters.png │ ├── calyptia-fluentd1-wizard.png │ ├── chronograf-explore-data.png │ ├── chronograf-query-2.png │ ├── chronograf-query.png │ ├── copy.png │ ├── csv (2) (2) (2) (2) (2) (2) (2) (1).png │ ├── csv (2) (2) (2) (2) (2) (2) (2) (2).png │ ├── csv (2) (2) (2) (2) (2) (2) (2).png │ ├── dashboard (1) (1) (1) (1).gif │ ├── dashboard (1) (1) (1) (2).gif │ ├── dashboard (1) (1) (1).gif │ ├── dummy.png │ ├── efk-kibana-1 (1) (1) (1) (1) (1) (1) (1) (1).png │ ├── efk-kibana-1 (1) (1) (1) (1) (1) (1) (1).png │ ├── efk-kibana-2 (1) (1) (1) (1).png │ ├── efk-kibana-2 (1) (1) (1).png │ ├── elasticsearch (2) (2) (2) (2) (2) (2) (2) (1).png │ ├── elasticsearch (2) (2) (2) (2) (2) (2) (2) (2).png │ ├── elasticsearch (2) (2) (2) (2) (2) (2) (2).png │ ├── elasticsearch (3).png │ ├── exec (1).png │ ├── exec.png │ ├── exec_filter.png │ ├── file.png │ ├── fluent-package5-menu.png │ ├── fluent-package5-prompt.png │ ├── fluent-package5-wizard.png │ ├── fluent-package6-prompt.png │ ├── fluent-package6-wizard.png │ ├── fluentd-architecture (1) (2) (2) (2) (1).png │ ├── fluentd-architecture (1) (2) (2) (2) (2).png │ ├── fluentd-architecture (1) (2) (2) (2) (3).png │ ├── fluentd-architecture (1) (2) (2) (2) (4).png │ ├── fluentd-architecture (1) (2) (2) (2).png │ ├── fluentd-elasticsearch-kibana (1).png │ ├── fluentd-elasticsearch-kibana.png │ ├── fluentd-kinesis (1) (1) (1).png │ ├── fluentd-kinesis (1) (1) (2).png │ ├── fluentd-kinesis (1) (1) (3).png │ ├── fluentd-kinesis (1) (1).png │ ├── fluentd-norikra-overview (1) (1).png │ ├── fluentd-norikra-overview (1) (2).png │ ├── fluentd-norikra-overview (1) (3).png │ ├── fluentd-norikra-overview (1).png │ ├── fluentd-ui (1) (2) (1).gif │ ├── fluentd-ui (1) (2) (2).gif │ ├── fluentd-ui (1) (2).gif │ ├── fluentd-v0.14-plugin-api-overview.png │ ├── fluentd-zero-downtime-restart-mechanism.png │ ├── fluentd_ha (1) (1) (1) (1).png │ ├── fluentd_ha (1) (1) (1).png │ ├── fluentd_kubernetes (1).png │ ├── fluentd_kubernetes.png │ ├── forward (1).png │ ├── forward.png │ ├── forward_.png │ ├── geoip.png │ ├── glusterfs-fluentd (1) (1).png │ ├── glusterfs-fluentd (1).png │ ├── graylog-graph.png │ ├── graylog-input.png │ ├── http-to-hdfs (1).png │ ├── http-to-hdfs.png │ ├── http.png │ ├── http_rest_api (1) (1).png │ ├── http_rest_api (1).png │ ├── image.png │ ├── in_tail (2) (1).gif │ ├── in_tail (2) (2).gif │ ├── in_tail (2).gif │ ├── js │ └── googleanalytics.js │ ├── json (3) (3) (3) (3) (3) (3) (2) (1).png │ ├── json (3) (3) (3) (3) (3) (3) (2) (2).png │ ├── json (3) (3) (3) (3) (3) (3) (2) (3).png │ ├── json (3) (3) (3) (3) (3) (3) (2).png │ ├── kafka.png │ ├── kibana5-screenshot (2) (2) (2) (2) (2) (2) (2) (1).png │ ├── kibana5-screenshot (2) (2) (2) (2) (2) (2) (2) (2).png │ ├── kibana5-screenshot (2) (2) (2) (2) (2) (2) (2).png │ ├── kibana6-screenshot-topmenu.png │ ├── kibana6-screenshot-visualize.png │ ├── kibana6-screenshot.png │ ├── logo_documentation_0.12 (1).png │ ├── logo_documentation_0.12.png │ ├── logo_documentation_1.0.png │ ├── minio-screenshot.png │ ├── mongo (2) (2) (2) (2) (2) (1) (1).png │ ├── mongo (2) (2) (2) (2) (2) (1) (2).png │ ├── mongo (2) (2) (2) (2) (2) (1).png │ ├── mongo (3).png │ ├── mongo_replset.png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (1).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (2).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (3).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (4).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (5).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (6).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (7).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6) (8).png │ ├── msi-td-agent-command-prompt (1) (3) (3) (3) (6).png │ ├── multi-process-workers (1) (1) (1).png │ ├── multi-process-workers (1) (1).png │ ├── multi_process_topology (1) (1).png │ ├── multi_process_topology (1).png │ ├── multiprocess.png │ ├── nginx (3) (3) (3) (3) (3) (2) (1).png │ ├── nginx (3) (3) (3) (3) (3) (2) (2).png │ ├── nginx (3) (3) (3) (3) (3) (2) (3).png │ ├── nginx (3) (3) (3) (3) (3) (2).png │ ├── null.png │ ├── others (1).png │ ├── others.png │ ├── plugin (1).gif │ ├── plugin.gif │ ├── prometheus-grafana (1) (1).png │ ├── prometheus-grafana (1) (2).png │ ├── prometheus-grafana (1) (3).png │ ├── prometheus-grafana (1) (4).png │ ├── prometheus-grafana (1).png │ ├── prometheus-graph (1).png │ ├── prometheus-graph (2).png │ ├── prometheus-graph (3).png │ ├── prometheus-graph (4).png │ ├── prometheus-graph.png │ ├── prometheus-metrics (1) (1).png │ ├── prometheus-metrics (1) (2).png │ ├── prometheus-metrics (1) (3).png │ ├── prometheus-metrics (1) (4).png │ ├── prometheus-metrics (1).png │ ├── prometheus-targets (1) (1) (1) (1) (1) (1).png │ ├── prometheus-targets (1) (1) (1) (1) (1) (2).png │ ├── prometheus-targets (1) (1) (1) (1) (1) (3).png │ ├── prometheus-targets (1) (1) (1) (1) (1) (4).png │ ├── prometheus-targets (1) (1) (1) (1) (1).png │ ├── raspberry-pi-cloud-data-logger (1).png │ ├── raspberry-pi-cloud-data-logger.png │ ├── relabel.png │ ├── rewrite_tag_filter.png │ ├── roundrobin.png │ ├── s3 (3) (3) (2) (1).png │ ├── s3 (3) (3) (2) (2).png │ ├── s3 (3) (3) (2) (3).png │ ├── s3 (3) (3) (2).png │ ├── s3 (4).png │ ├── screen-shot-2021-03-16-at-12.50.12-pm.png │ ├── screen-shot-2021-03-16-at-12.51.26-pm.png │ ├── screen-shot-2021-03-16-at-12.57.45-pm.png │ ├── screen-shot-2021-03-16-at-12.59.05-pm.png │ ├── scribe.png │ ├── secure_forward (1).png │ ├── secure_forward.png │ ├── sematext-configure-logs.png │ ├── sematext-dashboard.png │ ├── sematext-logs-overview.png │ ├── setting (1).gif │ ├── setting.gif │ ├── source-only-mode-mechanism.png │ ├── splunk.png │ ├── stdout.png │ ├── syslog (1) (1) (1) (1) (1) (1) (1) (1).png │ ├── syslog (1) (1) (1) (1) (1) (1) (1).png │ ├── syslog (2).png │ ├── syslog-fluentd-influxdb.png │ ├── tail.png │ ├── tcp.png │ ├── td-agent-windows-prompt.png │ ├── td-agent4-menu.png │ ├── td-agent4-prompt.png │ ├── td-agent4-wizard.png │ ├── treasure_data (3) (3) (3) (3) (3) (2) (1) (1).png │ ├── treasure_data (3) (3) (3) (3) (3) (2) (1) (2).png │ ├── treasure_data (3) (3) (3) (3) (3) (2) (1) (3).png │ ├── treasure_data (3) (3) (3) (3) (3) (2) (1).png │ ├── treasuredata_architecture (1).png │ ├── treasuredata_architecture.png │ ├── tsv (2) (2) (2) (2) (2) (2) (2) (1).png │ ├── tsv (2) (2) (2) (2) (2) (2) (2) (2).png │ ├── tsv (2) (2) (2) (2) (2) (2) (2).png │ ├── udp.png │ ├── unix.png │ └── webhdfs.png ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.yaml │ └── config.yml └── workflows │ ├── add-to-project.yml │ └── codespell.yml ├── LICENSE ├── README.md ├── SUMMARY.md ├── appendix ├── td-agent-v2-vs-v3-vs-v4.md └── update-from-v0.12.md ├── book.json ├── buffer ├── README.md ├── file.md ├── file_single.md └── memory.md ├── configuration ├── README.md ├── buffer-section.md ├── config-file-yaml.md ├── config-file.md ├── extract-section.md ├── format-section.md ├── inject-section.md ├── parse-section.md ├── plugin-common-parameters.md ├── routing-examples.md ├── service_discovery-section.md ├── storage-section.md └── transport-section.md ├── container-deployment ├── README.md ├── docker-compose.md ├── docker-logging-driver.md ├── install-by-docker.md └── kubernetes.md ├── deployment ├── README.md ├── command-line-option.md ├── failure-scenarios.md ├── fluentd-ui.md ├── high-availability.md ├── linux-capability.md ├── logging.md ├── multi-process-workers.md ├── performance-tuning-single-process.md ├── plugin-management.md ├── rpc.md ├── signals.md ├── source-only-mode.md ├── system-config.md ├── trouble-shooting.md └── zero-downtime-restart.md ├── filter ├── README.md ├── geoip.md ├── grep.md ├── parser.md ├── record_transformer.md └── stdout.md ├── formatter ├── README.md ├── csv.md ├── hash.md ├── json.md ├── ltsv.md ├── msgpack.md ├── out_file.md ├── single_value.md ├── stdout.md └── tsv.md ├── how-to-guides ├── README.md ├── apache-to-minio.md ├── apache-to-mongodb.md ├── apache-to-s3.md ├── cep-norikra.md ├── filter-modify-apache.md ├── free-alternative-to-splunk-by-fluentd.md ├── graylog.md ├── http-to-hdfs.md ├── http-to-td.md ├── kinesis-stream.md ├── logs-to-sematext.md ├── parse-syslog.md ├── raspberrypi-cloud-data-logger.md ├── splunk-like-grep-and-alert-email.md ├── stream-analytics-with-materialize.md └── syslog-influxdb.md ├── images ├── fluentd-riak.png ├── logo │ ├── Fluentd.ai │ ├── Fluentd_horizontal.svg │ ├── Fluentd_horizontal_banner.png │ ├── Fluentd_horizontal_blight_300dpi.png │ ├── Fluentd_horizontal_blight_72dpi.png │ ├── Fluentd_horizontal_dark.png │ ├── Fluentd_horizontal_dark_300dpi.png │ ├── Fluentd_horizontal_dark_72dpi.png │ ├── Fluentd_icon.svg │ ├── Fluentd_icon_dark.png │ ├── Fluentd_icon_square.png │ ├── Fluentd_square.svg │ ├── Fluentd_square_blight_300dpi.png │ ├── Fluentd_square_blight_72dpi.png │ ├── Fluentd_square_dark_300dpi.png │ └── Fluentd_square_dark_72dpi.png └── source │ ├── Fluentd_icon.svg │ ├── drawing-UDP.svg │ ├── drawing.svg │ ├── plugin.png │ └── right-arrow.svg ├── input ├── README.md ├── exec.md ├── forward.md ├── http.md ├── monitor_agent.md ├── sample.md ├── syslog.md ├── tail.md ├── tcp.md ├── udp.md ├── unix.md └── windows_eventlog.md ├── installation ├── README.md ├── before-install.md ├── install-by-deb-td-agent-v3.md ├── install-by-deb-td-agent-v4.md ├── install-by-dmg-td-agent-v3.md ├── install-by-dmg-td-agent-v4.md ├── install-by-gem.md ├── install-by-msi-td-agent-v3.md ├── install-by-msi-td-agent-v4.md ├── install-by-rpm-td-agent-v3.md ├── install-by-rpm-td-agent-v4.md ├── install-calyptia-fluentd │ ├── README.md │ ├── install-by-deb-calyptia-fluentd.md │ ├── install-by-dmg-calyptia-fluentd.md │ ├── install-by-msi-calyptia-fluentd.md │ └── install-by-rpm-calyptia-fluentd.md ├── install-fluent-package │ ├── README.md │ ├── install-by-deb-fluent-package.md │ ├── install-by-dmg-fluent-package.md │ ├── install-by-msi-fluent-package.md │ └── install-by-rpm-fluent-package.md ├── install-from-source.md └── post-installation-guide.md ├── language-bindings ├── README.md ├── java.md ├── nodejs.md ├── perl.md ├── php.md ├── python.md ├── ruby.md └── scala.md ├── metrics ├── README.md └── local.md ├── monitoring-fluentd ├── README.md ├── monitoring-prometheus.md ├── monitoring-rest-api.md └── overview.md ├── output ├── README.md ├── buffer.md ├── copy.md ├── elasticsearch.md ├── exec.md ├── exec_filter.md ├── file.md ├── forward.md ├── http.md ├── kafka.md ├── mongo.md ├── mongo_replset.md ├── null.md ├── opensearch.md ├── relabel.md ├── rewrite_tag_filter.md ├── roundrobin.md ├── s3.md ├── secondary_file.md ├── stdout.md └── webhdfs.md ├── parser ├── README.md ├── apache2.md ├── apache_error.md ├── csv.md ├── json.md ├── ltsv.md ├── msgpack.md ├── multiline.md ├── nginx.md ├── none.md ├── regexp.md ├── syslog.md └── tsv.md ├── plugin-development ├── README.md ├── api-config-types.md ├── api-plugin-base.md ├── api-plugin-buffer.md ├── api-plugin-filter.md ├── api-plugin-formatter.md ├── api-plugin-input.md ├── api-plugin-output.md ├── api-plugin-parser.md ├── api-plugin-service_discovery.md ├── api-plugin-storage.md ├── plugin-test-code.md └── plugin-update-from-v0.12.md ├── plugin-helper-overview ├── README.md ├── api-plugin-helper-child_process.md ├── api-plugin-helper-compat_parameters.md ├── api-plugin-helper-event_emitter.md ├── api-plugin-helper-event_loop.md ├── api-plugin-helper-extract.md ├── api-plugin-helper-formatter.md ├── api-plugin-helper-http_server.md ├── api-plugin-helper-inject.md ├── api-plugin-helper-metrics.md ├── api-plugin-helper-parser.md ├── api-plugin-helper-record_accessor.md ├── api-plugin-helper-server.md ├── api-plugin-helper-service_discovery.md ├── api-plugin-helper-socket.md ├── api-plugin-helper-storage.md ├── api-plugin-helper-thread.md └── api-plugin-helper-timer.md ├── plugins └── input │ └── dummy.md ├── quickstart ├── README.md ├── faq.md ├── fluent-package-v5-vs-td-agent.md ├── life-of-a-fluentd-event.md ├── logo.md └── support.md ├── service_discovery ├── README.md ├── file.md ├── srv.md └── static.md ├── storage ├── README.md └── local.md └── troubleshooting-guide.md /.bookignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.# 3 | *.sh 4 | *.mmd 5 | #* 6 | 7 | -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | skip = node_modules, _book, images 3 | count = 4 | ignore-words-list = aNULL, userA, followings 5 | -------------------------------------------------------------------------------- /.gitbook/assets/01 (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/01 (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/01 (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/01 (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/01 (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/01 (2).png -------------------------------------------------------------------------------- /.gitbook/assets/02 (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/02 (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/02 (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/02 (1).png -------------------------------------------------------------------------------- /.gitbook/assets/03 (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/03 (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/03 (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/03 (1).png -------------------------------------------------------------------------------- /.gitbook/assets/04 (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/04 (1).png -------------------------------------------------------------------------------- /.gitbook/assets/04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/04.png -------------------------------------------------------------------------------- /.gitbook/assets/05 (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/05 (1).png -------------------------------------------------------------------------------- /.gitbook/assets/05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/05.png -------------------------------------------------------------------------------- /.gitbook/assets/7.10_efk-kibana-discover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.10_efk-kibana-discover.png -------------------------------------------------------------------------------- /.gitbook/assets/7.10_efk-kibana-index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.10_efk-kibana-index.png -------------------------------------------------------------------------------- /.gitbook/assets/7.10_efk-kibana-timestamp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.10_efk-kibana-timestamp.png -------------------------------------------------------------------------------- /.gitbook/assets/7.10_kibana-homepage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.10_kibana-homepage.png -------------------------------------------------------------------------------- /.gitbook/assets/7.2_efk-kibana-discover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.2_efk-kibana-discover.png -------------------------------------------------------------------------------- /.gitbook/assets/7.2_efk-kibana-index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.2_efk-kibana-index.png -------------------------------------------------------------------------------- /.gitbook/assets/7.2_efk-kibana-timestamp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.2_efk-kibana-timestamp.png -------------------------------------------------------------------------------- /.gitbook/assets/7.2_kibana-homepage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/7.2_kibana-homepage.png -------------------------------------------------------------------------------- /.gitbook/assets/8.17_efk-kibana-create-data-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/8.17_efk-kibana-create-data-view.png -------------------------------------------------------------------------------- /.gitbook/assets/8.17_efk-kibana-discover-start-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/8.17_efk-kibana-discover-start-page.png -------------------------------------------------------------------------------- /.gitbook/assets/8.17_efk-kibana-discover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/8.17_efk-kibana-discover.png -------------------------------------------------------------------------------- /.gitbook/assets/Fluentd_horizontal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/Fluentd_horizontal.png -------------------------------------------------------------------------------- /.gitbook/assets/Fluentd_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/Fluentd_icon.png -------------------------------------------------------------------------------- /.gitbook/assets/Fluentd_square.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/Fluentd_square.png -------------------------------------------------------------------------------- /.gitbook/assets/apache-to-mongodb (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/apache-to-mongodb (1).png -------------------------------------------------------------------------------- /.gitbook/assets/apache-to-mongodb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/apache-to-mongodb.png -------------------------------------------------------------------------------- /.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/apache_logs (3) (3) (3) (3) (3) (3) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/buffer-internal-and-parameters (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/buffer-internal-and-parameters (1).png -------------------------------------------------------------------------------- /.gitbook/assets/buffer-internal-and-parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/buffer-internal-and-parameters.png -------------------------------------------------------------------------------- /.gitbook/assets/calyptia-fluentd1-wizard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/calyptia-fluentd1-wizard.png -------------------------------------------------------------------------------- /.gitbook/assets/chronograf-explore-data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/chronograf-explore-data.png -------------------------------------------------------------------------------- /.gitbook/assets/chronograf-query-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/chronograf-query-2.png -------------------------------------------------------------------------------- /.gitbook/assets/chronograf-query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/chronograf-query.png -------------------------------------------------------------------------------- /.gitbook/assets/copy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/copy.png -------------------------------------------------------------------------------- /.gitbook/assets/csv (2) (2) (2) (2) (2) (2) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/csv (2) (2) (2) (2) (2) (2) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/csv (2) (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/csv (2) (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/csv (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/csv (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/dashboard (1) (1) (1) (1).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/dashboard (1) (1) (1) (1).gif -------------------------------------------------------------------------------- /.gitbook/assets/dashboard (1) (1) (1) (2).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/dashboard (1) (1) (1) (2).gif -------------------------------------------------------------------------------- /.gitbook/assets/dashboard (1) (1) (1).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/dashboard (1) (1) (1).gif -------------------------------------------------------------------------------- /.gitbook/assets/dummy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/dummy.png -------------------------------------------------------------------------------- /.gitbook/assets/efk-kibana-1 (1) (1) (1) (1) (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/efk-kibana-1 (1) (1) (1) (1) (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/efk-kibana-1 (1) (1) (1) (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/efk-kibana-1 (1) (1) (1) (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/efk-kibana-2 (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/efk-kibana-2 (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/efk-kibana-2 (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/efk-kibana-2 (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/elasticsearch (2) (2) (2) (2) (2) (2) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/elasticsearch (2) (2) (2) (2) (2) (2) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/elasticsearch (2) (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/elasticsearch (2) (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/elasticsearch (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/elasticsearch (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/elasticsearch (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/elasticsearch (3).png -------------------------------------------------------------------------------- /.gitbook/assets/exec (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/exec (1).png -------------------------------------------------------------------------------- /.gitbook/assets/exec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/exec.png -------------------------------------------------------------------------------- /.gitbook/assets/exec_filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/exec_filter.png -------------------------------------------------------------------------------- /.gitbook/assets/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/file.png -------------------------------------------------------------------------------- /.gitbook/assets/fluent-package5-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluent-package5-menu.png -------------------------------------------------------------------------------- /.gitbook/assets/fluent-package5-prompt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluent-package5-prompt.png -------------------------------------------------------------------------------- /.gitbook/assets/fluent-package5-wizard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluent-package5-wizard.png -------------------------------------------------------------------------------- /.gitbook/assets/fluent-package6-prompt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluent-package6-prompt.png -------------------------------------------------------------------------------- /.gitbook/assets/fluent-package6-wizard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluent-package6-wizard.png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (4).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-architecture (1) (2) (2) (2) (4).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-architecture (1) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-architecture (1) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-elasticsearch-kibana (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-elasticsearch-kibana (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-elasticsearch-kibana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-elasticsearch-kibana.png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-kinesis (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-kinesis (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-kinesis (1) (1) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-kinesis (1) (1) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-kinesis (1) (1) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-kinesis (1) (1) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-kinesis (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-kinesis (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-norikra-overview (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-norikra-overview (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-norikra-overview (1) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-norikra-overview (1) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-norikra-overview (1) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-norikra-overview (1) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-norikra-overview (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-norikra-overview (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-ui (1) (2) (1).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-ui (1) (2) (1).gif -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-ui (1) (2) (2).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-ui (1) (2) (2).gif -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-ui (1) (2).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-ui (1) (2).gif -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-v0.14-plugin-api-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-v0.14-plugin-api-overview.png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd-zero-downtime-restart-mechanism.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd-zero-downtime-restart-mechanism.png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd_ha (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd_ha (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd_ha (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd_ha (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd_kubernetes (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd_kubernetes (1).png -------------------------------------------------------------------------------- /.gitbook/assets/fluentd_kubernetes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/fluentd_kubernetes.png -------------------------------------------------------------------------------- /.gitbook/assets/forward (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/forward (1).png -------------------------------------------------------------------------------- /.gitbook/assets/forward.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/forward.png -------------------------------------------------------------------------------- /.gitbook/assets/forward_.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/forward_.png -------------------------------------------------------------------------------- /.gitbook/assets/geoip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/geoip.png -------------------------------------------------------------------------------- /.gitbook/assets/glusterfs-fluentd (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/glusterfs-fluentd (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/glusterfs-fluentd (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/glusterfs-fluentd (1).png -------------------------------------------------------------------------------- /.gitbook/assets/graylog-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/graylog-graph.png -------------------------------------------------------------------------------- /.gitbook/assets/graylog-input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/graylog-input.png -------------------------------------------------------------------------------- /.gitbook/assets/http-to-hdfs (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/http-to-hdfs (1).png -------------------------------------------------------------------------------- /.gitbook/assets/http-to-hdfs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/http-to-hdfs.png -------------------------------------------------------------------------------- /.gitbook/assets/http.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/http.png -------------------------------------------------------------------------------- /.gitbook/assets/http_rest_api (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/http_rest_api (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/http_rest_api (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/http_rest_api (1).png -------------------------------------------------------------------------------- /.gitbook/assets/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/image.png -------------------------------------------------------------------------------- /.gitbook/assets/in_tail (2) (1).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/in_tail (2) (1).gif -------------------------------------------------------------------------------- /.gitbook/assets/in_tail (2) (2).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/in_tail (2) (2).gif -------------------------------------------------------------------------------- /.gitbook/assets/in_tail (2).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/in_tail (2).gif -------------------------------------------------------------------------------- /.gitbook/assets/js/googleanalytics.js: -------------------------------------------------------------------------------- 1 | (function(i, s, o, g, r, a, m) { 2 | i['GoogleAnalyticsObject'] = r; 3 | i[r] = i[r] || function() { 4 | (i[r].q = i[r].q || []).push(arguments) 5 | }, i[r].l = 1 * new Date(); 6 | a = s.createElement(o), 7 | m = s.getElementsByTagName(o)[0]; 8 | a.async = 1; 9 | a.src = g; 10 | m.parentNode.insertBefore(a, m) 11 | })(window, document, 'script', '//www.google-analytics.com/analytics.js', 'ga'); 12 | 13 | ga('create', 'UA-XXXXXXXXXX', 'auto'); 14 | ga('send', 'pageview'); -------------------------------------------------------------------------------- /.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/json (3) (3) (3) (3) (3) (3) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/kafka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/kafka.png -------------------------------------------------------------------------------- /.gitbook/assets/kibana5-screenshot (2) (2) (2) (2) (2) (2) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/kibana5-screenshot (2) (2) (2) (2) (2) (2) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/kibana5-screenshot (2) (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/kibana5-screenshot (2) (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/kibana5-screenshot (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/kibana5-screenshot (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/kibana6-screenshot-topmenu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/kibana6-screenshot-topmenu.png -------------------------------------------------------------------------------- /.gitbook/assets/kibana6-screenshot-visualize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/kibana6-screenshot-visualize.png -------------------------------------------------------------------------------- /.gitbook/assets/kibana6-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/kibana6-screenshot.png -------------------------------------------------------------------------------- /.gitbook/assets/logo_documentation_0.12 (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/logo_documentation_0.12 (1).png -------------------------------------------------------------------------------- /.gitbook/assets/logo_documentation_0.12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/logo_documentation_0.12.png -------------------------------------------------------------------------------- /.gitbook/assets/logo_documentation_1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/logo_documentation_1.0.png -------------------------------------------------------------------------------- /.gitbook/assets/minio-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/minio-screenshot.png -------------------------------------------------------------------------------- /.gitbook/assets/mongo (2) (2) (2) (2) (2) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/mongo (2) (2) (2) (2) (2) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/mongo (2) (2) (2) (2) (2) (1) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/mongo (2) (2) (2) (2) (2) (1) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/mongo (2) (2) (2) (2) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/mongo (2) (2) (2) (2) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/mongo (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/mongo (3).png -------------------------------------------------------------------------------- /.gitbook/assets/mongo_replset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/mongo_replset.png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (4).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (4).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (5).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (5).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (6).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (6).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (7).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (7).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (8).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6) (8).png -------------------------------------------------------------------------------- /.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/msi-td-agent-command-prompt (1) (3) (3) (3) (6).png -------------------------------------------------------------------------------- /.gitbook/assets/multi-process-workers (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/multi-process-workers (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/multi-process-workers (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/multi-process-workers (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/multi_process_topology (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/multi_process_topology (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/multi_process_topology (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/multi_process_topology (1).png -------------------------------------------------------------------------------- /.gitbook/assets/multiprocess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/multiprocess.png -------------------------------------------------------------------------------- /.gitbook/assets/nginx (3) (3) (3) (3) (3) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/nginx (3) (3) (3) (3) (3) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/nginx (3) (3) (3) (3) (3) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/nginx (3) (3) (3) (3) (3) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/nginx (3) (3) (3) (3) (3) (2) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/nginx (3) (3) (3) (3) (3) (2) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/nginx (3) (3) (3) (3) (3) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/nginx (3) (3) (3) (3) (3) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/null.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/null.png -------------------------------------------------------------------------------- /.gitbook/assets/others (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/others (1).png -------------------------------------------------------------------------------- /.gitbook/assets/others.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/others.png -------------------------------------------------------------------------------- /.gitbook/assets/plugin (1).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/plugin (1).gif -------------------------------------------------------------------------------- /.gitbook/assets/plugin.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/plugin.gif -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-grafana (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-grafana (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-grafana (1) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-grafana (1) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-grafana (1) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-grafana (1) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-grafana (1) (4).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-grafana (1) (4).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-grafana (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-grafana (1).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-graph (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-graph (1).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-graph (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-graph (2).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-graph (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-graph (3).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-graph (4).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-graph (4).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-graph.png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-metrics (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-metrics (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-metrics (1) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-metrics (1) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-metrics (1) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-metrics (1) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-metrics (1) (4).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-metrics (1) (4).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-metrics (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-metrics (1).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (4).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1) (4).png -------------------------------------------------------------------------------- /.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/prometheus-targets (1) (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/raspberry-pi-cloud-data-logger (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/raspberry-pi-cloud-data-logger (1).png -------------------------------------------------------------------------------- /.gitbook/assets/raspberry-pi-cloud-data-logger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/raspberry-pi-cloud-data-logger.png -------------------------------------------------------------------------------- /.gitbook/assets/relabel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/relabel.png -------------------------------------------------------------------------------- /.gitbook/assets/rewrite_tag_filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/rewrite_tag_filter.png -------------------------------------------------------------------------------- /.gitbook/assets/roundrobin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/roundrobin.png -------------------------------------------------------------------------------- /.gitbook/assets/s3 (3) (3) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/s3 (3) (3) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/s3 (3) (3) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/s3 (3) (3) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/s3 (3) (3) (2) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/s3 (3) (3) (2) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/s3 (3) (3) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/s3 (3) (3) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/s3 (4).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/s3 (4).png -------------------------------------------------------------------------------- /.gitbook/assets/screen-shot-2021-03-16-at-12.50.12-pm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/screen-shot-2021-03-16-at-12.50.12-pm.png -------------------------------------------------------------------------------- /.gitbook/assets/screen-shot-2021-03-16-at-12.51.26-pm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/screen-shot-2021-03-16-at-12.51.26-pm.png -------------------------------------------------------------------------------- /.gitbook/assets/screen-shot-2021-03-16-at-12.57.45-pm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/screen-shot-2021-03-16-at-12.57.45-pm.png -------------------------------------------------------------------------------- /.gitbook/assets/screen-shot-2021-03-16-at-12.59.05-pm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/screen-shot-2021-03-16-at-12.59.05-pm.png -------------------------------------------------------------------------------- /.gitbook/assets/scribe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/scribe.png -------------------------------------------------------------------------------- /.gitbook/assets/secure_forward (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/secure_forward (1).png -------------------------------------------------------------------------------- /.gitbook/assets/secure_forward.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/secure_forward.png -------------------------------------------------------------------------------- /.gitbook/assets/sematext-configure-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/sematext-configure-logs.png -------------------------------------------------------------------------------- /.gitbook/assets/sematext-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/sematext-dashboard.png -------------------------------------------------------------------------------- /.gitbook/assets/sematext-logs-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/sematext-logs-overview.png -------------------------------------------------------------------------------- /.gitbook/assets/setting (1).gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/setting (1).gif -------------------------------------------------------------------------------- /.gitbook/assets/setting.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/setting.gif -------------------------------------------------------------------------------- /.gitbook/assets/source-only-mode-mechanism.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/source-only-mode-mechanism.png -------------------------------------------------------------------------------- /.gitbook/assets/splunk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/splunk.png -------------------------------------------------------------------------------- /.gitbook/assets/stdout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/stdout.png -------------------------------------------------------------------------------- /.gitbook/assets/syslog (1) (1) (1) (1) (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/syslog (1) (1) (1) (1) (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/syslog (1) (1) (1) (1) (1) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/syslog (1) (1) (1) (1) (1) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/syslog (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/syslog (2).png -------------------------------------------------------------------------------- /.gitbook/assets/syslog-fluentd-influxdb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/syslog-fluentd-influxdb.png -------------------------------------------------------------------------------- /.gitbook/assets/tail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/tail.png -------------------------------------------------------------------------------- /.gitbook/assets/tcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/tcp.png -------------------------------------------------------------------------------- /.gitbook/assets/td-agent-windows-prompt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/td-agent-windows-prompt.png -------------------------------------------------------------------------------- /.gitbook/assets/td-agent4-menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/td-agent4-menu.png -------------------------------------------------------------------------------- /.gitbook/assets/td-agent4-prompt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/td-agent4-prompt.png -------------------------------------------------------------------------------- /.gitbook/assets/td-agent4-wizard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/td-agent4-wizard.png -------------------------------------------------------------------------------- /.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1) (3).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1) (3).png -------------------------------------------------------------------------------- /.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/treasure_data (3) (3) (3) (3) (3) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/treasuredata_architecture (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/treasuredata_architecture (1).png -------------------------------------------------------------------------------- /.gitbook/assets/treasuredata_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/treasuredata_architecture.png -------------------------------------------------------------------------------- /.gitbook/assets/tsv (2) (2) (2) (2) (2) (2) (2) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/tsv (2) (2) (2) (2) (2) (2) (2) (1).png -------------------------------------------------------------------------------- /.gitbook/assets/tsv (2) (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/tsv (2) (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/tsv (2) (2) (2) (2) (2) (2) (2).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/tsv (2) (2) (2) (2) (2) (2) (2).png -------------------------------------------------------------------------------- /.gitbook/assets/udp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/udp.png -------------------------------------------------------------------------------- /.gitbook/assets/unix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/unix.png -------------------------------------------------------------------------------- /.gitbook/assets/webhdfs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/.gitbook/assets/webhdfs.png -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yaml: -------------------------------------------------------------------------------- 1 | name: Bug Report for https://docs.fluentd.org/ 2 | description: Create a report about Fluentd's documentation 3 | labels: "waiting-for-triage" 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | This repository tracks an issue for Fluentd's documentation. 9 | If you found a bug of Fluentd, please forward it to https://github.com/fluent/fluentd. 10 | Do not create an issue about usage of Fluentd itself. Ask on https://discuss.fluentd.org/! 11 | - type: textarea 12 | id: description 13 | attributes: 14 | label: Describe the bug 15 | description: A clear and concise description of what the bug in https://docs.fluentd.org 16 | validations: 17 | required: true 18 | - type: textarea 19 | id: link 20 | attributes: 21 | label: Link to the problematic documentation 22 | description: Write the actual URL 23 | validations: 24 | required: true 25 | - type: textarea 26 | id: expected 27 | attributes: 28 | label: Expected explanation 29 | description: A clear and concise description of what you expected to happen 30 | validations: 31 | required: true 32 | - type: textarea 33 | id: addtional-context 34 | attributes: 35 | label: Additional context 36 | description: Add any other context about the problem here. 37 | validations: 38 | required: false 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Ask a Question 4 | url: https://discuss.fluentd.org/ 5 | about: I have questions about Fluentd and plugins. Please ask and answer questions at https://discuss.fluentd.org/. 6 | -------------------------------------------------------------------------------- /.github/workflows/add-to-project.yml: -------------------------------------------------------------------------------- 1 | name: Add bugs to fluent project 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | 8 | jobs: 9 | add-to-project: 10 | name: Add issue to project 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/add-to-project@v1.0.2 14 | with: 15 | project-url: https://github.com/orgs/fluent/projects/4 16 | github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} 17 | labeled: waiting-for-triage 18 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | name: Add check with codespell 2 | 3 | on: 4 | pull_request: 5 | workflow_dispatch: 6 | 7 | concurrency: 8 | group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} 9 | cancel-in-progress: true 10 | 11 | jobs: 12 | spell-check: 13 | name: check spelling with codespell 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: actions/setup-python@v5 18 | with: 19 | python-version: '3.13' 20 | - name: Install codespell 21 | run: pip install codespell 22 | - name: Run codespell 23 | run: codespell 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 |  4 | 5 | [Fluentd](https://www.fluentd.org/) is an open-source data collector for a unified logging layer. Fluentd allows you to unify data collection and consumption for better use and understanding of data. 6 | 7 | [Fluentd](https://www.fluentd.org/) is licensed under the terms of the [Apache License v2.0](http://www.apache.org/licenses/LICENSE-2.0). This project is made and sponsored by [Treasure Data](https://www.treasuredata.com). 8 | 9 | -------------------------------------------------------------------------------- /appendix/td-agent-v2-vs-v3-vs-v4.md: -------------------------------------------------------------------------------- 1 | # td-agent v2 vs v3 vs v4 2 | 3 | {% hint style='danger' %} 4 | The series of td-agent had already reached End of Life (EOL). td-agent should not be newly installed because no support, no new release and no security updates anymore. 5 | {% endhint %} 6 | 7 | ## Supported Platforms 8 | 9 | [ClearCode, Inc.](https://www.clear-code.com) maintains stable packages for Fluentd and canonical plugins as Treasure Agent \(the package is called `td-agent`\). `td-agent` has v2, v3 and v4. [Chronosphere](https://chronosphere.io/) (formerly Calyptia) maintains stable packages as Calyptia-fluentd as another option. Supported OS is the same as td-agent v4 currently. 10 | 11 | | Platform | v2 | v3 | v4\(x86\_64\) | v4\(Arm64\) | 12 | | :--- | :---: | :---: | :---: | :---: | 13 | | RedHat/CentOS 5 | ✔ | | | | 14 | | RedHat/CentOS 6 | ✔ | ✔ | | | 15 | | RedHat/CentOS 7 | ✔ | ✔ | ✔ | ✔ | 16 | | RedHat/CentOS 8 | | ✔ | ✔ | ✔ | 17 | | Amazon Linux 2 | | ✔ | ✔ | ✔ | 18 | | Ubuntu Precise | ✔ | | | | 19 | | Ubuntu Trusty | ✔ | ✔ | | | 20 | | Ubuntu Xenial | ✔ | ✔ | ✔ | | 21 | | Ubuntu Bionic | ✔ | ✔ | ✔ | ✔ | 22 | | Ubuntu Focal | | | ✔ | ✔ | 23 | | Debian Jessie | ✔ | ✔ | | | 24 | | Debian Stretch | ✔ | ✔ | | | 25 | | Debian Buster | | ✔ | ✔ | ✔ | 26 | | macOS | ✔ | ✔ | | | 27 | | Windows | | ✔ | ✔ | | 28 | 29 | ## Features 30 | 31 | ### `td-agent` v4 32 | 33 | New stable. Major feature updates to `td-agent` v4 are as follows: 34 | 35 | * Ruby 2.7 36 | * Fluentd v1 37 | * Arm64 Support 38 | 39 | See also [Changes from Treasure Agent 3](https://github.com/fluent-plugins-nursery/td-agent-builder#changes-from-treasure-agent-3). 40 | 41 | ### `td-agent` v3 42 | 43 | Old stable. 44 | 45 | * Ruby 2.4 46 | * Fluentd v1 47 | * Windows Support 48 | * Remove `fluentd-ui` from the package 49 | 50 | ### `td-agent` v2 51 | 52 | This is for v0.12 and old distribution users. We don't recommend this version for new deployments. 53 | 54 | ## How to Install 55 | 56 | * [Ubuntu/Debian](../installation/install-by-deb-td-agent-v4.md) 57 | * [RedHat/CentOS](../installation/install-by-rpm-td-agent-v4.md) 58 | * [Windows](../installation/install-by-msi-td-agent-v4.md) 59 | * [macOS](../installation/install-by-dmg-td-agent-v4.md) 60 | * [RubyGems](../installation/install-by-gem.md) 61 | 62 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 63 | 64 | -------------------------------------------------------------------------------- /book.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Fluentd 1.0 Documentation", 3 | "plugins": [ 4 | "addcssjs", 5 | "hints" 6 | ], 7 | 8 | "pluginsConfig": { 9 | "addcssjs": { 10 | "js": [".gitbook/assets/js/googleanalytics.js"] 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /buffer/memory.md: -------------------------------------------------------------------------------- 1 | # memory 2 | 3 | The `memory` buffer plugin provides a fast buffer implementation. It uses memory to store buffer chunks. When Fluentd is shut down, buffered logs that cannot be written quickly are deleted. 4 | 5 | ## Parameters 6 | 7 | * [Common Parameters](../configuration/plugin-common-parameters.md) 8 | * [Buffer Section Configurations](../configuration/buffer-section.md) 9 | 10 | `memory` plugin has no specific parameters. 11 | 12 | ## Example 13 | 14 | ```text 15 | 16 | 17 | @type memory 18 | 19 | 20 | ``` 21 | 22 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 23 | 24 | -------------------------------------------------------------------------------- /configuration/README.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | -------------------------------------------------------------------------------- /configuration/plugin-common-parameters.md: -------------------------------------------------------------------------------- 1 | # Config: Common Parameters 2 | 3 | Some common parameters are available for all or some of the Fluentd plugins. This page describes these parameters. 4 | 5 | ## Parameters for all the Plugins 6 | 7 | ### `@type` 8 | 9 | The `@type` parameter specifies the type of the plugin. 10 | 11 | ```text 12 | 13 | @type my_plugin_type 14 | 15 | 16 | 17 | @type my_filter 18 | 19 | ``` 20 | 21 | ### `@id` 22 | 23 | The `@id` parameter specifies a unique name for the configuration. It is used as paths for buffer, storage, logging and for other purposes. 24 | 25 | ```text 26 | 27 | @type file 28 | @id service_www_accesslog 29 | path /path/to/my/access.log 30 | # ... 31 | 32 | ``` 33 | 34 | This parameter should be specified for all the plugins to enable `root_dir` feature globally. 35 | 36 | See also: [System Configuration](../deployment/system-config.md) 37 | 38 | ### `@log_level` 39 | 40 | This parameter specifies the plugin-specific logging level. The default log level is `info`. Global log level can be specified by setting `log_level` in `` section or with `-v/-q` command line arguments. The `@log_level` parameter overrides the logging level only for the specified plugin instance. 41 | 42 | ```text 43 | 44 | log_level info 45 | 46 | 47 | 48 | # ... 49 | @log_level debug # shows debug log only for this plugin 50 | 51 | ``` 52 | 53 | The main purposes of this parameter are: 54 | 55 | 1. to suppress too many logs for that plugin; and, 56 | 2. to show the debug logs to help in the debugging process. 57 | 58 | Please see the [logging article](../deployment/logging.md) for further details. 59 | 60 | ## Plugin Parameters that Emit Events 61 | 62 | ### @label 63 | 64 | The `@label` parameter is to route the input events to `` sections, the set of the `` and `` subsections under ``. 65 | 66 | ```text 67 | 68 | @type ... 69 | @label @access_logs 70 | # ... 71 | 72 | 73 | 74 | @type ... 75 | @label @system_metrics 76 | # ... 77 | 78 | 79 | 80 | 81 | @type file 82 | path ... 83 | 84 | 85 | 86 | 87 | 88 | @type file 89 | path ... 90 | 91 | 92 | ``` 93 | 94 | NOTE: The values for the `@label` parameter MUST start with `@` character. 95 | 96 | Specifying `@label` is strongly recommended to route events to any plugin without modifying the tags. It helps make the complex configuration modular and simple. 97 | 98 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 99 | 100 | -------------------------------------------------------------------------------- /configuration/service_discovery-section.md: -------------------------------------------------------------------------------- 1 | # Config: Service Discovery Section 2 | 3 | Some of the Fluentd plugins support the `` section to set the target nodes dynamically. 4 | 5 | ## Service Discovery Section Overview 6 | 7 | The **`service_discovery`** section comes under ``. 8 | 9 | ```text 10 | 11 | @type forward 12 | # ... 13 | 14 | # ... 15 | 16 | 17 | ``` 18 | 19 | ## Service Discovery Plugin Type 20 | 21 | The `@type` parameter of `` section specifies the type of the plugin. Fluentd core bundles [some useful service discovery plugins](../service_discovery/) e.g. `file`. 22 | 23 | ```text 24 | 25 | @type file 26 | # ... 27 | 28 | ``` 29 | 30 | Here's the list of built-in service discovery plugins: 31 | 32 | * [`static`](../service_discovery/static.md) 33 | * [`file`](../service_discovery/file.md) 34 | * [`srv`](../service_discovery/srv.md) 35 | 36 | For more details, see plugins documentation. 37 | 38 | ### `@type` 39 | 40 | The `@type` parameter specifies the type of service discovery plugin. 41 | 42 | ```text 43 | 44 | @type static 45 | # ... 46 | 47 | ``` 48 | 49 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 50 | 51 | -------------------------------------------------------------------------------- /configuration/storage-section.md: -------------------------------------------------------------------------------- 1 | # Config: Storage Section 2 | 3 | Some of the Fluentd plugins support the `` section to specify how to handle the plugin's internal states. 4 | 5 | ## Storage Section Overview 6 | 7 | The **`storage`** section can be under ``, `` or `` section. It is enabled for the plugins that support storage plugin features. 8 | 9 | ```text 10 | 11 | @type windows_eventlog 12 | # ... 13 | 14 | # ... 15 | 16 | 17 | ``` 18 | 19 | ## Storage Plugin Type 20 | 21 | The `@type` parameter of `` section specifies the type of the storage plugin. Fluentd core bundles a useful [storage plugin](../storage/). 22 | 23 | ```text 24 | 25 | @type local 26 | 27 | ``` 28 | 29 | Some `storage` plugins may have argument\(s\) in `` section: 30 | 31 | ```text 32 | 33 | @type local 34 | 35 | ``` 36 | 37 | Third-party plugins may also be installed and configured. 38 | 39 | For more details, see plugins documentation. 40 | 41 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 42 | 43 | -------------------------------------------------------------------------------- /container-deployment/README.md: -------------------------------------------------------------------------------- 1 | # Container Deployment 2 | 3 | -------------------------------------------------------------------------------- /deployment/README.md: -------------------------------------------------------------------------------- 1 | # Deployment 2 | 3 | -------------------------------------------------------------------------------- /deployment/failure-scenarios.md: -------------------------------------------------------------------------------- 1 | # Failure Scenarios 2 | 3 | This article describes various Fluentd failure scenarios. We will assume that you have configured Fluentd for [High Availability](high-availability.md), so that each app node has its local **forwarders** and all logs are aggregated into multiple **aggregators**. 4 | 5 | ## Apps Cannot Post Records to Forwarder 6 | 7 | The application sometimes fails to post records to its local Fluentd instance when using logger libraries of various languages. Depending on the maturity of each logger library, some clever mechanisms have been implemented to prevent data loss. 8 | 9 | ### 1. Memory Buffering \(available for [Ruby](../language-bindings/ruby.md), [Java](../language-bindings/java.md), [Python](../language-bindings/python.md), [Perl](../language-bindings/perl.md)\) 10 | 11 | If the destination Fluentd instance dies, certain logger implementations will use extra memory to hold the incoming logs. When Fluentd comes back alive, these loggers will automatically retry to send the buffered logs to Fluentd again. Once the maximum buffer memory size is reached, most current implementations will write the data onto the disk or throw away the logs. 12 | 13 | ### 2. Exponential Backoff \(available for [Ruby](../language-bindings/ruby.md), [Java](../language-bindings/java.md)\) 14 | 15 | When trying to resend logs to the local forwarder, some implementations will use exponential backoff to prevent excessive re-connect requests. 16 | 17 | ## Forwarder or Aggregator Fluentd Goes Down 18 | 19 | What happens when a Fluentd process dies for any reason? It depends on your buffer configuration. 20 | 21 | ### `buf_memory` 22 | 23 | If you are using [`buf_memory`](../buffer/memory.md), the buffered data is completely lost. This is a tradeoff for higher performance. Lowering the `flush_interval` will reduce the probability of data loss, but will increase the number of transfers between forwarders and aggregators. 24 | 25 | ### `buf_file` 26 | 27 | If you are using [`buf_file`](../buffer/file.md), the buffered data is stored on the disk. After Fluentd recovers, it will try to send the buffered data to the destination again. 28 | 29 | Please note that the data will be lost if the buffer file is broken due to I/O errors. The data will also be lost if the disk is full. 30 | 31 | ## Storage Destination Goes Down 32 | 33 | If the storage destination \(e.g. Amazon S3, MongoDB, HDFS, etc.\) goes down, Fluentd will keep trying to resend the buffered data. The retry logic depends on the plugin implementation. 34 | 35 | If you're using [`buf_memory`](../buffer/memory.md), the aggregators will stop accepting new logs once they reach their buffer limits. If you are using [`buf_file`](../buffer/file.md), the aggregators will continue accepting logs until they run out of disk space. 36 | 37 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 38 | 39 | -------------------------------------------------------------------------------- /deployment/fluentd-ui.md: -------------------------------------------------------------------------------- 1 | # Fluentd UI 2 | 3 | [fluentd-ui](https://github.com/fluent/fluentd-ui) is a browser-based [fluentd](http://fluentd.org/) and [td-agent](https://docs.treasuredata.com/display/public/PD/About+Treasure+Data%27s+Server-Side+Agent) manager that supports the following operations: 4 | 5 | * Install, uninstall, and upgrade Fluentd plugins 6 | * Start/stop/restart fluentd process 7 | * Configure Fluentd settings such as config file, pid file path, etc. 8 | * View Fluentd log with simple error viewer 9 | 10 | `fluentd-ui` does not work with `fluentd` v1 and `td-agent` 3 and 4 do not include it. This content is for v0.12 for now. 11 | 12 | ## Enterprise 13 | 14 | [Chronosphere Telemetry Pipeline](https://chronosphere.io/platform/telemetry-pipeline/) provides an enterprise UI on top of Fluentd and Fluent Bit. Please visit their site for more information. 15 | 16 | 17 | ## Getting Started 18 | 19 | For `td-agent`, you can start it by `td-agent-ui start` like this: 20 | 21 | ```text 22 | $ sudo /usr/sbin/td-agent-ui start 23 | Puma 2.9.2 starting... 24 | * Min threads: 0, max threads: 16 25 | * Environment: production 26 | * Listening on tcp://0.0.0.0:9292 27 | ``` 28 | 29 | For `fluentd` gem installation, install `fluentd-ui` via `gem` command: 30 | 31 | ```text 32 | $ gem install -V fluentd-ui 33 | $ fluentd-ui start 34 | Puma 2.9.2 starting... 35 | * Min threads: 0, max threads: 16 36 | * Environment: production 37 | * Listening on tcp://0.0.0.0:9292 38 | ``` 39 | 40 | Then, open [http://localhost:9292/](http://localhost:9292/) in your browser. 41 | 42 | The default account credentials are: 43 | 44 | * `username="admin"` 45 | * `password="changeme"` 46 | 47 |  48 | 49 | ## Screenshots 50 | 51 | \(v0.3.9\) 52 | 53 | ### Dashboard 54 | 55 |  56 | 57 | ### Setting 58 | 59 |  60 | 61 | ### `in_tail` setting 62 | 63 |  64 | 65 | ### Plugin 66 | 67 |  68 | 69 |  70 | 71 |  72 | 73 |  74 | 75 |  76 | 77 |  78 | 79 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 80 | 81 | -------------------------------------------------------------------------------- /deployment/rpc.md: -------------------------------------------------------------------------------- 1 | # RPC 2 | 3 | HTTP RPC enables you to manage your Fluentd instance through HTTP endpoints. You can use this feature as a replacement of [Unix signals](signals.md). 4 | 5 | It is especially useful for environments where signals are not supported well e.g. Windows. This requires Fluentd to start not with `--no-supervisor` command-line option. 6 | 7 | ## Configuration 8 | 9 | By default, HTTP RPC is not enabled. To use this feature, set the `rpc_endpoint` option: 10 | 11 | ```text 12 | 13 | rpc_endpoint 127.0.0.1:24444 14 | 15 | ``` 16 | 17 | Now, you can manage your Fluentd instance using an HTTP client: 18 | 19 | ```text 20 | $ curl http://127.0.0.1:24444/api/plugins.flushBuffers 21 | {"ok":true} 22 | ``` 23 | 24 | As evident from the output above, each endpoint returns a JSON object as its response. 25 | 26 | ## HTTP Endpoints 27 | 28 | | Endpoint | Replacement of | Description | Version | 29 | | :--- | :---: | :---: | :---: | 30 | | `/api/processes.interruptWorkers` | [SIGINT](signals.md#sigint-or-sigterm) | Stops the daemon. | v1.0 | 31 | | `/api/processes.killWorkers` | [SIGTERM](signals.md#sigint-or-sigterm) | Stops the daemon. | v1.0 | 32 | | `/api/processes.zeroDowntimeRestart` | [SIGUSR2](signals.md#sigusr2) | Restarts Fluentd with zero-downtime. (Not supported on Windows) | v1.18 | 33 | | `/api/processes.flushBuffersAndKillWorkers` | [SIGUSR1](signals.md#sigusr1) and [SIGTERM](signals.md#sigint-or-sigterm) | Flushes buffer and stops the daemon. | v1.0 | 34 | | `/api/plugins.flushBuffers` | [SIGUSR1](signals.md#sigusr1) | Flushes the buffered messages. | v1.0 | 35 | | `/api/config.reload` | [SIGHUP](signals.md#sighup) | Reloads configuration. | v1.0 | 36 | | `/api/config.gracefulReload` | --- | Reloads configuration. | v1.9 | 37 | 38 | Appendix: 39 | 40 | * `/api/config.gracefulReload`: This is the replacement of `SIGUSR2` before v1.18. Please use `/api/processes.zeroDowntimeRestart` or `/api/config.reload` unless there is a special reason. See [SIGUSR2](signals.md#sigusr2) for details. 41 | 42 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 43 | 44 | -------------------------------------------------------------------------------- /deployment/zero-downtime-restart.md: -------------------------------------------------------------------------------- 1 | # Zero-downtime restart 2 | 3 | This feature supports a complete restart of Fluentd. 4 | This restarts Fluentd so that some input plugins don't have down time. 5 | 6 | Supported standard input plugins are as follows. 7 | 8 | | supported input plugin | version | 9 | | :--- | :--- | 10 | | in_udp | v1.18.0 | 11 | | in_tcp | v1.18.0 | 12 | | in_syslog | v1.18.0 | 13 | 14 | If these input plugins are down, client applications may fail to send data. 15 | You can use this feature to completely restart Fluentd without losing data for these plugins even if that client does not have a resend feature. 16 | 17 | ## How to use this feature 18 | 19 | You can use this feature in the following ways. 20 | 21 | * [Signals - SIGUSR2](signals.md#sigusr2) 22 | * [RPC](rpc.md) 23 | 24 | ## Mechanism of zero-downtime restart 25 | 26 |  27 | 28 | 1. Receive `SIGUSR2`. 29 | 2. Spawn a new supervisor. 30 | 3. Take over shared sockets. 31 | 4. Launch new workers, and stop old processes in parallel. 32 | * Launch new workers with [Source Only Mode](source-only-mode.md). 33 | * In addition to the source-only mode limitation, Fluentd further limits the starting plugins to only those that support this feature. 34 | * Data received by the new workers are stored in the temporary buffer of source-only mode. 35 | * For details on the temporary buffer, see [Source Only Mode - Temporary file buffer](source-only-mode.md#temporary-file-buffer). 36 | * Send `SIGTERM` to the old supervisor after `10s` delay. 37 | 5. The old supervisor stops and sends `SIGWINCH` to the new one. 38 | 6. The new workers starts to run fully. 39 | * The temporary buffer of source-only mode starts to load. 40 | 41 | ## Plugins: how to support this feature 42 | 43 | See [How to Write Input Plugin - zero_downtime_restart_ready?](../plugin-development/api-plugin-input.md#zero_downtime_restart_ready). 44 | 45 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 46 | -------------------------------------------------------------------------------- /filter/README.md: -------------------------------------------------------------------------------- 1 | # Filter Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](../input/) 6 | * [Parser](../parser/) 7 | * [Filter](./) 8 | * [Output](../output/) 9 | * [Formatter](../formatter/) 10 | * [Storage](../storage/) 11 | * [Service Discovery](../service_discovery/) 12 | * [Buffer](../buffer/) 13 | * [Metrics](../metrics/) 14 | 15 | This article gives an overview of the Filter Plugin. 16 | 17 | ## Overview 18 | 19 | Filter plugins enable Fluentd to modify event streams. Some use cases are: 20 | 21 | 1. Filtering out events by grepping the value of one or more fields. 22 | 2. Enriching events by adding new fields. 23 | 3. Deleting or masking certain fields for privacy and compliance. 24 | 25 | ## How To Use 26 | 27 | It is used with the `` directive: 28 | 29 | ```text 30 | 31 | @type grep 32 | regexp1 message cool 33 | 34 | ``` 35 | 36 | The above directive matches events with the tag `foo.bar`, and if the `message` field's value contains `cool`, the events go through the rest of the configuration. 37 | 38 | Like the `` directive for output plugins, `` matches against a tag. Once the event is processed by the filter, the event proceeds through the configuration top-down. Hence, if there are multiple filters for the same tag, they are applied in descending order. Hence, in the following example: 39 | 40 | ```text 41 | 42 | @type grep 43 | regexp1 message cool 44 | 45 | 46 | 47 | @type record_transformer 48 | 49 | hostname "#{Socket.gethostname}" 50 | 51 | 52 | ``` 53 | 54 | Only the events whose `message` field contain `cool` get the new field `hostname` with the machine's hostname as its value. 55 | 56 | Users can create their own custom plugins with a bit of Ruby. See [this section](../plugin-development/#filter-plugins) for more information. 57 | 58 | ## Filter Chain Optimization 59 | 60 | If you have multiple filters in the pipeline, fluentd tries to optimize filter calls to improve the performance. The condition for optimization is that all plugins in the pipeline use the `filter` method. If the plugin which uses `filter_stream` exists, chain optimization is disabled. 61 | 62 | If you see the following message in the log, the optimization is disabled: 63 | 64 | ```text 65 | disable filter chain optimization because [Fluent::Plugin::XXXFilter] uses `#filter_stream` method 66 | ``` 67 | 68 | This is not a critical log message and you can ignore it. 69 | 70 | ## List of Filter Plugins 71 | 72 | * [`grep`](grep.md) 73 | * [`record_transformer`](record_transformer.md) 74 | * [`filter_stdout`](stdout.md) 75 | 76 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 77 | 78 | -------------------------------------------------------------------------------- /filter/stdout.md: -------------------------------------------------------------------------------- 1 | # stdout 2 | 3 | The `filter_stdout` filter plugin prints events to the standard output \(or logs if launched as a daemon\). This filter plugin is useful for debugging purposes. 4 | 5 | It is included in the Fluentd's core. 6 | 7 | ## Example Configurations 8 | 9 | ```text 10 | 11 | @type stdout 12 | 13 | ``` 14 | 15 | A sample output is as follows: 16 | 17 | ```text 18 | 2017-11-28 11:43:13.814351757 +0900 tag: {"field1":"value1","field2":"value2"} 19 | ``` 20 | 21 | where the first part shows the output **time**, the second part shows the **tag**, and the third part shows the **record**. 22 | 23 | ## Plugin Helpers 24 | 25 | * [`formatter`](../plugin-helper-overview/api-plugin-helper-formatter.md) 26 | * [`compat_parameters`](../plugin-helper-overview/api-plugin-helper-compat_parameters.md) 27 | * [`inject`](../plugin-helper-overview/api-plugin-helper-inject.md) 28 | 29 | ## Parameters 30 | 31 | See [Common Parameters](../configuration/plugin-common-parameters.md). 32 | 33 | ### `@type` \(required\) 34 | 35 | The value must be `stdout`. 36 | 37 | ### `` Section 38 | 39 | See [Format section configurations](../configuration/format-section.md) for more details. 40 | 41 | #### `@type` 42 | 43 | | type | default | version | 44 | | :--- | :--- | :--- | 45 | | string | stdout | 0.14.5 | 46 | 47 | The format of the output. 48 | 49 | #### `output_type` 50 | 51 | | type | default | version | 52 | | :--- | :--- | :--- | 53 | | string | json | 0.14.5 | 54 | 55 | This is the option of `stdout` format. Configures the format of the record \(third part\). Any formatter plugins can be specified. 56 | 57 | ### `` Section 58 | 59 | See [Inject Section Configurations](../configuration/inject-section.md) for more details. 60 | 61 | ## Learn More 62 | 63 | * [Filter Plugin Overview](./) 64 | 65 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 66 | 67 | -------------------------------------------------------------------------------- /formatter/README.md: -------------------------------------------------------------------------------- 1 | # Formatter Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](../input/) 6 | * [Parser](../parser/) 7 | * [Filter](../filter/) 8 | * [Output](../output/) 9 | * [Formatter](./) 10 | * [Storage](../storage/) 11 | * [Service Discovery](../service_discovery/) 12 | * [Buffer](../buffer/) 13 | * [Metrics](../metrics/) 14 | 15 | This article gives an overview of the Formatter Plugin. 16 | 17 | ## Overview 18 | 19 | Sometimes, the output format for an output plugin does not meet one's needs. Fluentd has a pluggable system called Formatter that lets the user extend and reuse custom output formats. 20 | 21 | ## How To Use 22 | 23 | For an output plugin that supports Formatter, the `` directive can be used to change the output format. 24 | 25 | For example, by default, [`out_file`](../output/file.md) plugin outputs data as 26 | 27 | ```text 28 | 2014-08-25 00:00:00 +0000foo.bar{"k1":"v1", "k2":"v2"} 29 | ``` 30 | 31 | However, if you set `@type json` in `` like this: 32 | 33 | ```text 34 | 35 | @type file 36 | path /path/to/file 37 | 38 | @type json 39 | 40 | 41 | ``` 42 | 43 | The output changes to 44 | 45 | ```javascript 46 | {"k1":"v1", "k2":"v2"} 47 | ``` 48 | 49 | i.e., each line is a single JSON object without "time" and "tag" fields. If you want to include "time" and "tag", use [Inject section](../configuration/inject-section.md). 50 | 51 | See [this section](../plugin-development/#text-formatter-plugins) to learn how to develop a custom formatter. 52 | 53 | ## List of Built-in Formatters 54 | 55 | * [`out_file`](out_file.md) 56 | * [`json`](json.md) 57 | * [`ltsv`](ltsv.md) 58 | * [`csv`](csv.md) 59 | * [`msgpack`](msgpack.md) 60 | * [`hash`](hash.md) 61 | * [`single_value`](single_value.md) 62 | * [`tsv`](tsv.md) 63 | 64 | ## List of Output/Filter Plugins with Formatter Support 65 | 66 | * [`filter_stdout`](../filter/stdout.md) 67 | * [`out_file`](../output/file.md) 68 | * [`out_s3`](../output/s3.md) 69 | 70 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 71 | 72 | -------------------------------------------------------------------------------- /formatter/csv.md: -------------------------------------------------------------------------------- 1 | # csv 2 | 3 | The `csv` formatter plugin output an event as CSV. 4 | 5 | ```text 6 | "value1"[delimiter]"value2"[delimiter]"value3"[newline] 7 | ``` 8 | 9 | ## Parameters 10 | 11 | * [Common Parameters](../configuration/plugin-common-parameters.md) 12 | * [Format section configurations](../configuration/format-section.md) 13 | 14 | ### `fields` 15 | 16 | | type | default | version | 17 | | :--- | :--- | :--- | 18 | | array of string | `nil` | 0.14.0 | 19 | 20 | Specifies the output fields. It is a required parameter. 21 | 22 | ### delimiter \(String, Optional. defaults to ","\) 23 | 24 | | type | default | version | 25 | | :--- | :--- | :--- | 26 | | string | `,` | 0.14.0 | 27 | 28 | Delimiter for values. 29 | 30 | Use `\t` or `TAB` to specify the tab character. 31 | 32 | ### `force_quotes` 33 | 34 | | type | default | version | 35 | | :--- | :--- | :--- | 36 | | bool | `true` | 0.14.0 | 37 | 38 | If `false`, the value will not be framed by quotes. 39 | 40 | ### `add_newline` 41 | 42 | | type | default | version | 43 | | :--- | :--- | :--- | 44 | | bool | `true` | 0.14.12 | 45 | 46 | Add `\n` to the result. 47 | 48 | ## Example 49 | 50 | ```text 51 | 52 | @type csv 53 | fields host,method 54 | 55 | ``` 56 | 57 | With this configuration: 58 | 59 | ```text 60 | tag: app.event 61 | time: 1362020400 62 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 63 | ``` 64 | 65 | This incoming event is formatted to: 66 | 67 | In non-Windows: 68 | 69 | ```text 70 | "192.168.0.1","PUT"\n 71 | ``` 72 | 73 | In Windows: 74 | 75 | ```text 76 | "192.168.0.1","PUT"\r\n 77 | ``` 78 | 79 | With `force_quotes false`, the result is: 80 | 81 | In non-Windows: 82 | 83 | ```text 84 | 192.168.0.1,PUT\n 85 | ``` 86 | 87 | In Windows: 88 | 89 | ```text 90 | 192.168.0.1,PUT\r\n 91 | ``` 92 | 93 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 94 | 95 | -------------------------------------------------------------------------------- /formatter/hash.md: -------------------------------------------------------------------------------- 1 | # hash 2 | 3 | The `hash` formatter plugin converts an event to the Ruby hash. 4 | 5 | By default, `hash` formatter result doesn't contain `tag` and `time` fields. 6 | 7 | ## Parameters 8 | 9 | * [Common Parameters](../configuration/plugin-common-parameters.md) 10 | * [Format section configurations](../configuration/format-section.md) 11 | 12 | ### `add_newline` 13 | 14 | | type | default | version | 15 | | :--- | :--- | :--- | 16 | | bool | `true` | 0.14.12 | 17 | 18 | Adds `\n` to the result. 19 | 20 | ### `newline` 21 | 22 | | type | default | version | 23 | | :--- | :--- | :--- | 24 | | enum | `lf (for non-Windows)` or `crlf (for Windows)` | 1.11.5 | 25 | 26 | Specify newline characters. 27 | 28 | ## Example 29 | 30 | ```text 31 | tag: app.event 32 | time: 1362020400 33 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 34 | ``` 35 | 36 | This incoming event is formatted to: 37 | 38 | ```ruby 39 | {"host"=>"192.168.0.1","size"=>777,"method"=>"PUT"} 40 | ``` 41 | 42 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 43 | 44 | -------------------------------------------------------------------------------- /formatter/json.md: -------------------------------------------------------------------------------- 1 | # json 2 | 3 | The `json` formatter plugin format an event to JSON. 4 | 5 | By default, `json` formatter result doesn't contain `tag` and `time` fields. 6 | 7 | ## Parameters 8 | 9 | * [Common Parameters](../configuration/plugin-common-parameters.md) 10 | * [Format section configurations](../configuration/format-section.md) 11 | 12 | ### `add_newline` 13 | 14 | | type | default | version | 15 | | :--- | :--- | :--- | 16 | | bool | `true` | 0.14.12 | 17 | 18 | Adds `\n` to the result. 19 | 20 | ### `newline` 21 | 22 | | type | default | version | 23 | | :--- | :--- | :--- | 24 | | enum | `lf (for non-Windows)` or `crlf (for Windows)` | 1.11.5 | 25 | 26 | Specify newline characters. 27 | 28 | ## Example 29 | 30 | ```text 31 | tag: app.event 32 | time: 1362020400 33 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 34 | ``` 35 | 36 | This incoming event is formatted to: 37 | 38 | ```javascript 39 | {"host":"192.168.0.1","size":777,"method":"PUT"} 40 | ``` 41 | 42 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 43 | 44 | -------------------------------------------------------------------------------- /formatter/ltsv.md: -------------------------------------------------------------------------------- 1 | # ltsv 2 | 3 | The `ltsv` formatter plugin output an event as [LTSV](http://ltsv.org). 4 | 5 | ```text 6 | field1[label_delimiter]value1[delimiter]field2[label_delimiter]value2[newline] 7 | ``` 8 | 9 | ## Parameters 10 | 11 | * [Common Parameters](../configuration/plugin-common-parameters.md) 12 | * [Format section configurations](../configuration/format-section.md) 13 | 14 | ### `delimiter` 15 | 16 | | type | default | version | 17 | | :--- | :--- | :--- | 18 | | string | `\t` \(TAB\) | 0.14.0 | 19 | 20 | Delimiter for the fields. 21 | 22 | ### `label_delimiter` 23 | 24 | | type | default | version | 25 | | :--- | :--- | :--- | 26 | | string | `:` | 0.14.0 | 27 | 28 | Delimiter for the key-value field. 29 | 30 | ### `add_newline` 31 | 32 | | type | default | version | 33 | | :--- | :--- | :--- | 34 | | bool | `true` | 0.14.12 | 35 | 36 | Adds `\n` to the result. 37 | 38 | ### `newline` 39 | 40 | | type | default | version | 41 | | :--- | :--- | :--- | 42 | | enum | `lf (for non-Windows)` or `crlf (for Windows)` | 1.11.5 | 43 | 44 | Specify newline characters. 45 | 46 | ### `replacement` 47 | 48 | | type | default | version | 49 | | :--- | :--- | :--- | 50 | | string | \`\` \(SPACE\) | 1.12.2 | 51 | 52 | Safe replacement to substitute delimiters characters in records. 53 | 54 | ## Example 55 | 56 | ```text 57 | tag: app.event 58 | time: 1362020400 59 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 60 | ``` 61 | 62 | This incoming event is formatted to: 63 | 64 | In non-Windows: 65 | 66 | ```text 67 | host:192.168.0.1\tsize:777\tmethod:PUT\n 68 | ``` 69 | 70 | In Windows: 71 | 72 | ```text 73 | host:192.168.0.1\tsize:777\tmethod:PUT\r\n 74 | ``` 75 | 76 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 77 | 78 | -------------------------------------------------------------------------------- /formatter/msgpack.md: -------------------------------------------------------------------------------- 1 | # msgpack 2 | 3 | The `msgpack` formatter plugin converts an event to the msgpack binary. 4 | 5 | ## Parameters 6 | 7 | * [Common Parameters](../configuration/plugin-common-parameters.md) 8 | * [Format section configurations](../configuration/format-section.md) 9 | 10 | ## Example 11 | 12 | ```text 13 | tag: app.event 14 | time: 1362020400 15 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 16 | ``` 17 | 18 | This incoming event is formatted to: 19 | 20 | ```text 21 | \x83\xA4host\xAB192.168.0.1\xA4size\xCD\x03\t\xA6method\xA3PUT 22 | ``` 23 | 24 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 25 | 26 | -------------------------------------------------------------------------------- /formatter/out_file.md: -------------------------------------------------------------------------------- 1 | # out\_file 2 | 3 | The `out_file` formatter plugin outputs time, tag and json record separated by a delimiter. 4 | 5 | ```text 6 | time[delimiter]tag[delimiter]record[newline] 7 | ``` 8 | 9 | This format is a default format of `out_file` plugin. 10 | 11 | ## Parameters 12 | 13 | * [Common Parameters](../configuration/plugin-common-parameters.md) 14 | * [Format section configurations](../configuration/format-section.md) 15 | 16 | ### `delimiter` 17 | 18 | | type | default | version | 19 | | :--- | :--- | :--- | 20 | | string | `\t` \(TAB\) | 0.14.0 | 21 | 22 | Delimiter for each field. 23 | 24 | Supported: SPACE \(`' '`\) and COMMA \(`','`\) 25 | 26 | ### `output_tag` 27 | 28 | | type | default | version | 29 | | :--- | :--- | :--- | 30 | | bool | true | 0.14.0 | 31 | 32 | Output tag field if `true`. 33 | 34 | ### `output_time` 35 | 36 | | type | default | version | 37 | | :--- | :--- | :--- | 38 | | bool | true | 0.14.0 | 39 | 40 | Output time field if `true`. 41 | 42 | ### `time_type` 43 | 44 | | type | default | version | 45 | | :--- | :--- | :--- | 46 | | enum | string | 0.14.7 | 47 | 48 | Overwrites the default value in this plugin. 49 | 50 | ### `time_format` 51 | 52 | | type | default | version | 53 | | :--- | :--- | :--- | 54 | | string | nil\(iso8601\) | 0.14.7 | 55 | 56 | Overwrites the default value in this plugin. 57 | 58 | ### `newline` 59 | 60 | | type | default | version | 61 | | :--- | :--- | :--- | 62 | | enum | `lf (for non-Windows)` or `crlf (for Windows)` | 1.11.5 | 63 | 64 | Specify newline characters. 65 | 66 | ## Example 67 | 68 | ```text 69 | tag: app.event 70 | time: 1362020400t 71 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 72 | ``` 73 | 74 | This incoming event is formatted to: 75 | 76 | ```text 77 | 2013-02-28T12:00:00+09:00\tapp.event\t{"host":"192.168.0.1","size":777,"method":"PUT"} 78 | ``` 79 | 80 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 81 | 82 | -------------------------------------------------------------------------------- /formatter/single_value.md: -------------------------------------------------------------------------------- 1 | # single\_value 2 | 3 | The `single_value` formatter plugin output the value of a single field instead of the whole record. 4 | 5 | This formatter is often used in conjunction with [`none` parser](../parser/none.md) in input plugin. 6 | 7 | ## Parameters 8 | 9 | * [Common Parameters](../configuration/plugin-common-parameters.md) 10 | * [Format section configurations](../configuration/format-section.md) 11 | 12 | ### `add_newline` 13 | 14 | | type | default | version | 15 | | :--- | :--- | :--- | 16 | | bool | `true` | 0.14.0 | 17 | 18 | Adds `\n` to the result. If there is a trailing `\n` already, set it `false`. 19 | 20 | ### `message_key` 21 | 22 | | type | default | version | 23 | | :--- | :--- | :--- | 24 | | string | message | 0.14.0 | 25 | 26 | The value of this field is outputted. 27 | 28 | ### `newline` 29 | 30 | | type | default | version | 31 | | :--- | :--- | :--- | 32 | | enum | `lf (for non-Windows)` or `crlf (for Windows)` | 1.11.5 | 33 | 34 | Specify newline characters. 35 | 36 | ## Example 37 | 38 | ```text 39 | tag: app.event 40 | time: 1362020400 41 | record: {"message":"Hello from Fluentd!"} 42 | ``` 43 | 44 | This incoming event is formatted to: 45 | 46 | In non-Windows: 47 | 48 | ```text 49 | Hello from Fluentd!\n 50 | ``` 51 | 52 | In Windows: 53 | 54 | ```text 55 | Hello from Fluentd!\r\n 56 | ``` 57 | 58 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 59 | 60 | -------------------------------------------------------------------------------- /formatter/stdout.md: -------------------------------------------------------------------------------- 1 | # stdout 2 | 3 | The `stdout` formatter plugin converts an event to the stdout format. 4 | 5 | This plugin is only for [`stdout` Output Plugin](../output/stdout.md) and [`stdout` Filter Plugin](../filter/stdout.md). 6 | 7 | ```text 8 | 2015-05-02 12:12:17 +0900 tag: {"field1":"value1","field2":"value2"} 9 | ``` 10 | 11 | ## Parameters 12 | 13 | * [Common Parameters](../configuration/plugin-common-parameters.md) 14 | * [Format section configurations](../configuration/format-section.md) 15 | 16 | ### `output_type` \(string\) \(optional\) 17 | 18 | | type | default | version | 19 | | :--- | :--- | :--- | 20 | | string | `json` | 0.14.0 | 21 | 22 | Sets the sub-formatter type. Any formatter plugins can be specified. 23 | 24 | ## Example 25 | 26 | ```text 27 | 28 | @type stdout 29 | output_type json 30 | 31 | ``` 32 | 33 | with this configuration: 34 | 35 | ```text 36 | tag: app.event 37 | time: 1511156652 38 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 39 | ``` 40 | 41 | This incoming event is formatted to: 42 | 43 | ```text 44 | 2017-11-20 14:44:12 +0900 app.event: {"host":"192.168.0.1","size":777,"method":"PUT"} 45 | ``` 46 | 47 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 48 | 49 | -------------------------------------------------------------------------------- /formatter/tsv.md: -------------------------------------------------------------------------------- 1 | # tsv 2 | 3 | The `tsv` formatter plugin converts an event as TSV. 4 | 5 | ```text 6 | value1[delimiter]value2[newline] 7 | ``` 8 | 9 | ## Parameters 10 | 11 | * [Common Parameters](../configuration/plugin-common-parameters.md) 12 | * [Format section configurations](../configuration/format-section.md) 13 | 14 | ### `keys` 15 | 16 | | type | default | version | 17 | | :--- | :--- | :--- | 18 | | array of string | `nil` | 0.14.0 | 19 | 20 | Specifies the output fields. It is a required parameter. 21 | 22 | ### `delimiter` 23 | 24 | | type | default | version | 25 | | :--- | :--- | :--- | 26 | | string | `\t` \(TAB\) | 0.14.0 | 27 | 28 | Delimiter for the fields. 29 | 30 | ### `add_newline` 31 | 32 | | type | default | version | 33 | | :--- | :--- | :--- | 34 | | bool | `true` | 0.14.12 | 35 | 36 | Adds `\n` to the result. 37 | 38 | ### `newline` 39 | 40 | | type | default | version | 41 | | :--- | :--- | :--- | 42 | | enum | `lf (for non-Windows)` or `crlf (for Windows)` | 1.11.5 | 43 | 44 | Specify newline characters. 45 | 46 | ## Example 47 | 48 | ```text 49 | tag: app.event 50 | time: 1362020400 51 | record: {"host":"192.168.0.1","size":777,"method":"PUT"} 52 | ``` 53 | 54 | This incoming event is formatted to: 55 | 56 | In non-Windows: 57 | 58 | ```text 59 | 192.168.0.1\t777\tPUT\n 60 | ``` 61 | 62 | In non-Windows: 63 | 64 | ```text 65 | 192.168.0.1\t777\tPUT\r\n 66 | ``` 67 | 68 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 69 | 70 | -------------------------------------------------------------------------------- /how-to-guides/README.md: -------------------------------------------------------------------------------- 1 | # How-to Guides 2 | 3 | -------------------------------------------------------------------------------- /how-to-guides/filter-modify-apache.md: -------------------------------------------------------------------------------- 1 | # Simple Stream Processing with Fluentd 2 | 3 | Sometimes, you need to transform the data stream in a certain way. For example, you might want to extract a portion of data for error reporting, or need to append additional information to events for later inspection. 4 | 5 | This article explains the common data manipulation techniques in detail. 6 | 7 | ## How to Filter Events by Fields 8 | 9 | [`filter_grep`](../filter/grep.md) is a built-in plugin that allows to filter the data stream using regular expressions. 10 | 11 | Suppose you are managing a web service, and try to monitor the access logs using Fluentd. In this case, an event in the data stream will look like: 12 | 13 | ```text 14 | { 15 | "host": "192.168.1.1", 16 | "method": "GET", 17 | "path": "/index.html", 18 | "code": 200, 19 | "size": 2344, 20 | "referer": null 21 | } 22 | ``` 23 | 24 | Let's filter out all the successful responses with 2xx status codes so that we can easily inspect if any error has occurred in the service: 25 | 26 | ```text 27 | 28 | @type grep 29 | 30 | key code 31 | pattern ^2\d\d$ 32 | 33 | 34 | ``` 35 | 36 | You can also filter the data using multiple fields. The following configuration will keep all 5xx server errors, except those coming from the test directory: 37 | 38 | ```text 39 | 40 | @type grep 41 | 42 | key code 43 | pattern ^5\d\d$ 44 | 45 | 46 | key path 47 | pattern ^/test/ 48 | 49 | 50 | ``` 51 | 52 | ## How to Inject Custom Fields into Events 53 | 54 | [`filter_record_transformer`](../filter/record_transformer.md) is a built-in plugin that enables it to inject arbitrary data into events. 55 | 56 | Suppose you are running a web service on multiple web servers, and you want to record which server handled each request. This can be implemented trivially using this plugin: 57 | 58 | ```text 59 | 60 | @type record_transformer 61 | 62 | server "${hostname}" 63 | 64 | 65 | ``` 66 | 67 | This will produce an event like this: 68 | 69 | ```text 70 | { 71 | "host": "192.168.1.1", 72 | "method": "GET", 73 | "path": "/index.html", 74 | "code": 200, 75 | "size": 2344, 76 | "referer": null, 77 | "server": "app1" 78 | } 79 | ``` 80 | 81 | Note that `${hostname}` is a predefined variable supplied by the plugin. You can also define a custom variable, or even evaluate arbitrary ruby expressions. For more details, see [`record_transformer`](../filter/record_transformer.md). 82 | 83 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 84 | 85 | -------------------------------------------------------------------------------- /how-to-guides/stream-analytics-with-materialize.md: -------------------------------------------------------------------------------- 1 | # Stream Analytics with Materialize 2 | 3 | Fluentd and Fluent bit are vendor-neutral open-source log and metric collectors that are used throughout enterprises today. The plugins already include analytic backends like Elasticsearch, Splunk, and Datadog today. One of the newest integrations with Fluentd and Fluent Bit is the new streaming database, Materialize. Built on the open-source project, Timely Dataflow, Users can use standard SQL on top of vast amounts of streaming data to build low-latency, continually refreshed views across multiple sources of incoming data. Together these projects allow you to easily collect your data wherever it may be and analyze it in real-time, or push it to downstream business visualization systems like Metabase and others. 4 | 5 | ### **Advantages of using Fluentd / Fluent Bit** 6 | 7 | 1. Connect to data sources such as infrastructure, network, application, and system logs 8 | 2. Route important data for analysis, and remove unnecessary data 9 | 3. Use built-in reliability features to ensure data is collected and routed with error handling and persistent disk storage 10 | 11 | ### **Advantages of using Materialize** 12 | 13 | 1. Supports Postgres-formatted ANSI-standard SQL, allowing reuse of existing tools 14 | 2. Low-latency, incrementally updated views 15 | 3. Join data across multiple streams and datasets 16 | 17 | ### **How to get started?** 18 | 19 |  20 | 21 | If you’d like to start playing with this, here are step-by-step instructions to get setup in 15 minutes or less: 22 | 23 | 1. [Install FluentBit](https://docs.fluentbit.io/manual/installation/getting-started-with-fluent-bit) or [Fluentd](https://docs.fluentd.org/installation) and use any of the many input plugins to read from Syslog, Network data, Kubernetes, Docker containers, Application log files, and more 24 | 2. Leverage Fluent Bit or Fluentd’s ability to write to JSON with the following output configurations 25 | 26 | 27 | 28 | 29 | Fluent Bit 30 | Fluentd 31 | 32 | 33 | 34 | 35 | 36 | [OUTPUT] 37 | name file 38 | path /var/log/fluent 39 | format plain 40 | match * 41 | 42 | 43 | <match pattern> 44 | @type file 45 | path /var/log/fluent/myapp 46 | </match> 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | * [Install Materialize](https://materialize.com/docs/install/) and create a new JSON file source 55 | 56 | ```text 57 | CREATE SOURCE file_source FROM FILE ‘/var/log/fluent/file’ FORMAT TEXT; 58 | 59 | ``` 60 | 61 | * Start using Materialize on top of all this real-time data 62 | 63 | ```text 64 | CREATE MATERIALIZED VIEW jsonified_text AS 65 | SELECT text::jsonb AS jsonified FROM file_source; 66 | 67 | 68 | SELECT jsonified->>metric_name FROM jsonified_text; 69 | 70 | ?column? 71 | ---------- 72 | 2.0 73 | 2.0 74 | 3.0 75 | (3 rows) 76 | ``` 77 | 78 | -------------------------------------------------------------------------------- /images/fluentd-riak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/fluentd-riak.png -------------------------------------------------------------------------------- /images/logo/Fluentd.ai: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd.ai -------------------------------------------------------------------------------- /images/logo/Fluentd_horizontal_banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_horizontal_banner.png -------------------------------------------------------------------------------- /images/logo/Fluentd_horizontal_blight_300dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_horizontal_blight_300dpi.png -------------------------------------------------------------------------------- /images/logo/Fluentd_horizontal_blight_72dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_horizontal_blight_72dpi.png -------------------------------------------------------------------------------- /images/logo/Fluentd_horizontal_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_horizontal_dark.png -------------------------------------------------------------------------------- /images/logo/Fluentd_horizontal_dark_300dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_horizontal_dark_300dpi.png -------------------------------------------------------------------------------- /images/logo/Fluentd_horizontal_dark_72dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_horizontal_dark_72dpi.png -------------------------------------------------------------------------------- /images/logo/Fluentd_icon_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_icon_dark.png -------------------------------------------------------------------------------- /images/logo/Fluentd_icon_square.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_icon_square.png -------------------------------------------------------------------------------- /images/logo/Fluentd_square_blight_300dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_square_blight_300dpi.png -------------------------------------------------------------------------------- /images/logo/Fluentd_square_blight_72dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_square_blight_72dpi.png -------------------------------------------------------------------------------- /images/logo/Fluentd_square_dark_300dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_square_dark_300dpi.png -------------------------------------------------------------------------------- /images/logo/Fluentd_square_dark_72dpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/logo/Fluentd_square_dark_72dpi.png -------------------------------------------------------------------------------- /images/source/plugin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd-docs-gitbook/HEAD/images/source/plugin.png -------------------------------------------------------------------------------- /images/source/right-arrow.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /input/README.md: -------------------------------------------------------------------------------- 1 | # Input Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](./) 6 | * [Parser](../parser/) 7 | * [Filter](../filter/) 8 | * [Output](../output/) 9 | * [Formatter](../formatter/) 10 | * [Storage](../storage/) 11 | * [Service Discovery](../service_discovery/) 12 | * [Buffer](../buffer/) 13 | * [Metrics](../metrics/) 14 | 15 | This article gives an overview of the Input Plugin. 16 | 17 | ## Overview 18 | 19 | Input plugins extend Fluentd to retrieve and pull event logs from the external sources. An input plugin typically creates a thread, socket, and a listening socket. It can also be written to periodically pull data from the data sources. 20 | 21 | ## List of Input Plugins 22 | 23 | * [`in_tail`](tail.md) 24 | * [`in_forward`](forward.md) 25 | * [`in_udp`](udp.md) 26 | * [`in_tcp`](tcp.md) 27 | * [`in_unix`](unix.md) 28 | * [`in_http`](http.md) 29 | * [`in_syslog`](syslog.md) 30 | * [`in_exec`](exec.md) 31 | * [`in_sample`](sample.md) 32 | * [`in_windows_eventlog`](windows_eventlog.md) 33 | 34 | ## Other Input Plugins 35 | 36 | Refer to this list of available plugins to find out about other Input plugins: 37 | 38 | * [Fluentd plugins](http://fluentd.org/plugin/) 39 | 40 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 41 | 42 | -------------------------------------------------------------------------------- /input/unix.md: -------------------------------------------------------------------------------- 1 | # unix 2 | 3 | The `in_unix` Input plugin enables Fluentd to retrieve records from the Unix Domain Socket. The wire protocol is the same as [`in_forward`](forward.md), but the transport layer is different. 4 | 5 | It is included in Fluentd's core. 6 | 7 | ## Example Configuration 8 | 9 | ```text 10 | 11 | @type unix 12 | path /path/to/socket.sock 13 | 14 | ``` 15 | 16 | Refer to the [Configuration File](../configuration/config-file.md) article for the basic structure and syntax of the configuration file. 17 | 18 | ## Parameters 19 | 20 | ### `@type` \(required\) 21 | 22 | The value must be `unix`. 23 | 24 | ### `path` 25 | 26 | | type | default | version | 27 | | :---: | :---: | :---: | 28 | | string | `/var/run/fluent/fluent.sock` \(see below\) | 0.14.0 | 29 | 30 | The path to your Unix Domain Socket. 31 | 32 | Fluentd will use the environment variable `FLUENT_SOCKET` if defined. 33 | 34 | ### `backlog` 35 | 36 | | type | default | version | 37 | | :---: | :---: | :---: | 38 | | integer | 1024 | 0.14.0 | 39 | 40 | The backlog of Unix Domain Socket. 41 | 42 | ### `tag` 43 | 44 | | type | default | version | 45 | | :--- | :--- | :--- | 46 | | string | nil | 1.11.0 | 47 | 48 | `in_unix` uses incoming event's tag by default. If the `tag` parameter is set, its value is used instead. 49 | 50 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 51 | 52 | -------------------------------------------------------------------------------- /installation/README.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | -------------------------------------------------------------------------------- /installation/install-by-gem.md: -------------------------------------------------------------------------------- 1 | # Install by Ruby Gem 2 | 3 | This article explains how to install Fluentd using Ruby `gem`. 4 | 5 | ## Step 0: Before Installation 6 | 7 | Please follow the [Pre-installation Guide](before-install.md) to configure your OS properly. 8 | 9 | ## Step 1: Install Ruby Interpreter 10 | 11 | Install Ruby `>= 2.7` on your local environment. In addition, install `ruby-dev` package via Package Manager to build native extension gems. 12 | 13 | ## Step 2: Install Fluentd Gem 14 | 15 | Fetch and install the `fluentd` Ruby gem using `gem` command: 16 | 17 | ```text 18 | $ gem install fluentd --no-doc 19 | ``` 20 | 21 | The official RubyGems page is [here](https://rubygems.org/gems/fluentd). 22 | 23 | ## Step 3: Run 24 | 25 | Run the following commands to verify the Fluentd installation: 26 | 27 | ```text 28 | $ fluentd --setup ./fluent 29 | $ fluentd -c ./fluent/fluent.conf -vv & 30 | $ echo '{"json":"message"}' | fluent-cat debug.test 31 | ``` 32 | 33 | The second command starts Fluentd as a daemon. If you want to stop its daemon, you can use `$ pkill -f fluentd`. The last command sends Fluentd a message '{"json":"message"}' with a `debug.test` tag. If the installation is successful, Fluentd will output the following message: 34 | 35 | ```text 36 | 2011-07-10 16:49:50 +0900 debug.test: {"json":"message"} 37 | ``` 38 | 39 | It is highly recommended to set up `ntpd` on the node to prevent invalid timestamps in your logs. 40 | 41 | For large deployments, you must use [`jemalloc`](http://www.canonware.com/jemalloc/) to avoid memory fragmentation. This is already included in the [`rpm`](install-fluent-package/install-by-rpm-fluent-package.md) and [`deb`](install-fluent-package/install-by-deb-fluent-package.md) packages. 42 | 43 | The Fluentd gem does not come with `/etc/init.d/` scripts. You should use Process Management tools such as: 44 | 45 | * [`daemontools`](http://cr.yp.to/daemontools.html) 46 | * [`runit`](http://smarden.org/runit/) 47 | * [`supervisord`](http://supervisord.org/) 48 | * [`upstart`](http://upstart.ubuntu.com/) 49 | * `systemd` 50 | 51 | ## Next Steps 52 | 53 | You are now ready to collect real logs with Fluentd. Refer to the following tutorials on how to collect data from various sources: 54 | 55 | * Basic Configuration 56 | * [Config File](../configuration/config-file.md) 57 | * Application Logs 58 | * [Ruby](../language-bindings/ruby.md), [Java](../language-bindings/java.md), [Python](../language-bindings/python.md), [PHP](../language-bindings/php.md), 59 | 60 | [Perl](../language-bindings/perl.md), [Node.js](../language-bindings/nodejs.md), [Scala](../language-bindings/scala.md) 61 | * Examples 62 | * [Store Apache Log into Amazon S3](../how-to-guides/apache-to-s3.md) 63 | * [Store Apache Log into MongoDB](../how-to-guides/apache-to-mongodb.md) 64 | * [Data Collection into HDFS](../how-to-guides/http-to-hdfs.md) 65 | 66 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 67 | 68 | -------------------------------------------------------------------------------- /installation/install-calyptia-fluentd/README.md: -------------------------------------------------------------------------------- 1 | # calyptia-fluentd 2 | 3 | -------------------------------------------------------------------------------- /installation/install-fluent-package/README.md: -------------------------------------------------------------------------------- 1 | # fluent-package 2 | 3 | -------------------------------------------------------------------------------- /language-bindings/README.md: -------------------------------------------------------------------------------- 1 | # Language Bindings 2 | 3 | -------------------------------------------------------------------------------- /language-bindings/ruby.md: -------------------------------------------------------------------------------- 1 | # Ruby 2 | 3 | The [`fluent-logger-ruby`](http://github.com/fluent/fluent-logger-ruby) library is used to post records from Ruby applications to Fluentd. 4 | 5 | This article explains how to use it. 6 | 7 | ## Prerequisites 8 | 9 | * Basic knowledge of Ruby 10 | * Basic knowledge of Fluentd 11 | * Ruby 2.1 or later 12 | 13 | ## Installing Fluentd 14 | 15 | Please refer to the following document to install Fluentd: 16 | 17 | * [Installation](../installation/) 18 | 19 | ## Modifying the Config File 20 | 21 | Configure Fluentd to use the [`forward`](../input/forward.md) input plugin as its data source: 22 | 23 | ```text 24 | 25 | @type forward 26 | port 24224 27 | 28 | 29 | @type stdout 30 | 31 | ``` 32 | 33 | Restart agent after configuring. 34 | 35 | ```text 36 | # for rpm/deb only 37 | $ sudo /etc/init.d/td-agent restart 38 | 39 | # or systemd 40 | $ sudo systemctl restart td-agent.service 41 | ``` 42 | 43 | ## Using `fluent-logger-ruby` 44 | 45 | Add 'fluent-logger' gem to your `Gemfile`: 46 | 47 | ```text 48 | gem 'fluent-logger', "~> 0.7.1" 49 | ``` 50 | 51 | Initialize and post the records as shown below: 52 | 53 | ```text 54 | require 'fluent-logger' 55 | Fluent::Logger::FluentLogger.open(nil, :host=>'localhost', :port=>24224) 56 | Fluent::Logger.post("fluentd.test.follow", {"from"=>"userA", "to"=>"userB"}) 57 | ``` 58 | 59 | Executing the script will send the logs to Fluentd: 60 | 61 | ```text 62 | $ ruby test.rb 63 | ``` 64 | 65 | The logs should be output to `/var/log/td-agent/td-agent.log` or the standard output of the Fluentd process via the [`stdout`](../output/stdout.md) output plugin. 66 | 67 | ## Production Deployments 68 | 69 | ### Output Plugins 70 | 71 | Various [output plugins](../output/) are available for writing records to other destinations: 72 | 73 | * Examples 74 | * [Store Apache Logs into Amazon S3](../how-to-guides/apache-to-s3.md) 75 | * [Store Apache Logs into MongoDB](../how-to-guides/apache-to-mongodb.md) 76 | * [Data Collection into HDFS](../how-to-guides/http-to-hdfs.md) 77 | * List of Plugin References 78 | * [Output to Another Fluentd](../output/forward.md) 79 | * [Output to MongoDB](../output/mongo.md) or [MongoDB ReplicaSet](../output/mongo_replset.md) 80 | * [Output to Hadoop](../output/webhdfs.md) 81 | * [Output to File](../output/file.md) 82 | * [etc...](http://fluentd.org/plugin/) 83 | 84 | ### High-Availability Configurations of Fluentd 85 | 86 | For high-traffic websites \(more than 5 application nodes\), we recommend using the high-availability configuration for `td-agent`. This will improve the reliability of data transfer and query performance. 87 | 88 | * [High-Availability Configurations of Fluentd](../deployment/high-availability.md) 89 | 90 | ### Monitoring 91 | 92 | Monitoring Fluentd itself is also important. The article below describes the general monitoring methods for `td-agent`. 93 | 94 | * [Monitoring Fluentd](../monitoring-fluentd/overview.md) 95 | 96 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 97 | 98 | -------------------------------------------------------------------------------- /metrics/README.md: -------------------------------------------------------------------------------- 1 | # Metrics Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](../input/) 6 | * [Parser](../parser/) 7 | * [Filter](../filter/) 8 | * [Output](../output/) 9 | * [Formatter](../formatter/) 10 | * [Storage](../storage) 11 | * [Service Discovery](../service_discovery/) 12 | * [Buffer](../buffer/) 13 | * [Metrics](./) 14 | 15 | This article gives an overview of Metrics Plugin. 16 | 17 | ## Overview 18 | 19 | Sometimes, the input/filter/output plugin needs to save its internal metrics in memory, influxdb or prometheus format ready in instances. Fluentd has a pluggable system called Metrics that lets a plugin store and reuse its internal state as metrics instances. 20 | 21 | ## How To Use 22 | 23 | On Fluentd core, metrics plugin will handled on `` on `` to set up easily. 24 | 25 | Here is an example with `metrics_local`: 26 | 27 | ```text 28 | 29 | 30 | @type local 31 | 32 | 33 | ``` 34 | 35 | `local` type plugin should provide equivalent behavior before Fluentd v1.13. 36 | This metrics type should provide single numeric value storing functionality. 37 | 38 | And this `local` type plugin should be used by default. 39 | 40 | ## List of Built-in Metrics Plugins 41 | 42 | * [`local`](local.md) 43 | 44 | ## List of Base Plugin classes with Metrics support 45 | 46 | * `Fluent::Plugin::Input` for Input plugin base class 47 | * `Fluent::Plugin::Output` for most of output plugin base class 48 | * `Fluent::Plugin::Filter` for Filter plugin base class 49 | * `Fluent::Plugin::MultiOutput` for [out_copy](../output/copy.md) plugin base class 50 | * `Fluent::Plugin::BareOutput` for fluent-plugin-forest output plugin base class 51 | 52 | ## List of 3rd party metrics plugins 53 | 54 | NOTE: This 3rd party metrics plugin list does not fully covers all of them. 55 | 56 | * [fluent-plugin-metrics-cmetrics](https://github.com/chronosphereio/calyptia-fluent-plugin-metrics-cmetrics) 57 | 58 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 59 | -------------------------------------------------------------------------------- /metrics/local.md: -------------------------------------------------------------------------------- 1 | # local 2 | 3 | The `local` metrics plugin stores the values on memory. 4 | 5 | ## Parameters 6 | 7 | ### `default_labels` 8 | 9 | Specifies the default labels for the metrics. \(default: `{agent: "Fluentd", hostname: "#{Socket.gethostname}"}`\) 10 | 11 | ### `labels` 12 | 13 | Specifies other custom labels for the metrics. \(default: `{}`\) 14 | 15 | ## Example 16 | 17 | With this configuration: 18 | 19 | ```text 20 | 21 | 22 | @type local 23 | 24 | 25 | ``` 26 | 27 | The above configuration will save the internal metrics for plugins on memory. As a result, you can retrieve metrics from memory and also you can replace with your custom metrics plugin. 28 | 29 | Actually, @type local metrics plugin has equivalent functionality for previous single value based Ruby instance variables. 30 | This behavior will be changed by other 3rd party plugins. 31 | 32 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 33 | -------------------------------------------------------------------------------- /monitoring-fluentd/README.md: -------------------------------------------------------------------------------- 1 | # Monitoring Fluentd 2 | 3 | -------------------------------------------------------------------------------- /monitoring-fluentd/monitoring-rest-api.md: -------------------------------------------------------------------------------- 1 | # Monitoring by REST API 2 | 3 | This article describes how to get the internal Fluentd metrics via the REST API. 4 | 5 | ## Monitoring Agent 6 | 7 | Fluentd has a monitoring agent to retrieve internal metrics in JSON format via HTTP. 8 | 9 | Add these lines to your configuration file: 10 | 11 | ```text 12 | 13 | @type monitor_agent 14 | bind 0.0.0.0 15 | port 24220 16 | 17 | ``` 18 | 19 | Restart the agent and get the metrics via HTTP: 20 | 21 | ```text 22 | $ curl http://host:24220/api/plugins.json 23 | { 24 | "plugins":[ 25 | { 26 | "plugin_id":"object:3fec669d6ac4", 27 | "type":"forward", 28 | "output_plugin":false, 29 | "config":{ 30 | "type":"forward" 31 | } 32 | }, 33 | { 34 | "plugin_id":"object:3fec669dfa48", 35 | "type":"monitor_agent", 36 | "output_plugin":false, 37 | "config":{ 38 | "type":"monitor_agent", 39 | "port":"24220" 40 | } 41 | }, 42 | { 43 | "plugin_id":"object:3fec66aead48", 44 | "type":"forward", 45 | "output_plugin":true, 46 | "buffer_queue_length":0, 47 | "buffer_total_queued_size":0, 48 | "retry_count":0, 49 | "config":{ 50 | "type":"forward", 51 | "host":"192.168.0.11" 52 | } 53 | } 54 | ] 55 | } 56 | ``` 57 | 58 | See [`in_monitor_agent`](../input/monitor_agent.md) article for more detail. 59 | 60 | ## Monitoring the Event Flow 61 | 62 | Use [`flowcounter`](https://github.com/tagomoris/fluent-plugin-flowcounter) or [`flowcounter_simple`](https://github.com/sonots/fluent-plugin-flowcounter-simple) plugin. 63 | 64 | ## Datadog \(`dd-agent`\) Integration 65 | 66 | [`Datadog`](https://www.datadoghq.com/) is a cloud monitoring service, and its monitoring agent `dd-agent` has native integration with Fluentd. 67 | 68 | For more details: 69 | 70 | * [Datadog-Fluentd Integration](http://docs.datadoghq.com/integrations/fluentd/) 71 | 72 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 73 | 74 | -------------------------------------------------------------------------------- /monitoring-fluentd/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | This article describes how to monitor Fluentd. 4 | 5 | ## Fluentd Metrics Monitoring 6 | 7 | Fluentd can expose internal metrics via REST API, and works with monitoring tools such as [`Prometheus`](https://prometheus.io/), [`Datadog`](https://www.datadoghq.com/), etc. Our recommendation is to use `Prometheus`, since we will be collaborating more in the future under the [CNCF \(Cloud Native Computing Foundation\)](https://www.cncf.io/). 8 | 9 | * [Monitoring Fluentd \(Prometheus\)](monitoring-prometheus.md) 10 | * [Monitoring Fluentd \(Datadog\)](https://docs.datadoghq.com/integrations/fluentd/) 11 | * [Monitoring Fluentd \(REST API\)](monitoring-rest-api.md) 12 | 13 | ## Process Monitoring 14 | 15 | Two `ruby` processes \(parent and child\) are executed. Please make sure that these processes are running. 16 | 17 | Here's an example for `td-agent`: 18 | 19 | ```text 20 | /opt/td-agent/embedded/bin/ruby /usr/sbin/td-agent 21 | --daemon /var/run/td-agent/td-agent.pid 22 | --log /var/log/td-agent/td-agent.log 23 | ``` 24 | 25 | For `td-agent` on Linux, check the process statuses like this: 26 | 27 | ```text 28 | $ ps w -C ruby -C td-agent --no-heading 29 | 32342 ? Sl 0:00 /opt/td-agent/embedded/bin/ruby /usr/sbin/td-agent --daemon /var/run/td-agent/td-agent.pid --log /var/log/td-agent/td-agent.log 30 | 32345 ? Sl 0:01 /opt/td-agent/embedded/bin/ruby /usr/sbin/td-agent --daemon /var/run/td-agent/td-agent.pid --log /var/log/td-agent/td-agent.log 31 | ``` 32 | 33 | There should be two processes if there is no issue. 34 | 35 | ## Port Monitoring 36 | 37 | Fluentd opens several ports according to the configuration file. We recommend checking the availability of these ports. The default port settings are shown below: 38 | 39 | * TCP 0.0.0.0 9880 \(HTTP by default\) 40 | * TCP 0.0.0.0 24224 \(Forward by default\) 41 | 42 | ## Debug Port 43 | 44 | A debug port for local communication is recommended for troubleshooting. The following configuration will be required for the debug port: 45 | 46 | ```text 47 | 48 | @type debug_agent 49 | bind 127.0.0.1 50 | port 24230 51 | 52 | ``` 53 | 54 | You can attach the process using the `fluent-debug` command through `dRuby`. 55 | 56 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 57 | 58 | -------------------------------------------------------------------------------- /output/buffer.md: -------------------------------------------------------------------------------- 1 | # buffer 2 | 3 | The `buffer` output plugin buffers and re-labels events. 4 | This plugin is similar to [out_relabel](relabel.md), but uses [buffer](../buffer/). 5 | 6 | It is included in Fluentd's core (since v1.18.0). 7 | 8 | ## Example Configuration 9 | 10 | ``` 11 | 12 | @type udp 13 | @label @buffer 14 | tag foo.udp 15 | 16 | @type none 17 | 18 | 19 | 20 | 21 | 22 | @type buffer 23 | @label @ROOT 24 | 25 | path /path/to/buffer 26 | 27 | 28 | 29 | 30 | 31 | @type stdout 32 | 33 | ``` 34 | 35 | In the above example, events ingested by `in_udp` are once stored in the buffer of this plugin, then re-routed and output by `out_stdout`. 36 | 37 | ## Supported Modes 38 | 39 | * Synchronous Buffered 40 | 41 | See also: [Output Plugin Overview](./) 42 | 43 | ## Plugin Helpers 44 | 45 | * [`event_emitter`](../plugin-helper-overview/api-plugin-helper-event_emitter.md) 46 | 47 | ## Parameters 48 | 49 | [Common Parameters](../configuration/plugin-common-parameters.md) 50 | 51 | ### `@type` \(required\) 52 | 53 | The value must be `buffer`. 54 | 55 | ### `@label` \(required\) 56 | 57 | | type | default | version | 58 | | :--- | :--- | :--- | 59 | | string | `nil` | 1.18.0 | 60 | 61 | Specifies the label to re-route. 62 | 63 | Note: You can specify `@ROOT` to re-route to the root. 64 | 65 | ### `` Section 66 | 67 | #### `path` \(required\) 68 | 69 | | type | default | version | 70 | | :--- | :--- | :--- | 71 | | string | required parameter | 1.18.0 | 72 | 73 | #### `@type` 74 | 75 | | type | default | version | 76 | | :--- | :--- | :--- | 77 | | string | file | 1.18.0 | 78 | 79 | Overwrites the default value in this plugin. 80 | 81 | #### `chunk_keys` 82 | 83 | | type | default | version | 84 | | :--- | :--- | :--- | 85 | | array | tag | 1.18.0 | 86 | 87 | Overwrites the default value in this plugin. 88 | 89 | #### `flush_mode` 90 | 91 | | type | default | version | 92 | | :--- | :--- | :--- | 93 | | enum | interval | 1.18.0 | 94 | 95 | Overwrites the default value in this plugin. 96 | 97 | #### `flush_interval` 98 | 99 | | type | default | version | 100 | | :--- | :--- | :--- | 101 | | integer | 10 | 1.18.0 | 102 | 103 | Overwrites the default value in this plugin. 104 | 105 | #### Common Buffer / Output parameters 106 | 107 | In addition, you can configure other common settings. 108 | Please see the followings for details. 109 | 110 | * [Buffer Section Configurations](../configuration/buffer-section.md) 111 | * [Buffer Plugin Overview](../buffer/) 112 | * [Output Plugin Overview](./) 113 | 114 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 115 | -------------------------------------------------------------------------------- /output/null.md: -------------------------------------------------------------------------------- 1 | # null 2 | 3 |  4 | 5 | The `null` output plugin just throws away events. 6 | 7 | It is included in Fluentd's core. 8 | 9 | ## Example Configuration 10 | 11 | ```text 12 | 13 | @type null 14 | 15 | ``` 16 | 17 | Please see the [Configuration File](../configuration/config-file.md) article for the basic structure and syntax of the configuration file. 18 | 19 | ## Supported Modes 20 | 21 | * Non-Buffered 22 | * Synchronous 23 | * Asynchronous 24 | 25 | See [Output Plugin Overview](./) for more details. 26 | 27 | ## Parameters 28 | 29 | [Common Parameters](../configuration/plugin-common-parameters.md) 30 | 31 | ### `@type` 32 | 33 | The value must be `null`. 34 | 35 | ### `never_flush` 36 | 37 | | type | default | version | 38 | | :--- | :--- | :--- | 39 | | bool | false | 0.14.12 | 40 | 41 | The parameter for testing to simulate the output plugin that never succeeds to flush. 42 | 43 | ### `` Section 44 | 45 | See [Buffer section configurations](../configuration/buffer-section.md) for more details. 46 | 47 | #### `chunk_keys` 48 | 49 | | type | default | version | 50 | | :--- | :--- | :--- | 51 | | array | tag | 0.14.5 | 52 | 53 | Overwrites the default value in this plugin. 54 | 55 | #### `flush_at_shutdown` 56 | 57 | | type | default | version | 58 | | :--- | :--- | :--- | 59 | | bool | true | 0.14.5 | 60 | 61 | Overwrites the default value in this plugin. 62 | 63 | #### `chunk_limit_size` 64 | 65 | | type | default | version | 66 | | :--- | :--- | :--- | 67 | | size | 10240 | 0.14.5 | 68 | 69 | Overwrites the default value in this plugin. 70 | 71 | ## Common Buffer / Output parameters 72 | 73 | See [Buffer Plugin Overview](../buffer/) and [Output Plugin Overview](./). 74 | 75 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 76 | 77 | -------------------------------------------------------------------------------- /output/relabel.md: -------------------------------------------------------------------------------- 1 | # relabel 2 | 3 | The `relabel` output plugin re-labels events. 4 | 5 | It is included in Fluentd's core. 6 | 7 | ## Example Configuration 8 | 9 | ```text 10 | 11 | @type relabel 12 | @label @foo 13 | 14 | 15 | 16 | 17 | # ... 18 | 19 | 20 | ``` 21 | 22 | In the above example, the `relabel` output plugin uses a label `@foo` to route the matched events, and then the respective `label` directive takes care of these events. 23 | 24 | NOTE: All the input and output plugins support the `@label` parameter provided by the Fluentd core. The `relabel` plugin is a plugin that does nothing other than supporting the `@label` parameter. 25 | 26 | ## Supported Modes 27 | 28 | * Non-Buffered 29 | * See also: [Output Plugin Overview](./) 30 | 31 | ## Plugin Helpers 32 | 33 | * [`event_emitter`](../plugin-helper-overview/api-plugin-helper-event_emitter.md) 34 | 35 | ## Parameters 36 | 37 | [Common Parameters](../configuration/plugin-common-parameters.md) 38 | 39 | ### `@type` 40 | 41 | The value must be `relabel`. 42 | 43 | ### `@label` 44 | 45 | | type | default | version | 46 | | :--- | :--- | :--- | 47 | | string | `nil` | 0.14.0 | 48 | 49 | Specifies the label. It is a required parameter. 50 | 51 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 52 | 53 | -------------------------------------------------------------------------------- /output/roundrobin.md: -------------------------------------------------------------------------------- 1 | # roundrobin 2 | 3 | The `roundrobin` Output plugin distributes events to multiple outputs using a weighted round-robin algorithm. 4 | 5 | It is included in Fluentd's core. 6 | 7 | ## Example Configuration 8 | 9 | ```text 10 | 11 | @type roundrobin 12 | 13 | 14 | @type http 15 | endpoint http://192.168.1.21 16 | weight 3 17 | # ... 18 | 19 | 20 | @type http 21 | endpoint http://192.168.1.22 22 | weight 2 23 | # ... 24 | 25 | 26 | @type http 27 | endpoint http://192.168.1.23 28 | weight 1 29 | # ... 30 | 31 | 32 | ``` 33 | 34 | Refer to [Configuration File](../configuration/config-file.md) for the basic structure and syntax of the configuration file. 35 | 36 | ## Supported Modes 37 | 38 | * Non-Buffered 39 | 40 | ## Parameters 41 | 42 | [Common Parameters](../configuration/plugin-common-parameters.md) 43 | 44 | ### `@type` 45 | 46 | The value must be `roundrobin`. 47 | 48 | ### `` 49 | 50 | Specifies the storage destinations. The format is the same as the `` directive. 51 | 52 | #### `weight` 53 | 54 | | type | default | version | 55 | | :--- | :--- | :--- | 56 | | integer | 1 | 0.14.1 | 57 | 58 | Weight to distribute events to multiple outputs. 59 | 60 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 61 | 62 | -------------------------------------------------------------------------------- /parser/apache2.md: -------------------------------------------------------------------------------- 1 | # apache2 2 | 3 | The `apache2` parser plugin parses apache2 logs. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md) 8 | 9 | ## Regexp Patterns 10 | 11 | Here is the regexp and time format patterns of this plugin: 12 | 13 | ```text 14 | expression /^(?[^ ]*) [^ ]* (?[^ ]*) \[(?[^\]]*)\] "(?\S+)(?: +(?(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?[^ ]*) (?[^ ]*)(?: "(?(?:[^\"]|\\.)*)" "(?(?:[^\"]|\\.)*)")?$/ 15 | time_format %d/%b/%Y:%H:%M:%S %z 16 | ``` 17 | 18 | `host`, `user`, `method`, `path`, `code`, `size`, `referer` and `agent` are included in the event record. `time` is used for the event time. 19 | 20 | `code` and `size` fields are converted to integer type automatically. And, if the field value is `-`, it is interpreted as `nil`. See **Result Example**. 21 | 22 | ## Example 23 | 24 | This incoming event 25 | 26 | ```text 27 | 192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" 28 | ``` 29 | 30 | is parsed as: 31 | 32 | ```text 33 | time: 34 | 1362020400 (28/Feb/2013:12:00:00 +0900) 35 | 36 | record: 37 | { 38 | "user" : nil, 39 | "method" : "GET", 40 | "code" : 200, 41 | "size" : 777, 42 | "host" : "192.168.0.1", 43 | "path" : "/", 44 | "referer": nil, 45 | "agent" : "Opera/12.0" 46 | } 47 | ``` 48 | 49 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 50 | 51 | -------------------------------------------------------------------------------- /parser/apache_error.md: -------------------------------------------------------------------------------- 1 | # apache\_error 2 | 3 | The `apache_error` parser plugin parses apache error logs. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ## Regexp patterns 10 | 11 | This is regexp pattern of this plugin: 12 | 13 | ```text 14 | expression /^\[[^ ]* (?[^\]]*)\] \[(?[^\]]*)\](?: \[pid (?[^\]]*)\])? \[client (?[^\]]*)\] (?.*)$/ 15 | ``` 16 | 17 | `level`, `pid`, `client` and `message` are included in the event record. `time` is used for the event time. 18 | 19 | ## Example 20 | 21 | This incoming event: 22 | 23 | ```text 24 | [Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration 25 | ``` 26 | 27 | is parsed as: 28 | 29 | ```text 30 | time: 31 | 971242372 (Wed Oct 11 14:32:52 2000) 32 | 33 | record: 34 | { 35 | "level" : "error", 36 | "client" : "127.0.0.1", 37 | "message": "client denied by server configuration" 38 | } 39 | ``` 40 | 41 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 42 | 43 | -------------------------------------------------------------------------------- /parser/csv.md: -------------------------------------------------------------------------------- 1 | # csv 2 | 3 | The `csv` parser plugin parses CSV format. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `keys` 10 | 11 | | type | default | version | 12 | | :--- | :--- | :--- | 13 | | array of string | required parameter | 0.14.9 | 14 | 15 | The array of names for fields on each line. 16 | 17 | ### `delimiter` 18 | 19 | | type | default | version | 20 | | :--- | :--- | :--- | 21 | | string | , | 0.14.2 | 22 | 23 | The delimiter \(character or string\) separating CSV values. 24 | 25 | ### `parser_type` 26 | 27 | | type | default | available values | version | 28 | | :--- | :--- | :--- | :--- | 29 | | enum | normal | normal/fast | 1.7.0 | 30 | 31 | The parser type used to parse the log line. 32 | 33 | * `normal` uses Ruby's 34 | 35 | [`CSV.parse_line`](http://ruby-doc.org/stdlib-2.4.1/libdoc/csv/rdoc/CSV.html#method-c-parse_line) 36 | 37 | method. 38 | 39 | * `fast` uses its own lightweight implementation. This parser is several times 40 | 41 | faster than `normal` but it supports only typical patterns. 42 | 43 | Supported CSV formats by `fast`: 44 | 45 | ```text 46 | # non-quoted 47 | value1,value2,value3,value4,value5 48 | 49 | # quoted 50 | "value1","val,ue2","va,lu,e3","val ue4","" 51 | 52 | # escaped 53 | "message","mes""sage","""message""",,"""""" 54 | 55 | # mixed 56 | message,"mes,sage","me,ssa,ge",mess age,"" 57 | ``` 58 | 59 | If your CSV format is not matched with the above patterns, use `normal` parser instead. 60 | 61 | ## Example 62 | 63 | With this configuration: 64 | 65 | ```text 66 | 67 | @type csv 68 | keys time,host,req_id,user 69 | time_key time 70 | 71 | ``` 72 | 73 | This incoming event: 74 | 75 | ```text 76 | 2013/02/28 12:00:00,192.168.0.1,111,- 77 | ``` 78 | 79 | is parsed as: 80 | 81 | ```text 82 | time: 83 | 1362020400 (2013/02/28/ 12:00:00) 84 | 85 | record: 86 | { 87 | "host" : "192.168.0.1", 88 | "req_id" : "111", 89 | "user" : "-" 90 | } 91 | ``` 92 | 93 | If you set `null_value_pattern '-'` in the configuration, `user` field becomes `nil` instead of `"-"`. 94 | 95 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 96 | 97 | -------------------------------------------------------------------------------- /parser/json.md: -------------------------------------------------------------------------------- 1 | # json 2 | 3 | The `json` parser plugin parses JSON logs. One JSON map per line. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `json_parser` 10 | 11 | | type | default | available | version | 12 | | :--- | :--- | :--- | :--- | 13 | | enum | oj | oj, yajl, json | 0.14.0 | 14 | 15 | Sets the JSON parser. If you have a problem with the configured parser, check the other available parser types. 16 | 17 | NOTE: If you want to enable `json_parser oj` by default, The `oj` gem must be installed separately. This is because `oj` gem is not required from fluentd by default. If `oj` gem is not installed, `json` is used as a fallback. (The fallback has been changed from `yajl` to `json` since v1.19.0.) 18 | 19 | Here is a simple comparison: 20 | 21 | * `oj`: Faster json parser 22 | * `yajl`: Mainly for stream parsing 23 | * `json`: Standard bundled library 24 | 25 | ### `stream_buffer_size` 26 | 27 | | type | default | version | 28 | | :--- | :--- | :--- | 29 | | integer | 8192 | 1.0.0 | 30 | 31 | Set the buffer size that Yajl will use when parsing streaming input. 32 | 33 | If you specify the smaller buffer size, you could get outcome response instantaneously in exchange for performance penalty. 34 | 35 | See also: [Method: Yajl::Parser\#parse](https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse) 36 | 37 | ### `time_type` 38 | 39 | `json` parser changes the default value of `time_type` to `float`. If you want to parse string field, set `time_type` and `time_format` like this: 40 | 41 | ```text 42 | # conf 43 | @type json 44 | time_type string 45 | time_format %d/%b/%Y:%H:%M:%S %z 46 | 47 | # record example 48 | {"key":"value","time":"28/Feb/2013:12:00:00 +0900"} 49 | ``` 50 | 51 | See also [`parse` Section](../configuration/parse-section.md#time-parameters) article. 52 | 53 | ## Example 54 | 55 | This incoming event: 56 | 57 | ```text 58 | {"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"} 59 | ``` 60 | 61 | is parsed as: 62 | 63 | ```text 64 | time: 65 | 1362020400 (2013-02-28 12:00:00 +0900) 66 | 67 | record: 68 | { 69 | "host" : "192.168.0.1", 70 | "size" : 777, 71 | "method": "PUT", 72 | } 73 | ``` 74 | 75 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 76 | 77 | -------------------------------------------------------------------------------- /parser/ltsv.md: -------------------------------------------------------------------------------- 1 | # ltsv 2 | 3 | The `ltsv` parser plugin parses [LTSV](http://ltsv.org/) format. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `delimiter` 10 | 11 | | type | default | version | 12 | | :--- | :--- | :--- | 13 | | string | `\t` | 0.14.0 | 14 | 15 | The delimiter \(character or string\) separating LTSV values. 16 | 17 | ### `delimiter_pattern` 18 | 19 | | type | default | version | 20 | | :--- | :--- | :--- | 21 | | regexp | nil | 1.2.0 | 22 | 23 | The delimiter pattern of TSV values. This parameter overwrites `delimiter` parameter if specified. 24 | 25 | `delimiter_pattern` is string type before 1.2.0. 26 | 27 | ### `label_delimiter` 28 | 29 | | type | default | version | 30 | | :--- | :--- | :--- | 31 | | string | : | 0.14.0 | 32 | 33 | The delimiter character between field name and value. 34 | 35 | ## Example for LTSV 36 | 37 | This incoming event: 38 | 39 | ```text 40 | time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111\tuser:- 41 | ``` 42 | 43 | is parsed as: 44 | 45 | ```text 46 | time: 47 | 1362020400 (2013/02/28/ 12:00:00) 48 | 49 | record: 50 | { 51 | "host" : "192.168.0.1", 52 | "req_id" : "111", 53 | "user" : "-" 54 | } 55 | ``` 56 | 57 | If you set `null_value_pattern '-'` in the configuration, `user` field becomes `nil` instead of `"-"`. 58 | 59 | ## Example with `delimiter_pattern` 60 | 61 | With this configuration: 62 | 63 | ```text 64 | 65 | @type ltsv 66 | delimiter_pattern /\s+/ 67 | label_delimiter = 68 | 69 | ``` 70 | 71 | This incoming event: 72 | 73 | ```text 74 | timestamp=1362020400 host=192.168.0.1 req_id=111 user=- 75 | ``` 76 | 77 | is parsed as: 78 | 79 | ```text 80 | record: 81 | { 82 | "timestamp": "1362020400", 83 | "host" : "192.168.0.1", 84 | "req_id" : "111", 85 | "user" : "-" 86 | } 87 | ``` 88 | 89 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 90 | 91 | -------------------------------------------------------------------------------- /parser/msgpack.md: -------------------------------------------------------------------------------- 1 | # msgpack 2 | 3 | The `msgpack` parser plugin parses the MessagePack data. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ## Example 10 | 11 | This incoming event: 12 | 13 | ```text 14 | \x82\xA7message\xADHello msgpack\xA3numd 15 | ``` 16 | 17 | is parsed as: 18 | 19 | ```text 20 | time: 21 | 1582648111.269613 (current time) 22 | 23 | record: 24 | {"message":"Hello msgpack", "num":100} 25 | ``` 26 | 27 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 28 | 29 | -------------------------------------------------------------------------------- /parser/nginx.md: -------------------------------------------------------------------------------- 1 | # nginx 2 | 3 | The `nginx` parser plugin parses the default Nginx logs. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ## Regexp Patterns 10 | 11 | Here is the regexp and time format patterns of this plugin: 12 | 13 | ```text 14 | expression /^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)?" (?[^ ]*) (?[^ ]*)(?: "(?[^\"]*)" "(?[^\"]*)"(?:\s+(?[^ ]+))?)?$/ 15 | time_format %d/%b/%Y:%H:%M:%S %z 16 | ``` 17 | 18 | `remote`, `user`, `method`, `path`, `code`, `size`, `referer`, `agent` and `http_x_forwarded_for` are included in the event record. `time` is used for the event time. 19 | 20 | ## Example 21 | 22 | This incoming event: 23 | 24 | ```text 25 | 127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" - 26 | ``` 27 | 28 | is parsed as: 29 | 30 | ```text 31 | time: 32 | 1362020400 (28/Feb/2013:12:00:00 +0900) 33 | 34 | record: 35 | { 36 | "remote" : "127.0.0.1", 37 | "host" : "192.168.0.1", 38 | "user" : "-", 39 | "method" : "GET", 40 | "path" : "/", 41 | "code" : "200", 42 | "size" : "777", 43 | "referer" : "-", 44 | "agent" : "Opera/12.0", 45 | "http_x_forwarded_for": "-" 46 | } 47 | ``` 48 | 49 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 50 | 51 | -------------------------------------------------------------------------------- /parser/none.md: -------------------------------------------------------------------------------- 1 | # none 2 | 3 | The `none` parser plugin parses the line as-is with the single field. This format is to defer the parsing/structuring of the data. 4 | 5 | This parser is often used in conjunction with [`single_value`](../formatter/single_value.md) format in output plugin. 6 | 7 | ## Parameters 8 | 9 | See [Parse Section Configurations](../configuration/parse-section.md). 10 | 11 | ### `message_key` 12 | 13 | | type | default | version | 14 | | :--- | :--- | :--- | 15 | | string | message | 0.14.0 | 16 | 17 | Specifies the field name to contain logs. 18 | 19 | ## Example 20 | 21 | This incoming event: 22 | 23 | ```text 24 | Hello world. I am a line of log! 25 | ``` 26 | 27 | is parsed as: 28 | 29 | ```text 30 | time: 31 | 1362020400 (current time) 32 | 33 | record: 34 | {"message":"Hello world. I am a line of log!"} 35 | ``` 36 | 37 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 38 | 39 | -------------------------------------------------------------------------------- /parser/tsv.md: -------------------------------------------------------------------------------- 1 | # tsv 2 | 3 | The `tsv` parser plugin parses the TSV format. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `keys` 10 | 11 | | type | default | version | 12 | | :--- | :--- | :--- | 13 | | array of string | required parameter | 0.14.9 | 14 | 15 | The array of names for fields on each line. 16 | 17 | ### `delimiter` 18 | 19 | | type | default | version | 20 | | :--- | :--- | :--- | 21 | | string | `\t` | 0.14.0 | 22 | 23 | The delimiter \(character or string\) separating TSV values. 24 | 25 | ## Example 26 | 27 | With this configuration: 28 | 29 | ```text 30 | 31 | @type tsv 32 | keys time,host,req_id,user 33 | time_key time 34 | 35 | ``` 36 | 37 | This incoming event: 38 | 39 | ```text 40 | 2013/02/28 12:00:00\t192.168.0.1\t111\t- 41 | ``` 42 | 43 | is parsed as: 44 | 45 | ```text 46 | time: 47 | 1362020400 (2013/02/28/ 12:00:00) 48 | 49 | record: 50 | { 51 | "host" : "192.168.0.1", 52 | "req_id" : "111", 53 | "user" : "-" 54 | } 55 | ``` 56 | 57 | If you set `null_value_pattern '-'` in the configuration, `user` field becomes `nil` instead of `"-"`. 58 | 59 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 60 | 61 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-buffer.md: -------------------------------------------------------------------------------- 1 | # How to Write Buffer Plugin 2 | 3 | TODO: Write 4 | 5 | ## Methods 6 | 7 | ### evacuate_chunk 8 | 9 | You can override this method to add feature to evacuate chunks before clearing the queue when reaching the retry limit. 10 | See [Buffer - Handling Successive Failures](../buffer/README.md#handling-successive-failures) for details. 11 | 12 | ```ruby 13 | def evacuate_chunk(chunk) 14 | unless chunk.is_a?(Fluent::Plugin::Buffer::FileChunk) 15 | raise ArgumentError, "The chunk must be FileChunk, but it was #{chunk.class}." 16 | end 17 | 18 | backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id) 19 | FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir) 20 | 21 | FileUtils.copy([chunk.path, chunk.meta_path], backup_dir) 22 | log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id) 23 | rescue => e 24 | log.error "unexpected error while evacuating chunk files.", error: e 25 | end 26 | ``` 27 | 28 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 29 | 30 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-service_discovery.md: -------------------------------------------------------------------------------- 1 | # How to Write Service Discovery Plugin 2 | 3 | TODO: Write 4 | 5 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 6 | 7 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-storage.md: -------------------------------------------------------------------------------- 1 | # How to Write Storage Plugin 2 | 3 | TODO: Write 4 | 5 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 6 | 7 | -------------------------------------------------------------------------------- /plugin-helper-overview/README.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper API 2 | 3 | Fluentd provides plugin helpers to encapsulate and make commonly implemented features available such as timer, threading, formatting, parsing, ensuring configuration syntax's backward compatibility, etc. 4 | 5 | ## How To Use 6 | 7 | To use plugin helpers, call `helpers(*snake_case_symbols)` method to make them available. 8 | 9 | Example: 10 | 11 | ```ruby 12 | helpers :timer, :storage, :compat_parameters 13 | ``` 14 | 15 | It will include `Timer`, `Storage`. and `CompatParameters` plugin helpers. 16 | 17 | ## Built-in Plugin Helpers 18 | 19 | * [`child_process`](api-plugin-helper-child_process.md) 20 | * [`compat_parameters`](api-plugin-helper-compat_parameters.md) 21 | * [`event_emitter`](api-plugin-helper-event_emitter.md) 22 | * [`event_loop`](api-plugin-helper-event_loop.md) 23 | * [`extract`](api-plugin-helper-extract.md) 24 | * [`formatter`](api-plugin-helper-formatter.md) 25 | * [`inject`](api-plugin-helper-inject.md) 26 | * [`metrics`](api-plugin-helper-metrics.md) 27 | * [`parser`](api-plugin-helper-parser.md) 28 | * [`record_accessor`](api-plugin-helper-record_accessor.md) 29 | * [`server`](api-plugin-helper-server.md) 30 | * [`socket`](api-plugin-helper-socket.md) 31 | * [`storage`](api-plugin-helper-storage.md) 32 | * [`thread`](api-plugin-helper-thread.md) 33 | * [`timer`](api-plugin-helper-timer.md) 34 | * [`http_server`](api-plugin-helper-http_server.md) 35 | 36 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 37 | 38 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-child_process.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Child Process 2 | 3 | The `child_process` helper manages the child processes' lifecycle. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load child_process helper 15 | helpers :child_process 16 | 17 | # Omit configure, shutdown and other plugin API 18 | 19 | def start 20 | super 21 | # 2. Execute child process with unique name 22 | child_process_execute(:exec_external_command, 'external_command', immediate: true, mode: [:read]) do 23 | # ... 24 | end 25 | end 26 | end 27 | end 28 | ``` 29 | 30 | The launched child process is managed by the plugin helper which automatically `shutdown`s it. 31 | 32 | ## Methods 33 | 34 | ### `child_process_execute(title, sub_process_name, arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false, mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil, internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil, wait_timeout: nil, on_exit_callback: nil, &block)` 35 | 36 | This method executes `child_process` with the given parameters and routine. 37 | 38 | * `title`: unique symbol value 39 | * `sub_process_name`: sub process name value 40 | * `interval`: Second unit `integer`/`float` value. 41 | * `immediate`: `true`/`false`. Default is `false`. 42 | * `parallel`: `true`/`false`. Default is `false`. 43 | * `mode`: \[`:read`, `:write`\]. Default is \[`:read`, `:write`\]. 44 | * `stderr`: Connect stderr or not. Default is `:discard`. 45 | * `env`: Environment valuables. Default is {}. 46 | * `unsetenv`: `true`/`false`. Default is `false` 47 | * `chdir`: Working directory. Default is `nil`. 48 | * `internal_encoding`: Internal character encoding. Default is `utf-8`. 49 | * `external_encoding`: External character encoding. Default is `ascii-8bit`' 50 | * `scrub`: `true`/`false`. Default is `true`. 51 | * `replace_string`: Replace the invalid code point with the specified character. 52 | 53 | Default is `nil`. 54 | 55 | * `wait_timeout`: Set timeout seconds. Default is `nil`. 56 | * `on_exit_callback`: Set callback function. Default is `nil` 57 | 58 | Code examples: 59 | 60 | ```ruby 61 | # Pass block directly. block is executed with 10 second interval. 62 | child_process_execute(:exec_awesome_command, @command, interval: 10, mode: [:read]) do |io| 63 | # ... 64 | end 65 | 66 | # Pass block with existing method. block is executed once. 67 | child_process_execute(:exec_awesome_command, @command, immediate: true, mode: [:read], &method(:run)) 68 | 69 | def run(io) 70 | # ... 71 | end 72 | ``` 73 | 74 | ## Plugins using `child_process` 75 | 76 | * [`in_exec`](../input/exec.md) 77 | * [`out_exec`](../output/exec.md) 78 | * [`out_exec_filter`](../output/exec_filter.md) 79 | 80 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 81 | 82 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-event_emitter.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Event Emitter 2 | 3 | The `event_emitter` plugin helper introduces the `router` method to the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class RelabelOutput < Output 12 | Fluent::Plugin.register_output('relabel', self) 13 | helpers :event_emitter 14 | 15 | def multi_workers_ready? 16 | true 17 | end 18 | 19 | def process(tag, es) 20 | router.emit_stream(tag, es) 21 | end 22 | end 23 | end 24 | ``` 25 | 26 | ## Methods 27 | 28 | ### `router` 29 | 30 | This method returns a `Fluent::EventRouter` instance. 31 | 32 | Code example: 33 | 34 | ```ruby 35 | # emit event 36 | router.emit(tag, time, record) 37 | 38 | # emit event stream 39 | router.emit_stream(tag, es) 40 | 41 | # emit error event 42 | begin 43 | # do something 44 | rescue => error 45 | # Route event to @ERROR label or 46 | # log error message when @ERROR label is not defined in configuration 47 | router.emit_error_event(tag, time, record, error) 48 | end 49 | ``` 50 | 51 | ## Plugins using `event_emitter` 52 | 53 | * All input plugins 54 | * All filter plugins 55 | * [`out_exec`](../output/exec.md) 56 | * [`out_relabel`](../output/relabel.md) 57 | * [`out_copy`](../output/copy.md) 58 | * [`out_roundrobin`](../output/roundrobin.md) 59 | 60 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 61 | 62 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-event_loop.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Event Loop 2 | 3 | The `event_loop` plugin helper manages event loops. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load `event_loop_helper` 15 | helpers :event_loop 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Attach watcher 23 | watcher = Coolio::TCPServer.new(...) 24 | event_loop_attach(watcher) 25 | end 26 | end 27 | end 28 | ``` 29 | 30 | The attached watcher is managed by the plugin. No need of watcher detach code in plugin's `shutdown`. The plugin shutdowns the attached watchers automatically. 31 | 32 | ## Methods 33 | 34 | ### `event_loop_attach(watcher)` 35 | 36 | This method attaches watcher to the event loop. 37 | 38 | * `watcher`: `Coolio::Watcher` instances 39 | 40 | ## Plugins using `event_loop` 41 | 42 | * [`in_http`](../input/http.md) 43 | * [`in_tail`](../input/tail.md) 44 | 45 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 46 | 47 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-extract.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Extract 2 | 3 | The `extract` plugin helper extracts `tag` or `time` from the event `record` according to the configuration. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput > Output 12 | Fluent::Plugin.register_output('Example') 13 | 14 | def process(tag, es) 15 | es.each do |time, record| 16 | new_tag = extract_tag_from_record(record) 17 | new_time = extract_time_from_record(record) 18 | end 19 | # ... 20 | end 21 | end 22 | ``` 23 | 24 | For more details, see [Extract section](../configuration/extract-section.md). 25 | 26 | ## Methods 27 | 28 | ### `extract_tag_from_record(record)` 29 | 30 | This method extracts `tag` from the given record. 31 | 32 | * `record`: event record 33 | 34 | Example: 35 | 36 | ```ruby 37 | new_tag = extract_tag_from_record(record) 38 | ``` 39 | 40 | ### `extract_time_from_record(record)` 41 | 42 | This method extracts `time` from the given record. 43 | 44 | * `record`: event record 45 | 46 | Example: 47 | 48 | ```ruby 49 | new_time = extract_time_from_record(record) 50 | ``` 51 | 52 | ## Plugins using `extract` 53 | 54 | * [`in_exec`](../input/exec.md) 55 | * [`in_tcp`](../input/tcp.md) 56 | * [`in_udp`](../input/udp.md) 57 | * [`out_exec_filter`](../output/exec_filter.md) 58 | 59 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 60 | 61 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-formatter.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Formatter 2 | 3 | The `formatter` plugin helper manages the lifecycle of the formatter plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load formatter helper 15 | helpers :formatter 16 | 17 | # Omit `shutdown` and other plugin APIs 18 | 19 | def configure(conf) 20 | super 21 | 22 | # 2. Call `formatter_create` to create object 23 | @formatter = formatter_create(usage: 'awesome_formatter') 24 | end 25 | 26 | def format(tag, time, record) 27 | # 3. Call `format` method to format `record` 28 | @formatter.format(tag, time, record) 29 | end 30 | end 31 | end 32 | ``` 33 | 34 | For more details, see the following articles: 35 | 36 | * [Formatter Plugin Overview](../formatter/) 37 | * [Writing Formatter Plugins](../plugin-development/api-plugin-formatter.md) 38 | * [Format Section](../configuration/format-section.md) 39 | 40 | ## Methods 41 | 42 | ### `formatter_create(usage: "", type: nil, conf: nil, default_type: nil)` 43 | 44 | * `usage`: unique name required for multiple formatters 45 | * `type`: formatter type 46 | * `conf`: formatter plugin configuration 47 | * `default_type`: default formatter type 48 | 49 | **Examples**: 50 | 51 | ```ruby 52 | def configure(conf) 53 | super 54 | # Create formatter plugin instance using section in `fluent.conf` during configure phase 55 | @formatter = formatter_create 56 | end 57 | 58 | def format(tag, time, record) 59 | @formatter.format(tag, time, record) 60 | end 61 | ``` 62 | 63 | JSON formatter example: 64 | 65 | ```ruby 66 | # Create JSON formatter 67 | def configure(conf) 68 | super 69 | @json_formatter = formatter_create(usage: 'formatter_in_example_json', type: 'json') 70 | end 71 | 72 | def format(tag, time, record) 73 | @json_formatter.format(tag, time, record) 74 | end 75 | ``` 76 | 77 | MsgPack formatter example: 78 | 79 | ```ruby 80 | # Create MsgPack formatter 81 | def configure(conf) 82 | super 83 | @msgpack_formatter = formatter_create(usage: 'formatter_in_example_msgpack', type: 'msgpack') 84 | end 85 | 86 | def format(tag, time, record) 87 | @msgpack_formatter.format(tag, time, record) 88 | end 89 | ``` 90 | 91 | ## Plugins using `formatter` 92 | 93 | * [`filter_stdout`](../filter/stdout.md) 94 | * [`out_stdout`](../output/stdout.md) 95 | * [`out_exec`](../output/exec.md) 96 | * [`out_exec_filter`](../output/exec_filter.md) 97 | * [`out_file`](../output/file.md) 98 | * [`out_s3`](../output/s3.md) 99 | 100 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 101 | 102 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-inject.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Inject 2 | 3 | The `inject` plugin helper injects values into events according to the configuration. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/filter' 9 | 10 | module Fluent::Plugin 11 | class ExampleFilter < Filter 12 | Fluent::Plugin.register_filter('example', self) 13 | 14 | # 1. Load inject helper 15 | helpers :inject 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def filter(tag, time, record) 20 | # 2. Inject values into `record` 21 | new_record = inject_values_to_record(tag, time, record) 22 | # edit new_record ... 23 | new_record 24 | end 25 | end 26 | end 27 | ``` 28 | 29 | For more details, see [Inject section](../configuration/inject-section.md). 30 | 31 | ## Methods 32 | 33 | ### `inject_values_to_record(tag, time, record)` 34 | 35 | This method injects values to the given record and returns the new record. 36 | 37 | * `tag`: the tag of the event 38 | * `time`: event timestamp 39 | * `record`: event record 40 | 41 | Example: 42 | 43 | ```ruby 44 | def filter(tag, time, record) 45 | new_record = inject_values_to_record(tag, time, record) 46 | # edit new_record ... 47 | new_record 48 | end 49 | ``` 50 | 51 | ### `inject_values_to_event_stream(tag, es)` 52 | 53 | This method injects values into the given event stream and returns the new event stream. 54 | 55 | * `tag`: the tag of the event 56 | * `es`: event stream 57 | 58 | Example: 59 | 60 | ```ruby 61 | def process(tag, es) 62 | new_es = inject_values_to_event_stream(tag, es) 63 | # do something using new_es 64 | end 65 | ``` 66 | 67 | ## Plugins using `inject` 68 | 69 | * [`filter_stdout`](../filter/stdout.md) 70 | * [`out_exec`](../output/exec.md) 71 | * [`out_exec_filter`](../output/exec_filter.md) 72 | * [`out_file`](../output/file.md) 73 | * [`out_stdout`](../output/stdout.md) 74 | 75 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 76 | 77 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-metrics.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Metrics 2 | 3 | The `metrics` plugin helper manages the metrics values in plugins. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load metrics helper 15 | helpers :metrics 16 | 17 | def configure(conf) 18 | super 19 | 20 | # 2. Create parser plugin instance 21 | @metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "example", help_text: "Example metrics") 22 | end 23 | 24 | def start 25 | super 26 | 27 | # 3. Increase metrics value 28 | @metrics.inc 29 | 30 | end 31 | 32 | def statistics 33 | stats = super 34 | 35 | # 4. Retrieve metrics value 36 | stats = { 37 | 'input' => stats["input"].merge({ 'example' => @metrics.get }) 38 | } 39 | stats 40 | end 41 | end 42 | ``` 43 | 44 | For more details, see the following articles: 45 | 46 | * [Metrics Plugins Overview](../metrics/) 47 | 48 | ## Methods 49 | 50 | ### `metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)` 51 | 52 | This method creates a metrics instance. 53 | 54 | * `namespace`: The namespace for the metrics. 55 | * `subsystem`: The names that represent specific functions or components. 56 | * `name`: The metrics name. 57 | * `help_text`: The description for metrics. 58 | * `labels`: The key/value pair for metrics labels. 59 | * `prefer_gauge`: Use gauge instead of counter for the metrics if `true`. 60 | 61 | Since 1.19.0, `metrics_create` method generates a getter method with the specified `name` on the calling instance. 62 | 63 | ## Plugins using `metrics` 64 | 65 | * [`buffer`](../buffer/) 66 | * [`filter`](../filter/) 67 | * [`input`](../input/) 68 | * [`output`](../output/) 69 | 70 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 71 | 72 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-parser.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Parser 2 | 3 | The `parser` plugin helper manages the lifecycle of the parser plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load parser helper 15 | helpers :parser 16 | 17 | # Omit `shutdown` and other plugin APIs 18 | 19 | def configure(conf) 20 | super 21 | 22 | # 2. Create parser plugin instance 23 | @parser = parser_create 24 | end 25 | 26 | def start 27 | super 28 | 29 | # Use parser helper in combination usually with other plugin helpers 30 | timer_execute(:example_timer, 10) do 31 | read_raw_data do |text| 32 | # 3. Call `@parser.parse(text)` to parse raw data 33 | @parser.parse(text) do |time, record| 34 | router.emit(tag, time, record) 35 | end 36 | end 37 | end 38 | end 39 | end 40 | end 41 | ``` 42 | 43 | For more details, see the following articles: 44 | 45 | * [Parser Plugin Overview](../parser/) 46 | * [Writing Parser Plugins](../plugin-development/api-plugin-parser.md) 47 | * [Parser Plugin](../configuration/parse-section.md) 48 | 49 | ## Methods 50 | 51 | ### `parser_create(usage: "", type: nil, conf: nil, default_type: nil)` 52 | 53 | This method creates a parser plugin instance with the given parameters. 54 | 55 | * `usage`: unique name required for multiple parsers 56 | * `type`: parser type 57 | * `conf`: parser plugin configuration 58 | * `default_type`: default parser type 59 | 60 | **Examples** 61 | 62 | ```ruby 63 | # Create parser plugin instance using section in fluent.conf during configure phase 64 | @parser = parser_create 65 | @parser.parse(text) do |time, record| 66 | # ... 67 | end 68 | 69 | # Create JSON parser 70 | @json_parser = parser_create(usage: 'parser_in_example_json', type: 'json') 71 | @json_parser.parse(json) do |time, record| 72 | # ... 73 | end 74 | 75 | # Create MessagePack parser 76 | @msgpack_parser = parser_create(usage: 'parser_in_example_msgpack', type: 'msgpack') 77 | @msgpack_parser.parse(msgpack_binary) do |time, record| 78 | # ... 79 | end 80 | ``` 81 | 82 | ## Plugins using `parser` 83 | 84 | * [`filter_parser`](../filter/parser.md) 85 | * [`in_exec`](../input/exec.md) 86 | * [`in_http`](../input/http.md) 87 | * [`in_syslog`](../input/syslog.md) 88 | * [`in_tail`](../input/tail.md) 89 | * [`in_tcp`](../input/tcp.md) 90 | * [`in_udp`](../input/udp.md) 91 | * [`out_exec_filter`](../output/exec_filter.md) 92 | 93 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 94 | 95 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-record_accessor.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Record Accessor 2 | 3 | The `record_accessor` plugin helper provides unified access to the event record. It uses `jsonpath` like syntax for the target field. With this helper, you can easily access/delete a nested field in the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/filter' 9 | 10 | module Fluent::Plugin 11 | class ExampleFilter < Filter 12 | Fluent::Plugin.register_filter('example', self) 13 | 14 | # 1. Load record_accessor helper 15 | helpers :record_accessor 16 | 17 | def configure(conf) 18 | super 19 | 20 | # 2. Call `record_accessor_create` to create object 21 | @accessor = record_accessor_create('$.user.name') 22 | end 23 | 24 | # Omit `super`, `shutdown` and other plugin APIs 25 | 26 | def filter(tag, time, record) 27 | # 3. Call `call` method to get value 28 | value = @accessor.call(record) # With `$.user.name`, access to record["user"]["name"] 29 | # ... 30 | end 31 | end 32 | end 33 | ``` 34 | 35 | ## Syntax 36 | 37 | * dot notation: `$.` is the starting parameter. Chain fields with dots `.`. 38 | 39 | For example: 40 | 41 | `$.event.level` for `record["event"]["level"]` 42 | 43 | `$.key1[0].key2` for `record["key1"][0]["key2"]` 44 | 45 | * bracket notation: `$[` starting parameter. Chain fields with `[]`. 46 | 47 | Useful for special characters, `.`, whitespace, etc. 48 | 49 | `$['dot.key'][0]['space key']` for `record["dot.key"][0]["space key"]` 50 | 51 | If you set non `$.` or `$[` starting value, e.g. `key log`, it is the same as `record["log"]`. So, using `record_accessor` does not break the existing plugin behavior. 52 | 53 | ## Methods 54 | 55 | ### `record_accessor_create(param)` 56 | 57 | This method returns the accessor object of the event record. 58 | 59 | The `param` is a `String`. 60 | 61 | See the "Syntax" section for more details. 62 | 63 | ```ruby 64 | record_accessor_create("log") 65 | record_accessor_create("$.key1.key2") 66 | record_accessor_create("$['key1'][0]['key2']") 67 | ``` 68 | 69 | After creating an object, call `call`/`delete`/`set` method with the record object. 70 | 71 | ```ruby 72 | accessor.call(record) # get record field 73 | accessor.delete(record) # delete record field 74 | accessor.set(record, value) # set new value to record field 75 | ``` 76 | 77 | NOTE: `set` method is supported since v1.10.3 78 | 79 | ## Plugins using `record_accessor` 80 | 81 | * [`filter_grep`](../filter/grep.md) 82 | * [`filter_parser`](../filter/parser.md) 83 | * [`filter_record_transformer`](../filter/record_transformer.md) 84 | 85 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 86 | 87 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-thread.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Thread 2 | 3 | The `thread` plugin helper manages threads and these threads are integrated with plugins. No need manual run or shutdown in the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load thread helper 15 | helpers :thread 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Create and run thread with unique name. 23 | thread_create(:example_thread_run, &method(:run)) 24 | end 25 | 26 | def run 27 | # ... 28 | end 29 | end 30 | end 31 | ``` 32 | 33 | The launched thread is managed by the plugin helper. No need of thread shutdown code in plugin's `shutdown` method. It shutdowns the launched threads automatically. 34 | 35 | ## Methods 36 | 37 | ### `thread_create(title)` 38 | 39 | This method creates a thread and runs it with the given routine. 40 | 41 | `title` must be unique. 42 | 43 | ```ruby 44 | # Pass block directly 45 | thread_create(:example_plugin_main) { 46 | # ... 47 | } 48 | 49 | # Pass method object with existing method 50 | thread_create(:foo_plugin_body, &method(:run)) 51 | def run 52 | # ... 53 | end 54 | ``` 55 | 56 | ### `thread_current_running?` 57 | 58 | Checks whether the current thread is running or not. This method is available in the running block. 59 | 60 | ```ruby 61 | thread_create(:example_plugin_main) { 62 | while thread_current_running? 63 | # ... 64 | end 65 | } 66 | ``` 67 | 68 | ## Plugins using `thread` 69 | 70 | * [`out_forward`](../output/forward.md) 71 | * [`in_monitor_agent`](../input/monitor_agent.md) 72 | * [`in_sample`](../input/sample.md) 73 | 74 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 75 | 76 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-timer.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Timer 2 | 3 | The `timer` plugin helper manages event timers. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load timer helper 15 | helpers :timer 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Execute timer with unique name and second unit interval 23 | timer_execute(:example_timer, 10) { 24 | # ... 25 | } 26 | end 27 | end 28 | end 29 | ``` 30 | 31 | The launched timer is managed by the plugin helper. No need of timer shutdown code in plugin's `shutdown` method. The plugin shutdowns the launched timers automatically. 32 | 33 | ## Methods 34 | 35 | ### `timer_execute(title, interval, repeat: true, &block)` 36 | 37 | This method executes the timer with the given parameters and routine. 38 | 39 | * `title`: unique symbol value 40 | * `interval`: second unit `integer`/`float` value. 41 | * `repeat`: `true`/`false` \(default: `true`\). If `false`, timer is one-shot. 42 | 43 | Code examples: 44 | 45 | ```ruby 46 | # Pass block directly. block is executed in 10 second interval. 47 | timer_execute(:example_timer, 10) { 48 | # ... 49 | } 50 | 51 | # Pass block with existing method. block is executed in 5 second and one-shot. 52 | timer_execute(:example_timer_run, 5s, repeat: false, &method(:run)) 53 | def run 54 | # ... 55 | end 56 | ``` 57 | 58 | ## Plugins using `timer` 59 | 60 | * [`out_forward`](../output/forward.md) 61 | * [`in_monitor_agent`](../input/monitor_agent.md) 62 | * [`in_sample`](../input/sample.md) 63 | 64 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 65 | 66 | -------------------------------------------------------------------------------- /plugins/input/dummy.md: -------------------------------------------------------------------------------- 1 | # `dummy` Input Plugin 2 | 3 | `in_dummy` has been renamed `in_sample` since Fluentd v1.11.12. 4 | See [`sample` input plugin](/input/sample.md) article for more details. 5 | 6 | You can keep to use following configuration in v1: 7 | 8 | ```text 9 | 10 | @type dummy 11 | dummy {"hello":"world"} 12 | 13 | ``` 14 | 15 | Fluentd v2 will remove this plugin name. 16 | 17 | 18 | ------------------------------------------------------------------------ 19 | 20 | If this article is incorrect or outdated, or omits critical information, please 21 | [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). 22 | [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native 23 | Computing Foundation (CNCF)](https://cncf.io/). All components are available 24 | under the Apache 2 License. 25 | -------------------------------------------------------------------------------- /quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | Let's get started with **Fluentd**! **Fluentd** is a fully free and fully open-source log collector that instantly enables you to have a '**Log Everything**' architecture with [125+ types of systems](https://www.fluentd.org/plugins). 4 | 5 |  6 | 7 | Fluentd treats logs as JSON, a popular machine-readable format. It is written primarily in C with a thin-Ruby wrapper that gives users flexibility. 8 | 9 | Fluentd's scalability has been proven in the field: its largest user currently collects logs from **50,000+ servers**. 10 | 11 | ## Step 1: Installing Fluentd 12 | 13 | You can install Fluentd as a package, Ruby gem, or from source. 14 | 15 | Please see the following article for details. 16 | 17 | * [Installation](../installation/) 18 | 19 | ## Step 2: Use Cases 20 | 21 | The following articles cover the typical use cases of Fluentd: 22 | 23 | * Use Cases 24 | * [Data Search like Splunk](../how-to-guides/free-alternative-to-splunk-by-fluentd.md) 25 | * [Data Filtering and Alerting](../how-to-guides/splunk-like-grep-and-alert-email.md) 26 | * [Data Analytics with Treasure Data](../how-to-guides/http-to-td.md) 27 | * [Data Collection to MongoDB](../how-to-guides/apache-to-mongodb.md) 28 | * [Data Collection to HDFS](../how-to-guides/http-to-hdfs.md) 29 | * [Data Archiving to Amazon S3](../how-to-guides/apache-to-s3.md) 30 | * Basic Configuration 31 | * [Config File](../configuration/config-file.md) 32 | * Application Logs 33 | * [Ruby](../language-bindings/ruby.md), [Java](../language-bindings/java.md), [Python](../language-bindings/python.md), [PHP](../language-bindings/php.md), 34 | 35 | [Perl](../language-bindings/perl.md), [Node.js](../language-bindings/nodejs.md), [Scala](../language-bindings/scala.md) 36 | * Happy Users :\) 37 | * [Users](https://www.fluentd.org/testimonials) 38 | 39 | ## Step 3: Learn More 40 | 41 | The following articles will provide detailed information about Fluentd: 42 | 43 | * [Architecture Overview](https://www.fluentd.org/architecture) 44 | * [Lifecycle of a Fluentd Event](life-of-a-fluentd-event.md) 45 | * Plugin Overview 46 | * [Input Plugins](../input/) 47 | * [Output Plugins](../output/) 48 | * [Buffer Plugins](../buffer/) 49 | * [Filter Plugins](../filter/) 50 | * [Parser Plugins](../parser/) 51 | * [Formatter Plugins](../formatter/) 52 | * [High Availability Configuration](../deployment/high-availability.md) 53 | * [FAQ](faq.md) 54 | 55 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 56 | 57 | -------------------------------------------------------------------------------- /quickstart/logo.md: -------------------------------------------------------------------------------- 1 | # Logo 2 | 3 | Feel free to use these logos on your slides, blog posts, etc. 4 | 5 | ## Square 6 | 7 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.png) 8 | 9 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.svg) 10 | 11 | ## Horizontal 12 | 13 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.png) 14 | 15 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.svg) 16 | 17 | ## Icon 18 | 19 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.png) 20 | 21 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.svg) 22 | 23 | ## Links 24 | 25 | * [Fluentd.ai](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd.ai) 26 | * [Complete List of Logos](https://github.com/fluent/fluentd-docs-gitbook/tree/1.0/images/logo) 27 | 28 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 29 | 30 | -------------------------------------------------------------------------------- /quickstart/support.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## Community Forum \(GitHub Discussions\) 4 | 5 | Got a question? Have some cool Fluentd-related projects? The GitHub discussion is the place to be: 6 | 7 | * [Fluentd Discussions](https://github.com/fluent/fluentd/discussions) 8 | 9 | ## Slack 10 | 11 | Join our Slack channel and talk with Fluentd developers and users directly, now. 12 | 13 | * [Slack](https://slack.fluentd.org/) 14 | 15 | ## Enterprise Support 16 | 17 | If you need support for running in production from enterprise companies you can find providers at the following link 18 | 19 | * [Enterprise Support providers](https://www.fluentd.org/enterprise_services) 20 | 21 | ## Community Resources 22 | 23 | Here are other community resources: 24 | 25 | * [StackOverflow](https://stackoverflow.com/questions/tagged/fluentd?sort=newest) 26 | * [Twitter @fluentd](https://www.twitter.com/fluentd) 27 | * [Facebook Page](https://www.facebook.com/pages/Fluentd-Log-Everything-in-JSON/196064987183037) 28 | 29 | ## Change Logs 30 | 31 | If you are interested in what's new in each version, please refer to the following links: 32 | 33 | * [Fluentd's ChangeLog](https://github.com/fluent/fluentd/blob/master/CHANGELOG.md). 34 | * [fluent-package's ChangeLog](https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md) (Successor of td-agent package) 35 | * [td-agent's ChangeLog](https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md) (End of Life in Dec, 2023) 36 | 37 | {% hint style='danger' %} 38 | The series of td-agent had already reached End of Life (EOL). td-agent should not be newly installed because no support, no new release and no security updates anymore. 39 | Use fluent-package instead. See [Upgrade to fluent-package v5](https://www.fluentd.org/blog/upgrade-td-agent-v4-to-v5) for migration. 40 | {% endhint %} 41 | 42 | ## Development 43 | 44 | If you find bugs, please report to the GitHub Issue. 45 | 46 | * [Bug/Feature Tracker](https://github.com/fluent/fluentd/issues) 47 | 48 | The source code is available at GitHub: 49 | 50 | * [Fluentd](https://github.com/fluent/fluentd/) 51 | 52 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 53 | 54 | -------------------------------------------------------------------------------- /service_discovery/README.md: -------------------------------------------------------------------------------- 1 | # Service Discovery Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](../input/) 6 | * [Parser](../parser/) 7 | * [Filter](../filter/) 8 | * [Output](../output/) 9 | * [Formatter](../formatter/) 10 | * [Storage](../storage/) 11 | * [Service Discovery](./) 12 | * [Buffer](../buffer/) 13 | * [Metrics](../metrics/) 14 | 15 | This article gives an overview of the Service Discovery Plugin. 16 | 17 | ## Overview 18 | 19 | Some plugins support `` \(e.g. [`out_forward`](../output/forward.md)\). Sometimes, the service discovery for an output plugin does not meet one's needs. Fluentd has a pluggable system called Service Discovery that lets the user extend and reuse custom output service discovery. 20 | 21 | ## How To Use 22 | 23 | Here is a simple example to update target by reading file \(`/etc/fluentd/sd.yaml`\) with `out_forward` and `service_discovery`: 24 | 25 | ```text 26 | 27 | @type forward 28 | 29 | 30 | @type file 31 | path "/etc/fluentd/sd.yaml" 32 | 33 | 34 | ``` 35 | 36 | ## List of Built-in Service Discovery 37 | 38 | * [`static`](static.md) 39 | * [`file`](file.md) 40 | * [`srv`](srv.md) 41 | 42 | ## List of Core Output Plugins with Service Discovery support 43 | 44 | * [`out_forward`](../output/forward.md) 45 | 46 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 47 | 48 | -------------------------------------------------------------------------------- /service_discovery/file.md: -------------------------------------------------------------------------------- 1 | # file 2 | 3 | The `file` service discovery plugin updates the targets by reading the local file. YAML and JSON are the allowed file formats. 4 | 5 | ## Example Configuration 6 | 7 | Here is an example with `out_forward` updating targets by sending data: 8 | 9 | ```text 10 | 11 | @type forward 12 | 13 | 14 | @type file 15 | path "/etc/fluentd/sd.yaml" 16 | 17 | 18 | ``` 19 | 20 | Here is an example of target list file \(`/etc/fluentd/sd.yaml`\): 21 | 22 | ```text 23 | - 'host': 127.0.0.1 24 | 'port': 24224 25 | 'weight': 1 26 | 'name': server1 27 | - 'host': 127.0.0.1 28 | 'port': 24225 29 | 'weight': 1 30 | 'name': server2 31 | ``` 32 | 33 | ## Parameters 34 | 35 | ### `@type` 36 | 37 | The value must be `file`. 38 | 39 | ### `path` 40 | 41 | | type | default | version | 42 | | :--- | :--- | :--- | 43 | | string | `'/etc/fluent/sd.yaml'` | 1.8.0 | 44 | 45 | The path of the target list. 46 | 47 | The `Content-Type` is determined by file extension i.e.: 48 | 49 | * YAML: yaml, yml 50 | * JSON: json 51 | 52 | ### `conf_encoding` 53 | 54 | | type | default | version | 55 | | :--- | :--- | :--- | 56 | | string | `'utf-8'` | 1.8.0 | 57 | 58 | The encoding of the configuration file. 59 | 60 | ### Parameters in Target List File 61 | 62 | Each target has following parameters: 63 | 64 | * `host` 65 | * `name` 66 | * `port` 67 | * `shared_key` 68 | * `username` 69 | * `password` 70 | * `standby` 71 | * `weight` 72 | 73 | #### `host` 74 | 75 | | type | default | version | 76 | | :--- | :--- | :--- | 77 | | string | `nil` | 1.8.0 | 78 | 79 | The IP address or hostname of the server. It is a required parameter. 80 | 81 | #### `port` 82 | 83 | | type | default | version | 84 | | :--- | :--- | :--- | 85 | | integer | | 1.8.0 | 86 | 87 | The port number of the host. It is a required parameter. 88 | 89 | #### `name` 90 | 91 | | type | default | version | 92 | | :--- | :--- | :--- | 93 | | string | `nil` | 1.8.0 | 94 | 95 | The name of the server. 96 | 97 | #### `shared_key` 98 | 99 | | type | default | version | 100 | | :--- | :--- | :--- | 101 | | string | `nil` | 1.8.0 | 102 | 103 | The shared key per server. 104 | 105 | #### `username` 106 | 107 | | type | default | version | 108 | | :--- | :--- | :--- | 109 | | string | `nil` | 1.8.0 | 110 | 111 | The username for authentication. 112 | 113 | #### `password` 114 | 115 | | type | default | version | 116 | | :--- | :--- | :--- | 117 | | string | `nil` | 1.8.0 | 118 | 119 | The password for authentication. 120 | 121 | #### `standby` 122 | 123 | | type | default | version | 124 | | :--- | :--- | :--- | 125 | | bool | `nil` | 1.8.0 | 126 | 127 | #### `weight` 128 | 129 | | type | default | version | 130 | | :--- | :--- | :--- | 131 | | integer | 60 | 1.8.0 | 132 | 133 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 134 | 135 | -------------------------------------------------------------------------------- /service_discovery/srv.md: -------------------------------------------------------------------------------- 1 | # srv 2 | 3 | The `srv` service discovery plugin updates the targets by [SRV Record](https://tools.ietf.org/html/rfc2782). 4 | 5 | ## Example Configuration 6 | 7 | Here is an example of `out_forward` updating the targets to get the SRV record from `_fluentd._tcp.example.com`: 8 | 9 | ```text 10 | 11 | @type forward 12 | 13 | 14 | @type srv 15 | service fluentd 16 | proto tcp 17 | hostname example.com 18 | 19 | 20 | ``` 21 | 22 | ## Parameters 23 | 24 | ### `@type` 25 | 26 | The value must be `srv`. 27 | 28 | ### `service` \(required\) 29 | 30 | | type | default | version | 31 | | :--- | :--- | :--- | 32 | | string | `nil` | 1.10.0 | 33 | 34 | Service without the underscore in [RFC2782](https://tools.ietf.org/html/rfc2782). 35 | 36 | ### `proto` 37 | 38 | | type | default | version | 39 | | :--- | :--- | :--- | 40 | | string | `nil` | 1.10.0 | 41 | 42 | Proto without the underscore in [RFC2782](https://tools.ietf.org/html/rfc2782). 43 | 44 | It is a required parameter. 45 | 46 | ### `hostname` 47 | 48 | | type | default | version | 49 | | :--- | :--- | :--- | 50 | | string | required parameters | 1.10.0 | 51 | 52 | The name in [RFC2782](https://tools.ietf.org/html/rfc2782). 53 | 54 | ### `dns_server_host` 55 | 56 | | type | default | version | 57 | | :--- | :--- | :--- | 58 | | string | `nil` | 1.10.0 | 59 | 60 | The hostname of the DNS server to request the SRV record. 61 | 62 | ### `interval` 63 | 64 | | type | default | version | 65 | | :--- | :--- | :--- | 66 | | integer | 60 | 1.10.0 | 67 | 68 | The interval of sending requests to DNS server. 69 | 70 | ### `dns_lookup` 71 | 72 | | type | default | version | 73 | | :--- | :--- | :--- | 74 | | bool | `true` | 1.10.0 | 75 | 76 | Resolves the hostname to IP address of the SRV's Target. 77 | 78 | #### `shared_key` 79 | 80 | | type | default | version | 81 | | :--- | :--- | :--- | 82 | | string | `nil` | 1.10.0 | 83 | 84 | The shared key per server. 85 | 86 | #### `username` 87 | 88 | | type | default | version | 89 | | :--- | :--- | :--- | 90 | | string | `nil` | 1.10.0 | 91 | 92 | The username for authentication. 93 | 94 | #### `password` 95 | 96 | | type | default | version | 97 | | :--- | :--- | :--- | 98 | | string | `nil` | 1.10.0 | 99 | 100 | The password for authentication. 101 | 102 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 103 | 104 | -------------------------------------------------------------------------------- /service_discovery/static.md: -------------------------------------------------------------------------------- 1 | # static 2 | 3 | The `static` service discovery plugin sets the list of targets. 4 | 5 | ## Example Configuration 6 | 7 | Here is an example with `out_forward` that defines the list of targets similar to the `server` directive: 8 | 9 | ```text 10 | 11 | @type forward 12 | 13 | 14 | @type static 15 | 16 | host 127.0.0.1 17 | 18 | 19 | 20 | ``` 21 | 22 | ## Parameters 23 | 24 | ### `@type` 25 | 26 | The value must be `static`. 27 | 28 | #### `host` 29 | 30 | | type | default | version | 31 | | :--- | :--- | :--- | 32 | | string | | 1.8.0 | 33 | 34 | The IP address or hostname of the server. It is a required parameter. 35 | 36 | #### `port` 37 | 38 | | type | default | version | 39 | | :--- | :--- | :--- | 40 | | integer | 24224 | 1.8.0 | 41 | 42 | The port number of the host. 43 | 44 | #### `name` 45 | 46 | | type | default | version | 47 | | :--- | :--- | :--- | 48 | | string | nil | 1.8.0 | 49 | 50 | The name of the server. 51 | 52 | #### `shared_key` 53 | 54 | | type | default | version | 55 | | :--- | :--- | :--- | 56 | | string | nil | 1.8.0 | 57 | 58 | The shared key per server. 59 | 60 | #### `username` 61 | 62 | | type | default | version | 63 | | :--- | :--- | :--- | 64 | | string | nil | 1.8.0 | 65 | 66 | The username for authentication. 67 | 68 | #### `password` 69 | 70 | | type | default | version | 71 | | :--- | :--- | :--- | 72 | | string | `''`\(empty string\) | 1.8.0 | 73 | 74 | The password for authentication. 75 | 76 | #### `standby` 77 | 78 | | type | default | version | 79 | | :--- | :--- | :--- | 80 | | bool | false | 1.8.0 | 81 | 82 | #### `weight` 83 | 84 | | type | default | version | 85 | | :--- | :--- | :--- | 86 | | integer | 60 | 1.8.0 | 87 | 88 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 89 | 90 | -------------------------------------------------------------------------------- /storage/README.md: -------------------------------------------------------------------------------- 1 | # Storage Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](../input/) 6 | * [Parser](../parser/) 7 | * [Filter](../filter/) 8 | * [Output](../output/) 9 | * [Formatter](../formatter/) 10 | * [Storage](./) 11 | * [Service Discovery](../service_discovery/) 12 | * [Buffer](../buffer/) 13 | * [Metrics](../metrics/) 14 | 15 | This article gives an overview of Storage Plugin. 16 | 17 | ## Overview 18 | 19 | Sometimes, the input/filter/output plugin needs to save its internal state in memory, storage, or key-value store. Fluentd has a pluggable system called Storage that lets a plugin store and reuse its internal state as key-value pairs. 20 | 21 | ## How To Use 22 | 23 | For an input, an output, and filter plugin that supports Storage, the `` directive can be used to store key-value pair into a key-value store such as a JSON file, MongoDB, Redis, etc. 24 | 25 | Here is an example with `in_sample`: 26 | 27 | ```text 28 | 29 | @type sample 30 | tag docs.fluentd.storage 31 | 32 | @type my_custom_storage 33 | 34 | 35 | ``` 36 | 37 | ## List of Built-in Storage 38 | 39 | * [`local`](local.md) 40 | 41 | ## List of Core Plugins with Storage support 42 | 43 | * [`in_sample`](../input/sample.md) 44 | 45 | ## List of 3rd party storage plugins 46 | 47 | NOTE: This 3rd party storage plugin list does not fully covers all of them. 48 | 49 | * [fluent-plugin-storage-leveldb](https://github.com/cosmo0920/fluent-plugin-storage-leveldb) 50 | * [fluent-plugin-storage-memcached](https://github.com/cosmo0920/fluent-plugin-storage-memcached) 51 | * [fluent-plugin-storage-mongo](https://github.com/cosmo0920/fluent-plugin-storage-mongo) 52 | * [fluent-plugin-storage-redis](https://github.com/cosmo0920/fluent-plugin-storage-redis) 53 | 54 | ## List of 3rd party Plugins with Storage support 55 | 56 | NOTE: This 3rd party plugin list does not fully covers all of them. 57 | 58 | * [fluent-plugin-systemd](https://github.com/fluent-plugin-systemd/fluent-plugin-systemd) 59 | * [fluent-plugin-windows-eventlog](https://github.com/fluent/fluent-plugin-windows-eventlog) 60 | 61 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 62 | 63 | -------------------------------------------------------------------------------- /storage/local.md: -------------------------------------------------------------------------------- 1 | # local 2 | 3 | The `local` storage plugin stores the key-value pair into JSON file on local storage. 4 | 5 | ## Parameters 6 | 7 | ### `path` 8 | 9 | Specifies the pathname to save the key-value pair. \(default: `nil`\) 10 | 11 | ### `mode` 12 | 13 | Specifies the file access mode. \(default: `0644`\) 14 | 15 | ### `dir_mode` 16 | 17 | Specifies the directory access mode. \(default: `0755`\) 18 | 19 | ### `pretty_print` 20 | 21 | Outputs the human-readable formatted JSON. \(default: `false`\) 22 | 23 | ## Attributes 24 | 25 | ### `conf.arg` 26 | 27 | Note that `conf.arg` provides an alternative `path` parameter. 28 | 29 | ```text 30 | 31 | @type local 32 | 33 | 34 | 35 | root_dir tmp 36 | 37 | ``` 38 | 39 | The above configuration will save the internal states, which are handled by `storage_local` under `tmp` directory. 40 | 41 | NOTE: Specifying the file path in the `path` parameter does not support the multi-workers feature. Instead, you should specify a directory there. 42 | 43 | ## Example 44 | 45 | With this configuration: 46 | 47 | ```text 48 | 49 | @type sample 50 | auto_increment_key count 51 | tag storage.sample 52 | 53 | 54 | @type local 55 | path storage/sample.json 56 | 57 | 58 | ``` 59 | 60 | The above configuration will save the internal states such as `auto_increment_value` to `storage/sample.json`. As a result, you can resume from the next value of previous `count` when restarting fluentd. 61 | 62 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 63 | 64 | --------------------------------------------------------------------------------
[OUTPUT]
name file
path /var/log/fluent
format plain
match *
<match pattern>
@type file
path /var/log/fluent/myapp
</match> 47 | 48 |
[^ ]*) (?[^ ]*)(?: "(?(?:[^\"]|\\.)*)" "(?(?:[^\"]|\\.)*)")?$/ 15 | time_format %d/%b/%Y:%H:%M:%S %z 16 | ``` 17 | 18 | `host`, `user`, `method`, `path`, `code`, `size`, `referer` and `agent` are included in the event record. `time` is used for the event time. 19 | 20 | `code` and `size` fields are converted to integer type automatically. And, if the field value is `-`, it is interpreted as `nil`. See **Result Example**. 21 | 22 | ## Example 23 | 24 | This incoming event 25 | 26 | ```text 27 | 192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" 28 | ``` 29 | 30 | is parsed as: 31 | 32 | ```text 33 | time: 34 | 1362020400 (28/Feb/2013:12:00:00 +0900) 35 | 36 | record: 37 | { 38 | "user" : nil, 39 | "method" : "GET", 40 | "code" : 200, 41 | "size" : 777, 42 | "host" : "192.168.0.1", 43 | "path" : "/", 44 | "referer": nil, 45 | "agent" : "Opera/12.0" 46 | } 47 | ``` 48 | 49 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 50 | 51 | -------------------------------------------------------------------------------- /parser/apache_error.md: -------------------------------------------------------------------------------- 1 | # apache\_error 2 | 3 | The `apache_error` parser plugin parses apache error logs. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ## Regexp patterns 10 | 11 | This is regexp pattern of this plugin: 12 | 13 | ```text 14 | expression /^\[[^ ]* (?[^\]]*)\] \[(?[^\]]*)\](?: \[pid (?[^\]]*)\])? \[client (?[^\]]*)\] (?.*)$/ 15 | ``` 16 | 17 | `level`, `pid`, `client` and `message` are included in the event record. `time` is used for the event time. 18 | 19 | ## Example 20 | 21 | This incoming event: 22 | 23 | ```text 24 | [Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration 25 | ``` 26 | 27 | is parsed as: 28 | 29 | ```text 30 | time: 31 | 971242372 (Wed Oct 11 14:32:52 2000) 32 | 33 | record: 34 | { 35 | "level" : "error", 36 | "client" : "127.0.0.1", 37 | "message": "client denied by server configuration" 38 | } 39 | ``` 40 | 41 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 42 | 43 | -------------------------------------------------------------------------------- /parser/csv.md: -------------------------------------------------------------------------------- 1 | # csv 2 | 3 | The `csv` parser plugin parses CSV format. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `keys` 10 | 11 | | type | default | version | 12 | | :--- | :--- | :--- | 13 | | array of string | required parameter | 0.14.9 | 14 | 15 | The array of names for fields on each line. 16 | 17 | ### `delimiter` 18 | 19 | | type | default | version | 20 | | :--- | :--- | :--- | 21 | | string | , | 0.14.2 | 22 | 23 | The delimiter \(character or string\) separating CSV values. 24 | 25 | ### `parser_type` 26 | 27 | | type | default | available values | version | 28 | | :--- | :--- | :--- | :--- | 29 | | enum | normal | normal/fast | 1.7.0 | 30 | 31 | The parser type used to parse the log line. 32 | 33 | * `normal` uses Ruby's 34 | 35 | [`CSV.parse_line`](http://ruby-doc.org/stdlib-2.4.1/libdoc/csv/rdoc/CSV.html#method-c-parse_line) 36 | 37 | method. 38 | 39 | * `fast` uses its own lightweight implementation. This parser is several times 40 | 41 | faster than `normal` but it supports only typical patterns. 42 | 43 | Supported CSV formats by `fast`: 44 | 45 | ```text 46 | # non-quoted 47 | value1,value2,value3,value4,value5 48 | 49 | # quoted 50 | "value1","val,ue2","va,lu,e3","val ue4","" 51 | 52 | # escaped 53 | "message","mes""sage","""message""",,"""""" 54 | 55 | # mixed 56 | message,"mes,sage","me,ssa,ge",mess age,"" 57 | ``` 58 | 59 | If your CSV format is not matched with the above patterns, use `normal` parser instead. 60 | 61 | ## Example 62 | 63 | With this configuration: 64 | 65 | ```text 66 | 67 | @type csv 68 | keys time,host,req_id,user 69 | time_key time 70 | 71 | ``` 72 | 73 | This incoming event: 74 | 75 | ```text 76 | 2013/02/28 12:00:00,192.168.0.1,111,- 77 | ``` 78 | 79 | is parsed as: 80 | 81 | ```text 82 | time: 83 | 1362020400 (2013/02/28/ 12:00:00) 84 | 85 | record: 86 | { 87 | "host" : "192.168.0.1", 88 | "req_id" : "111", 89 | "user" : "-" 90 | } 91 | ``` 92 | 93 | If you set `null_value_pattern '-'` in the configuration, `user` field becomes `nil` instead of `"-"`. 94 | 95 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 96 | 97 | -------------------------------------------------------------------------------- /parser/json.md: -------------------------------------------------------------------------------- 1 | # json 2 | 3 | The `json` parser plugin parses JSON logs. One JSON map per line. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `json_parser` 10 | 11 | | type | default | available | version | 12 | | :--- | :--- | :--- | :--- | 13 | | enum | oj | oj, yajl, json | 0.14.0 | 14 | 15 | Sets the JSON parser. If you have a problem with the configured parser, check the other available parser types. 16 | 17 | NOTE: If you want to enable `json_parser oj` by default, The `oj` gem must be installed separately. This is because `oj` gem is not required from fluentd by default. If `oj` gem is not installed, `json` is used as a fallback. (The fallback has been changed from `yajl` to `json` since v1.19.0.) 18 | 19 | Here is a simple comparison: 20 | 21 | * `oj`: Faster json parser 22 | * `yajl`: Mainly for stream parsing 23 | * `json`: Standard bundled library 24 | 25 | ### `stream_buffer_size` 26 | 27 | | type | default | version | 28 | | :--- | :--- | :--- | 29 | | integer | 8192 | 1.0.0 | 30 | 31 | Set the buffer size that Yajl will use when parsing streaming input. 32 | 33 | If you specify the smaller buffer size, you could get outcome response instantaneously in exchange for performance penalty. 34 | 35 | See also: [Method: Yajl::Parser\#parse](https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse) 36 | 37 | ### `time_type` 38 | 39 | `json` parser changes the default value of `time_type` to `float`. If you want to parse string field, set `time_type` and `time_format` like this: 40 | 41 | ```text 42 | # conf 43 | @type json 44 | time_type string 45 | time_format %d/%b/%Y:%H:%M:%S %z 46 | 47 | # record example 48 | {"key":"value","time":"28/Feb/2013:12:00:00 +0900"} 49 | ``` 50 | 51 | See also [`parse` Section](../configuration/parse-section.md#time-parameters) article. 52 | 53 | ## Example 54 | 55 | This incoming event: 56 | 57 | ```text 58 | {"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"} 59 | ``` 60 | 61 | is parsed as: 62 | 63 | ```text 64 | time: 65 | 1362020400 (2013-02-28 12:00:00 +0900) 66 | 67 | record: 68 | { 69 | "host" : "192.168.0.1", 70 | "size" : 777, 71 | "method": "PUT", 72 | } 73 | ``` 74 | 75 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 76 | 77 | -------------------------------------------------------------------------------- /parser/ltsv.md: -------------------------------------------------------------------------------- 1 | # ltsv 2 | 3 | The `ltsv` parser plugin parses [LTSV](http://ltsv.org/) format. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `delimiter` 10 | 11 | | type | default | version | 12 | | :--- | :--- | :--- | 13 | | string | `\t` | 0.14.0 | 14 | 15 | The delimiter \(character or string\) separating LTSV values. 16 | 17 | ### `delimiter_pattern` 18 | 19 | | type | default | version | 20 | | :--- | :--- | :--- | 21 | | regexp | nil | 1.2.0 | 22 | 23 | The delimiter pattern of TSV values. This parameter overwrites `delimiter` parameter if specified. 24 | 25 | `delimiter_pattern` is string type before 1.2.0. 26 | 27 | ### `label_delimiter` 28 | 29 | | type | default | version | 30 | | :--- | :--- | :--- | 31 | | string | : | 0.14.0 | 32 | 33 | The delimiter character between field name and value. 34 | 35 | ## Example for LTSV 36 | 37 | This incoming event: 38 | 39 | ```text 40 | time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111\tuser:- 41 | ``` 42 | 43 | is parsed as: 44 | 45 | ```text 46 | time: 47 | 1362020400 (2013/02/28/ 12:00:00) 48 | 49 | record: 50 | { 51 | "host" : "192.168.0.1", 52 | "req_id" : "111", 53 | "user" : "-" 54 | } 55 | ``` 56 | 57 | If you set `null_value_pattern '-'` in the configuration, `user` field becomes `nil` instead of `"-"`. 58 | 59 | ## Example with `delimiter_pattern` 60 | 61 | With this configuration: 62 | 63 | ```text 64 | 65 | @type ltsv 66 | delimiter_pattern /\s+/ 67 | label_delimiter = 68 | 69 | ``` 70 | 71 | This incoming event: 72 | 73 | ```text 74 | timestamp=1362020400 host=192.168.0.1 req_id=111 user=- 75 | ``` 76 | 77 | is parsed as: 78 | 79 | ```text 80 | record: 81 | { 82 | "timestamp": "1362020400", 83 | "host" : "192.168.0.1", 84 | "req_id" : "111", 85 | "user" : "-" 86 | } 87 | ``` 88 | 89 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 90 | 91 | -------------------------------------------------------------------------------- /parser/msgpack.md: -------------------------------------------------------------------------------- 1 | # msgpack 2 | 3 | The `msgpack` parser plugin parses the MessagePack data. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ## Example 10 | 11 | This incoming event: 12 | 13 | ```text 14 | \x82\xA7message\xADHello msgpack\xA3numd 15 | ``` 16 | 17 | is parsed as: 18 | 19 | ```text 20 | time: 21 | 1582648111.269613 (current time) 22 | 23 | record: 24 | {"message":"Hello msgpack", "num":100} 25 | ``` 26 | 27 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 28 | 29 | -------------------------------------------------------------------------------- /parser/nginx.md: -------------------------------------------------------------------------------- 1 | # nginx 2 | 3 | The `nginx` parser plugin parses the default Nginx logs. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ## Regexp Patterns 10 | 11 | Here is the regexp and time format patterns of this plugin: 12 | 13 | ```text 14 | expression /^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)?" (?[^ ]*) (?[^ ]*)(?: "(?[^\"]*)" "(?[^\"]*)"(?:\s+(?[^ ]+))?)?$/ 15 | time_format %d/%b/%Y:%H:%M:%S %z 16 | ``` 17 | 18 | `remote`, `user`, `method`, `path`, `code`, `size`, `referer`, `agent` and `http_x_forwarded_for` are included in the event record. `time` is used for the event time. 19 | 20 | ## Example 21 | 22 | This incoming event: 23 | 24 | ```text 25 | 127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" - 26 | ``` 27 | 28 | is parsed as: 29 | 30 | ```text 31 | time: 32 | 1362020400 (28/Feb/2013:12:00:00 +0900) 33 | 34 | record: 35 | { 36 | "remote" : "127.0.0.1", 37 | "host" : "192.168.0.1", 38 | "user" : "-", 39 | "method" : "GET", 40 | "path" : "/", 41 | "code" : "200", 42 | "size" : "777", 43 | "referer" : "-", 44 | "agent" : "Opera/12.0", 45 | "http_x_forwarded_for": "-" 46 | } 47 | ``` 48 | 49 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 50 | 51 | -------------------------------------------------------------------------------- /parser/none.md: -------------------------------------------------------------------------------- 1 | # none 2 | 3 | The `none` parser plugin parses the line as-is with the single field. This format is to defer the parsing/structuring of the data. 4 | 5 | This parser is often used in conjunction with [`single_value`](../formatter/single_value.md) format in output plugin. 6 | 7 | ## Parameters 8 | 9 | See [Parse Section Configurations](../configuration/parse-section.md). 10 | 11 | ### `message_key` 12 | 13 | | type | default | version | 14 | | :--- | :--- | :--- | 15 | | string | message | 0.14.0 | 16 | 17 | Specifies the field name to contain logs. 18 | 19 | ## Example 20 | 21 | This incoming event: 22 | 23 | ```text 24 | Hello world. I am a line of log! 25 | ``` 26 | 27 | is parsed as: 28 | 29 | ```text 30 | time: 31 | 1362020400 (current time) 32 | 33 | record: 34 | {"message":"Hello world. I am a line of log!"} 35 | ``` 36 | 37 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 38 | 39 | -------------------------------------------------------------------------------- /parser/tsv.md: -------------------------------------------------------------------------------- 1 | # tsv 2 | 3 | The `tsv` parser plugin parses the TSV format. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `keys` 10 | 11 | | type | default | version | 12 | | :--- | :--- | :--- | 13 | | array of string | required parameter | 0.14.9 | 14 | 15 | The array of names for fields on each line. 16 | 17 | ### `delimiter` 18 | 19 | | type | default | version | 20 | | :--- | :--- | :--- | 21 | | string | `\t` | 0.14.0 | 22 | 23 | The delimiter \(character or string\) separating TSV values. 24 | 25 | ## Example 26 | 27 | With this configuration: 28 | 29 | ```text 30 | 31 | @type tsv 32 | keys time,host,req_id,user 33 | time_key time 34 | 35 | ``` 36 | 37 | This incoming event: 38 | 39 | ```text 40 | 2013/02/28 12:00:00\t192.168.0.1\t111\t- 41 | ``` 42 | 43 | is parsed as: 44 | 45 | ```text 46 | time: 47 | 1362020400 (2013/02/28/ 12:00:00) 48 | 49 | record: 50 | { 51 | "host" : "192.168.0.1", 52 | "req_id" : "111", 53 | "user" : "-" 54 | } 55 | ``` 56 | 57 | If you set `null_value_pattern '-'` in the configuration, `user` field becomes `nil` instead of `"-"`. 58 | 59 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 60 | 61 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-buffer.md: -------------------------------------------------------------------------------- 1 | # How to Write Buffer Plugin 2 | 3 | TODO: Write 4 | 5 | ## Methods 6 | 7 | ### evacuate_chunk 8 | 9 | You can override this method to add feature to evacuate chunks before clearing the queue when reaching the retry limit. 10 | See [Buffer - Handling Successive Failures](../buffer/README.md#handling-successive-failures) for details. 11 | 12 | ```ruby 13 | def evacuate_chunk(chunk) 14 | unless chunk.is_a?(Fluent::Plugin::Buffer::FileChunk) 15 | raise ArgumentError, "The chunk must be FileChunk, but it was #{chunk.class}." 16 | end 17 | 18 | backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id) 19 | FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir) 20 | 21 | FileUtils.copy([chunk.path, chunk.meta_path], backup_dir) 22 | log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id) 23 | rescue => e 24 | log.error "unexpected error while evacuating chunk files.", error: e 25 | end 26 | ``` 27 | 28 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 29 | 30 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-service_discovery.md: -------------------------------------------------------------------------------- 1 | # How to Write Service Discovery Plugin 2 | 3 | TODO: Write 4 | 5 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 6 | 7 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-storage.md: -------------------------------------------------------------------------------- 1 | # How to Write Storage Plugin 2 | 3 | TODO: Write 4 | 5 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 6 | 7 | -------------------------------------------------------------------------------- /plugin-helper-overview/README.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper API 2 | 3 | Fluentd provides plugin helpers to encapsulate and make commonly implemented features available such as timer, threading, formatting, parsing, ensuring configuration syntax's backward compatibility, etc. 4 | 5 | ## How To Use 6 | 7 | To use plugin helpers, call `helpers(*snake_case_symbols)` method to make them available. 8 | 9 | Example: 10 | 11 | ```ruby 12 | helpers :timer, :storage, :compat_parameters 13 | ``` 14 | 15 | It will include `Timer`, `Storage`. and `CompatParameters` plugin helpers. 16 | 17 | ## Built-in Plugin Helpers 18 | 19 | * [`child_process`](api-plugin-helper-child_process.md) 20 | * [`compat_parameters`](api-plugin-helper-compat_parameters.md) 21 | * [`event_emitter`](api-plugin-helper-event_emitter.md) 22 | * [`event_loop`](api-plugin-helper-event_loop.md) 23 | * [`extract`](api-plugin-helper-extract.md) 24 | * [`formatter`](api-plugin-helper-formatter.md) 25 | * [`inject`](api-plugin-helper-inject.md) 26 | * [`metrics`](api-plugin-helper-metrics.md) 27 | * [`parser`](api-plugin-helper-parser.md) 28 | * [`record_accessor`](api-plugin-helper-record_accessor.md) 29 | * [`server`](api-plugin-helper-server.md) 30 | * [`socket`](api-plugin-helper-socket.md) 31 | * [`storage`](api-plugin-helper-storage.md) 32 | * [`thread`](api-plugin-helper-thread.md) 33 | * [`timer`](api-plugin-helper-timer.md) 34 | * [`http_server`](api-plugin-helper-http_server.md) 35 | 36 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 37 | 38 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-child_process.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Child Process 2 | 3 | The `child_process` helper manages the child processes' lifecycle. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load child_process helper 15 | helpers :child_process 16 | 17 | # Omit configure, shutdown and other plugin API 18 | 19 | def start 20 | super 21 | # 2. Execute child process with unique name 22 | child_process_execute(:exec_external_command, 'external_command', immediate: true, mode: [:read]) do 23 | # ... 24 | end 25 | end 26 | end 27 | end 28 | ``` 29 | 30 | The launched child process is managed by the plugin helper which automatically `shutdown`s it. 31 | 32 | ## Methods 33 | 34 | ### `child_process_execute(title, sub_process_name, arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false, mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil, internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil, wait_timeout: nil, on_exit_callback: nil, &block)` 35 | 36 | This method executes `child_process` with the given parameters and routine. 37 | 38 | * `title`: unique symbol value 39 | * `sub_process_name`: sub process name value 40 | * `interval`: Second unit `integer`/`float` value. 41 | * `immediate`: `true`/`false`. Default is `false`. 42 | * `parallel`: `true`/`false`. Default is `false`. 43 | * `mode`: \[`:read`, `:write`\]. Default is \[`:read`, `:write`\]. 44 | * `stderr`: Connect stderr or not. Default is `:discard`. 45 | * `env`: Environment valuables. Default is {}. 46 | * `unsetenv`: `true`/`false`. Default is `false` 47 | * `chdir`: Working directory. Default is `nil`. 48 | * `internal_encoding`: Internal character encoding. Default is `utf-8`. 49 | * `external_encoding`: External character encoding. Default is `ascii-8bit`' 50 | * `scrub`: `true`/`false`. Default is `true`. 51 | * `replace_string`: Replace the invalid code point with the specified character. 52 | 53 | Default is `nil`. 54 | 55 | * `wait_timeout`: Set timeout seconds. Default is `nil`. 56 | * `on_exit_callback`: Set callback function. Default is `nil` 57 | 58 | Code examples: 59 | 60 | ```ruby 61 | # Pass block directly. block is executed with 10 second interval. 62 | child_process_execute(:exec_awesome_command, @command, interval: 10, mode: [:read]) do |io| 63 | # ... 64 | end 65 | 66 | # Pass block with existing method. block is executed once. 67 | child_process_execute(:exec_awesome_command, @command, immediate: true, mode: [:read], &method(:run)) 68 | 69 | def run(io) 70 | # ... 71 | end 72 | ``` 73 | 74 | ## Plugins using `child_process` 75 | 76 | * [`in_exec`](../input/exec.md) 77 | * [`out_exec`](../output/exec.md) 78 | * [`out_exec_filter`](../output/exec_filter.md) 79 | 80 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 81 | 82 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-event_emitter.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Event Emitter 2 | 3 | The `event_emitter` plugin helper introduces the `router` method to the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class RelabelOutput < Output 12 | Fluent::Plugin.register_output('relabel', self) 13 | helpers :event_emitter 14 | 15 | def multi_workers_ready? 16 | true 17 | end 18 | 19 | def process(tag, es) 20 | router.emit_stream(tag, es) 21 | end 22 | end 23 | end 24 | ``` 25 | 26 | ## Methods 27 | 28 | ### `router` 29 | 30 | This method returns a `Fluent::EventRouter` instance. 31 | 32 | Code example: 33 | 34 | ```ruby 35 | # emit event 36 | router.emit(tag, time, record) 37 | 38 | # emit event stream 39 | router.emit_stream(tag, es) 40 | 41 | # emit error event 42 | begin 43 | # do something 44 | rescue => error 45 | # Route event to @ERROR label or 46 | # log error message when @ERROR label is not defined in configuration 47 | router.emit_error_event(tag, time, record, error) 48 | end 49 | ``` 50 | 51 | ## Plugins using `event_emitter` 52 | 53 | * All input plugins 54 | * All filter plugins 55 | * [`out_exec`](../output/exec.md) 56 | * [`out_relabel`](../output/relabel.md) 57 | * [`out_copy`](../output/copy.md) 58 | * [`out_roundrobin`](../output/roundrobin.md) 59 | 60 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 61 | 62 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-event_loop.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Event Loop 2 | 3 | The `event_loop` plugin helper manages event loops. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load `event_loop_helper` 15 | helpers :event_loop 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Attach watcher 23 | watcher = Coolio::TCPServer.new(...) 24 | event_loop_attach(watcher) 25 | end 26 | end 27 | end 28 | ``` 29 | 30 | The attached watcher is managed by the plugin. No need of watcher detach code in plugin's `shutdown`. The plugin shutdowns the attached watchers automatically. 31 | 32 | ## Methods 33 | 34 | ### `event_loop_attach(watcher)` 35 | 36 | This method attaches watcher to the event loop. 37 | 38 | * `watcher`: `Coolio::Watcher` instances 39 | 40 | ## Plugins using `event_loop` 41 | 42 | * [`in_http`](../input/http.md) 43 | * [`in_tail`](../input/tail.md) 44 | 45 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 46 | 47 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-extract.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Extract 2 | 3 | The `extract` plugin helper extracts `tag` or `time` from the event `record` according to the configuration. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput > Output 12 | Fluent::Plugin.register_output('Example') 13 | 14 | def process(tag, es) 15 | es.each do |time, record| 16 | new_tag = extract_tag_from_record(record) 17 | new_time = extract_time_from_record(record) 18 | end 19 | # ... 20 | end 21 | end 22 | ``` 23 | 24 | For more details, see [Extract section](../configuration/extract-section.md). 25 | 26 | ## Methods 27 | 28 | ### `extract_tag_from_record(record)` 29 | 30 | This method extracts `tag` from the given record. 31 | 32 | * `record`: event record 33 | 34 | Example: 35 | 36 | ```ruby 37 | new_tag = extract_tag_from_record(record) 38 | ``` 39 | 40 | ### `extract_time_from_record(record)` 41 | 42 | This method extracts `time` from the given record. 43 | 44 | * `record`: event record 45 | 46 | Example: 47 | 48 | ```ruby 49 | new_time = extract_time_from_record(record) 50 | ``` 51 | 52 | ## Plugins using `extract` 53 | 54 | * [`in_exec`](../input/exec.md) 55 | * [`in_tcp`](../input/tcp.md) 56 | * [`in_udp`](../input/udp.md) 57 | * [`out_exec_filter`](../output/exec_filter.md) 58 | 59 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 60 | 61 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-formatter.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Formatter 2 | 3 | The `formatter` plugin helper manages the lifecycle of the formatter plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load formatter helper 15 | helpers :formatter 16 | 17 | # Omit `shutdown` and other plugin APIs 18 | 19 | def configure(conf) 20 | super 21 | 22 | # 2. Call `formatter_create` to create object 23 | @formatter = formatter_create(usage: 'awesome_formatter') 24 | end 25 | 26 | def format(tag, time, record) 27 | # 3. Call `format` method to format `record` 28 | @formatter.format(tag, time, record) 29 | end 30 | end 31 | end 32 | ``` 33 | 34 | For more details, see the following articles: 35 | 36 | * [Formatter Plugin Overview](../formatter/) 37 | * [Writing Formatter Plugins](../plugin-development/api-plugin-formatter.md) 38 | * [Format Section](../configuration/format-section.md) 39 | 40 | ## Methods 41 | 42 | ### `formatter_create(usage: "", type: nil, conf: nil, default_type: nil)` 43 | 44 | * `usage`: unique name required for multiple formatters 45 | * `type`: formatter type 46 | * `conf`: formatter plugin configuration 47 | * `default_type`: default formatter type 48 | 49 | **Examples**: 50 | 51 | ```ruby 52 | def configure(conf) 53 | super 54 | # Create formatter plugin instance using section in `fluent.conf` during configure phase 55 | @formatter = formatter_create 56 | end 57 | 58 | def format(tag, time, record) 59 | @formatter.format(tag, time, record) 60 | end 61 | ``` 62 | 63 | JSON formatter example: 64 | 65 | ```ruby 66 | # Create JSON formatter 67 | def configure(conf) 68 | super 69 | @json_formatter = formatter_create(usage: 'formatter_in_example_json', type: 'json') 70 | end 71 | 72 | def format(tag, time, record) 73 | @json_formatter.format(tag, time, record) 74 | end 75 | ``` 76 | 77 | MsgPack formatter example: 78 | 79 | ```ruby 80 | # Create MsgPack formatter 81 | def configure(conf) 82 | super 83 | @msgpack_formatter = formatter_create(usage: 'formatter_in_example_msgpack', type: 'msgpack') 84 | end 85 | 86 | def format(tag, time, record) 87 | @msgpack_formatter.format(tag, time, record) 88 | end 89 | ``` 90 | 91 | ## Plugins using `formatter` 92 | 93 | * [`filter_stdout`](../filter/stdout.md) 94 | * [`out_stdout`](../output/stdout.md) 95 | * [`out_exec`](../output/exec.md) 96 | * [`out_exec_filter`](../output/exec_filter.md) 97 | * [`out_file`](../output/file.md) 98 | * [`out_s3`](../output/s3.md) 99 | 100 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 101 | 102 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-inject.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Inject 2 | 3 | The `inject` plugin helper injects values into events according to the configuration. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/filter' 9 | 10 | module Fluent::Plugin 11 | class ExampleFilter < Filter 12 | Fluent::Plugin.register_filter('example', self) 13 | 14 | # 1. Load inject helper 15 | helpers :inject 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def filter(tag, time, record) 20 | # 2. Inject values into `record` 21 | new_record = inject_values_to_record(tag, time, record) 22 | # edit new_record ... 23 | new_record 24 | end 25 | end 26 | end 27 | ``` 28 | 29 | For more details, see [Inject section](../configuration/inject-section.md). 30 | 31 | ## Methods 32 | 33 | ### `inject_values_to_record(tag, time, record)` 34 | 35 | This method injects values to the given record and returns the new record. 36 | 37 | * `tag`: the tag of the event 38 | * `time`: event timestamp 39 | * `record`: event record 40 | 41 | Example: 42 | 43 | ```ruby 44 | def filter(tag, time, record) 45 | new_record = inject_values_to_record(tag, time, record) 46 | # edit new_record ... 47 | new_record 48 | end 49 | ``` 50 | 51 | ### `inject_values_to_event_stream(tag, es)` 52 | 53 | This method injects values into the given event stream and returns the new event stream. 54 | 55 | * `tag`: the tag of the event 56 | * `es`: event stream 57 | 58 | Example: 59 | 60 | ```ruby 61 | def process(tag, es) 62 | new_es = inject_values_to_event_stream(tag, es) 63 | # do something using new_es 64 | end 65 | ``` 66 | 67 | ## Plugins using `inject` 68 | 69 | * [`filter_stdout`](../filter/stdout.md) 70 | * [`out_exec`](../output/exec.md) 71 | * [`out_exec_filter`](../output/exec_filter.md) 72 | * [`out_file`](../output/file.md) 73 | * [`out_stdout`](../output/stdout.md) 74 | 75 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 76 | 77 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-metrics.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Metrics 2 | 3 | The `metrics` plugin helper manages the metrics values in plugins. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load metrics helper 15 | helpers :metrics 16 | 17 | def configure(conf) 18 | super 19 | 20 | # 2. Create parser plugin instance 21 | @metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "example", help_text: "Example metrics") 22 | end 23 | 24 | def start 25 | super 26 | 27 | # 3. Increase metrics value 28 | @metrics.inc 29 | 30 | end 31 | 32 | def statistics 33 | stats = super 34 | 35 | # 4. Retrieve metrics value 36 | stats = { 37 | 'input' => stats["input"].merge({ 'example' => @metrics.get }) 38 | } 39 | stats 40 | end 41 | end 42 | ``` 43 | 44 | For more details, see the following articles: 45 | 46 | * [Metrics Plugins Overview](../metrics/) 47 | 48 | ## Methods 49 | 50 | ### `metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)` 51 | 52 | This method creates a metrics instance. 53 | 54 | * `namespace`: The namespace for the metrics. 55 | * `subsystem`: The names that represent specific functions or components. 56 | * `name`: The metrics name. 57 | * `help_text`: The description for metrics. 58 | * `labels`: The key/value pair for metrics labels. 59 | * `prefer_gauge`: Use gauge instead of counter for the metrics if `true`. 60 | 61 | Since 1.19.0, `metrics_create` method generates a getter method with the specified `name` on the calling instance. 62 | 63 | ## Plugins using `metrics` 64 | 65 | * [`buffer`](../buffer/) 66 | * [`filter`](../filter/) 67 | * [`input`](../input/) 68 | * [`output`](../output/) 69 | 70 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 71 | 72 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-parser.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Parser 2 | 3 | The `parser` plugin helper manages the lifecycle of the parser plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load parser helper 15 | helpers :parser 16 | 17 | # Omit `shutdown` and other plugin APIs 18 | 19 | def configure(conf) 20 | super 21 | 22 | # 2. Create parser plugin instance 23 | @parser = parser_create 24 | end 25 | 26 | def start 27 | super 28 | 29 | # Use parser helper in combination usually with other plugin helpers 30 | timer_execute(:example_timer, 10) do 31 | read_raw_data do |text| 32 | # 3. Call `@parser.parse(text)` to parse raw data 33 | @parser.parse(text) do |time, record| 34 | router.emit(tag, time, record) 35 | end 36 | end 37 | end 38 | end 39 | end 40 | end 41 | ``` 42 | 43 | For more details, see the following articles: 44 | 45 | * [Parser Plugin Overview](../parser/) 46 | * [Writing Parser Plugins](../plugin-development/api-plugin-parser.md) 47 | * [Parser Plugin](../configuration/parse-section.md) 48 | 49 | ## Methods 50 | 51 | ### `parser_create(usage: "", type: nil, conf: nil, default_type: nil)` 52 | 53 | This method creates a parser plugin instance with the given parameters. 54 | 55 | * `usage`: unique name required for multiple parsers 56 | * `type`: parser type 57 | * `conf`: parser plugin configuration 58 | * `default_type`: default parser type 59 | 60 | **Examples** 61 | 62 | ```ruby 63 | # Create parser plugin instance using section in fluent.conf during configure phase 64 | @parser = parser_create 65 | @parser.parse(text) do |time, record| 66 | # ... 67 | end 68 | 69 | # Create JSON parser 70 | @json_parser = parser_create(usage: 'parser_in_example_json', type: 'json') 71 | @json_parser.parse(json) do |time, record| 72 | # ... 73 | end 74 | 75 | # Create MessagePack parser 76 | @msgpack_parser = parser_create(usage: 'parser_in_example_msgpack', type: 'msgpack') 77 | @msgpack_parser.parse(msgpack_binary) do |time, record| 78 | # ... 79 | end 80 | ``` 81 | 82 | ## Plugins using `parser` 83 | 84 | * [`filter_parser`](../filter/parser.md) 85 | * [`in_exec`](../input/exec.md) 86 | * [`in_http`](../input/http.md) 87 | * [`in_syslog`](../input/syslog.md) 88 | * [`in_tail`](../input/tail.md) 89 | * [`in_tcp`](../input/tcp.md) 90 | * [`in_udp`](../input/udp.md) 91 | * [`out_exec_filter`](../output/exec_filter.md) 92 | 93 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 94 | 95 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-record_accessor.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Record Accessor 2 | 3 | The `record_accessor` plugin helper provides unified access to the event record. It uses `jsonpath` like syntax for the target field. With this helper, you can easily access/delete a nested field in the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/filter' 9 | 10 | module Fluent::Plugin 11 | class ExampleFilter < Filter 12 | Fluent::Plugin.register_filter('example', self) 13 | 14 | # 1. Load record_accessor helper 15 | helpers :record_accessor 16 | 17 | def configure(conf) 18 | super 19 | 20 | # 2. Call `record_accessor_create` to create object 21 | @accessor = record_accessor_create('$.user.name') 22 | end 23 | 24 | # Omit `super`, `shutdown` and other plugin APIs 25 | 26 | def filter(tag, time, record) 27 | # 3. Call `call` method to get value 28 | value = @accessor.call(record) # With `$.user.name`, access to record["user"]["name"] 29 | # ... 30 | end 31 | end 32 | end 33 | ``` 34 | 35 | ## Syntax 36 | 37 | * dot notation: `$.` is the starting parameter. Chain fields with dots `.`. 38 | 39 | For example: 40 | 41 | `$.event.level` for `record["event"]["level"]` 42 | 43 | `$.key1[0].key2` for `record["key1"][0]["key2"]` 44 | 45 | * bracket notation: `$[` starting parameter. Chain fields with `[]`. 46 | 47 | Useful for special characters, `.`, whitespace, etc. 48 | 49 | `$['dot.key'][0]['space key']` for `record["dot.key"][0]["space key"]` 50 | 51 | If you set non `$.` or `$[` starting value, e.g. `key log`, it is the same as `record["log"]`. So, using `record_accessor` does not break the existing plugin behavior. 52 | 53 | ## Methods 54 | 55 | ### `record_accessor_create(param)` 56 | 57 | This method returns the accessor object of the event record. 58 | 59 | The `param` is a `String`. 60 | 61 | See the "Syntax" section for more details. 62 | 63 | ```ruby 64 | record_accessor_create("log") 65 | record_accessor_create("$.key1.key2") 66 | record_accessor_create("$['key1'][0]['key2']") 67 | ``` 68 | 69 | After creating an object, call `call`/`delete`/`set` method with the record object. 70 | 71 | ```ruby 72 | accessor.call(record) # get record field 73 | accessor.delete(record) # delete record field 74 | accessor.set(record, value) # set new value to record field 75 | ``` 76 | 77 | NOTE: `set` method is supported since v1.10.3 78 | 79 | ## Plugins using `record_accessor` 80 | 81 | * [`filter_grep`](../filter/grep.md) 82 | * [`filter_parser`](../filter/parser.md) 83 | * [`filter_record_transformer`](../filter/record_transformer.md) 84 | 85 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 86 | 87 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-thread.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Thread 2 | 3 | The `thread` plugin helper manages threads and these threads are integrated with plugins. No need manual run or shutdown in the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load thread helper 15 | helpers :thread 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Create and run thread with unique name. 23 | thread_create(:example_thread_run, &method(:run)) 24 | end 25 | 26 | def run 27 | # ... 28 | end 29 | end 30 | end 31 | ``` 32 | 33 | The launched thread is managed by the plugin helper. No need of thread shutdown code in plugin's `shutdown` method. It shutdowns the launched threads automatically. 34 | 35 | ## Methods 36 | 37 | ### `thread_create(title)` 38 | 39 | This method creates a thread and runs it with the given routine. 40 | 41 | `title` must be unique. 42 | 43 | ```ruby 44 | # Pass block directly 45 | thread_create(:example_plugin_main) { 46 | # ... 47 | } 48 | 49 | # Pass method object with existing method 50 | thread_create(:foo_plugin_body, &method(:run)) 51 | def run 52 | # ... 53 | end 54 | ``` 55 | 56 | ### `thread_current_running?` 57 | 58 | Checks whether the current thread is running or not. This method is available in the running block. 59 | 60 | ```ruby 61 | thread_create(:example_plugin_main) { 62 | while thread_current_running? 63 | # ... 64 | end 65 | } 66 | ``` 67 | 68 | ## Plugins using `thread` 69 | 70 | * [`out_forward`](../output/forward.md) 71 | * [`in_monitor_agent`](../input/monitor_agent.md) 72 | * [`in_sample`](../input/sample.md) 73 | 74 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 75 | 76 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-timer.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Timer 2 | 3 | The `timer` plugin helper manages event timers. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load timer helper 15 | helpers :timer 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Execute timer with unique name and second unit interval 23 | timer_execute(:example_timer, 10) { 24 | # ... 25 | } 26 | end 27 | end 28 | end 29 | ``` 30 | 31 | The launched timer is managed by the plugin helper. No need of timer shutdown code in plugin's `shutdown` method. The plugin shutdowns the launched timers automatically. 32 | 33 | ## Methods 34 | 35 | ### `timer_execute(title, interval, repeat: true, &block)` 36 | 37 | This method executes the timer with the given parameters and routine. 38 | 39 | * `title`: unique symbol value 40 | * `interval`: second unit `integer`/`float` value. 41 | * `repeat`: `true`/`false` \(default: `true`\). If `false`, timer is one-shot. 42 | 43 | Code examples: 44 | 45 | ```ruby 46 | # Pass block directly. block is executed in 10 second interval. 47 | timer_execute(:example_timer, 10) { 48 | # ... 49 | } 50 | 51 | # Pass block with existing method. block is executed in 5 second and one-shot. 52 | timer_execute(:example_timer_run, 5s, repeat: false, &method(:run)) 53 | def run 54 | # ... 55 | end 56 | ``` 57 | 58 | ## Plugins using `timer` 59 | 60 | * [`out_forward`](../output/forward.md) 61 | * [`in_monitor_agent`](../input/monitor_agent.md) 62 | * [`in_sample`](../input/sample.md) 63 | 64 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 65 | 66 | -------------------------------------------------------------------------------- /plugins/input/dummy.md: -------------------------------------------------------------------------------- 1 | # `dummy` Input Plugin 2 | 3 | `in_dummy` has been renamed `in_sample` since Fluentd v1.11.12. 4 | See [`sample` input plugin](/input/sample.md) article for more details. 5 | 6 | You can keep to use following configuration in v1: 7 | 8 | ```text 9 | 10 | @type dummy 11 | dummy {"hello":"world"} 12 | 13 | ``` 14 | 15 | Fluentd v2 will remove this plugin name. 16 | 17 | 18 | ------------------------------------------------------------------------ 19 | 20 | If this article is incorrect or outdated, or omits critical information, please 21 | [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). 22 | [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native 23 | Computing Foundation (CNCF)](https://cncf.io/). All components are available 24 | under the Apache 2 License. 25 | -------------------------------------------------------------------------------- /quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | Let's get started with **Fluentd**! **Fluentd** is a fully free and fully open-source log collector that instantly enables you to have a '**Log Everything**' architecture with [125+ types of systems](https://www.fluentd.org/plugins). 4 | 5 |  6 | 7 | Fluentd treats logs as JSON, a popular machine-readable format. It is written primarily in C with a thin-Ruby wrapper that gives users flexibility. 8 | 9 | Fluentd's scalability has been proven in the field: its largest user currently collects logs from **50,000+ servers**. 10 | 11 | ## Step 1: Installing Fluentd 12 | 13 | You can install Fluentd as a package, Ruby gem, or from source. 14 | 15 | Please see the following article for details. 16 | 17 | * [Installation](../installation/) 18 | 19 | ## Step 2: Use Cases 20 | 21 | The following articles cover the typical use cases of Fluentd: 22 | 23 | * Use Cases 24 | * [Data Search like Splunk](../how-to-guides/free-alternative-to-splunk-by-fluentd.md) 25 | * [Data Filtering and Alerting](../how-to-guides/splunk-like-grep-and-alert-email.md) 26 | * [Data Analytics with Treasure Data](../how-to-guides/http-to-td.md) 27 | * [Data Collection to MongoDB](../how-to-guides/apache-to-mongodb.md) 28 | * [Data Collection to HDFS](../how-to-guides/http-to-hdfs.md) 29 | * [Data Archiving to Amazon S3](../how-to-guides/apache-to-s3.md) 30 | * Basic Configuration 31 | * [Config File](../configuration/config-file.md) 32 | * Application Logs 33 | * [Ruby](../language-bindings/ruby.md), [Java](../language-bindings/java.md), [Python](../language-bindings/python.md), [PHP](../language-bindings/php.md), 34 | 35 | [Perl](../language-bindings/perl.md), [Node.js](../language-bindings/nodejs.md), [Scala](../language-bindings/scala.md) 36 | * Happy Users :\) 37 | * [Users](https://www.fluentd.org/testimonials) 38 | 39 | ## Step 3: Learn More 40 | 41 | The following articles will provide detailed information about Fluentd: 42 | 43 | * [Architecture Overview](https://www.fluentd.org/architecture) 44 | * [Lifecycle of a Fluentd Event](life-of-a-fluentd-event.md) 45 | * Plugin Overview 46 | * [Input Plugins](../input/) 47 | * [Output Plugins](../output/) 48 | * [Buffer Plugins](../buffer/) 49 | * [Filter Plugins](../filter/) 50 | * [Parser Plugins](../parser/) 51 | * [Formatter Plugins](../formatter/) 52 | * [High Availability Configuration](../deployment/high-availability.md) 53 | * [FAQ](faq.md) 54 | 55 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 56 | 57 | -------------------------------------------------------------------------------- /quickstart/logo.md: -------------------------------------------------------------------------------- 1 | # Logo 2 | 3 | Feel free to use these logos on your slides, blog posts, etc. 4 | 5 | ## Square 6 | 7 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.png) 8 | 9 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.svg) 10 | 11 | ## Horizontal 12 | 13 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.png) 14 | 15 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.svg) 16 | 17 | ## Icon 18 | 19 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.png) 20 | 21 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.svg) 22 | 23 | ## Links 24 | 25 | * [Fluentd.ai](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd.ai) 26 | * [Complete List of Logos](https://github.com/fluent/fluentd-docs-gitbook/tree/1.0/images/logo) 27 | 28 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 29 | 30 | -------------------------------------------------------------------------------- /quickstart/support.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## Community Forum \(GitHub Discussions\) 4 | 5 | Got a question? Have some cool Fluentd-related projects? The GitHub discussion is the place to be: 6 | 7 | * [Fluentd Discussions](https://github.com/fluent/fluentd/discussions) 8 | 9 | ## Slack 10 | 11 | Join our Slack channel and talk with Fluentd developers and users directly, now. 12 | 13 | * [Slack](https://slack.fluentd.org/) 14 | 15 | ## Enterprise Support 16 | 17 | If you need support for running in production from enterprise companies you can find providers at the following link 18 | 19 | * [Enterprise Support providers](https://www.fluentd.org/enterprise_services) 20 | 21 | ## Community Resources 22 | 23 | Here are other community resources: 24 | 25 | * [StackOverflow](https://stackoverflow.com/questions/tagged/fluentd?sort=newest) 26 | * [Twitter @fluentd](https://www.twitter.com/fluentd) 27 | * [Facebook Page](https://www.facebook.com/pages/Fluentd-Log-Everything-in-JSON/196064987183037) 28 | 29 | ## Change Logs 30 | 31 | If you are interested in what's new in each version, please refer to the following links: 32 | 33 | * [Fluentd's ChangeLog](https://github.com/fluent/fluentd/blob/master/CHANGELOG.md). 34 | * [fluent-package's ChangeLog](https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md) (Successor of td-agent package) 35 | * [td-agent's ChangeLog](https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md) (End of Life in Dec, 2023) 36 | 37 | {% hint style='danger' %} 38 | The series of td-agent had already reached End of Life (EOL). td-agent should not be newly installed because no support, no new release and no security updates anymore. 39 | Use fluent-package instead. See [Upgrade to fluent-package v5](https://www.fluentd.org/blog/upgrade-td-agent-v4-to-v5) for migration. 40 | {% endhint %} 41 | 42 | ## Development 43 | 44 | If you find bugs, please report to the GitHub Issue. 45 | 46 | * [Bug/Feature Tracker](https://github.com/fluent/fluentd/issues) 47 | 48 | The source code is available at GitHub: 49 | 50 | * [Fluentd](https://github.com/fluent/fluentd/) 51 | 52 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 53 | 54 | -------------------------------------------------------------------------------- /service_discovery/README.md: -------------------------------------------------------------------------------- 1 | # Service Discovery Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](../input/) 6 | * [Parser](../parser/) 7 | * [Filter](../filter/) 8 | * [Output](../output/) 9 | * [Formatter](../formatter/) 10 | * [Storage](../storage/) 11 | * [Service Discovery](./) 12 | * [Buffer](../buffer/) 13 | * [Metrics](../metrics/) 14 | 15 | This article gives an overview of the Service Discovery Plugin. 16 | 17 | ## Overview 18 | 19 | Some plugins support `` \(e.g. [`out_forward`](../output/forward.md)\). Sometimes, the service discovery for an output plugin does not meet one's needs. Fluentd has a pluggable system called Service Discovery that lets the user extend and reuse custom output service discovery. 20 | 21 | ## How To Use 22 | 23 | Here is a simple example to update target by reading file \(`/etc/fluentd/sd.yaml`\) with `out_forward` and `service_discovery`: 24 | 25 | ```text 26 | 27 | @type forward 28 | 29 | 30 | @type file 31 | path "/etc/fluentd/sd.yaml" 32 | 33 | 34 | ``` 35 | 36 | ## List of Built-in Service Discovery 37 | 38 | * [`static`](static.md) 39 | * [`file`](file.md) 40 | * [`srv`](srv.md) 41 | 42 | ## List of Core Output Plugins with Service Discovery support 43 | 44 | * [`out_forward`](../output/forward.md) 45 | 46 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 47 | 48 | -------------------------------------------------------------------------------- /service_discovery/file.md: -------------------------------------------------------------------------------- 1 | # file 2 | 3 | The `file` service discovery plugin updates the targets by reading the local file. YAML and JSON are the allowed file formats. 4 | 5 | ## Example Configuration 6 | 7 | Here is an example with `out_forward` updating targets by sending data: 8 | 9 | ```text 10 | 11 | @type forward 12 | 13 | 14 | @type file 15 | path "/etc/fluentd/sd.yaml" 16 | 17 |
[^ ]*) (?[^ ]*)(?: "(?[^\"]*)" "(?[^\"]*)"(?:\s+(?[^ ]+))?)?$/ 15 | time_format %d/%b/%Y:%H:%M:%S %z 16 | ``` 17 | 18 | `remote`, `user`, `method`, `path`, `code`, `size`, `referer`, `agent` and `http_x_forwarded_for` are included in the event record. `time` is used for the event time. 19 | 20 | ## Example 21 | 22 | This incoming event: 23 | 24 | ```text 25 | 127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" - 26 | ``` 27 | 28 | is parsed as: 29 | 30 | ```text 31 | time: 32 | 1362020400 (28/Feb/2013:12:00:00 +0900) 33 | 34 | record: 35 | { 36 | "remote" : "127.0.0.1", 37 | "host" : "192.168.0.1", 38 | "user" : "-", 39 | "method" : "GET", 40 | "path" : "/", 41 | "code" : "200", 42 | "size" : "777", 43 | "referer" : "-", 44 | "agent" : "Opera/12.0", 45 | "http_x_forwarded_for": "-" 46 | } 47 | ``` 48 | 49 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 50 | 51 | -------------------------------------------------------------------------------- /parser/none.md: -------------------------------------------------------------------------------- 1 | # none 2 | 3 | The `none` parser plugin parses the line as-is with the single field. This format is to defer the parsing/structuring of the data. 4 | 5 | This parser is often used in conjunction with [`single_value`](../formatter/single_value.md) format in output plugin. 6 | 7 | ## Parameters 8 | 9 | See [Parse Section Configurations](../configuration/parse-section.md). 10 | 11 | ### `message_key` 12 | 13 | | type | default | version | 14 | | :--- | :--- | :--- | 15 | | string | message | 0.14.0 | 16 | 17 | Specifies the field name to contain logs. 18 | 19 | ## Example 20 | 21 | This incoming event: 22 | 23 | ```text 24 | Hello world. I am a line of log! 25 | ``` 26 | 27 | is parsed as: 28 | 29 | ```text 30 | time: 31 | 1362020400 (current time) 32 | 33 | record: 34 | {"message":"Hello world. I am a line of log!"} 35 | ``` 36 | 37 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 38 | 39 | -------------------------------------------------------------------------------- /parser/tsv.md: -------------------------------------------------------------------------------- 1 | # tsv 2 | 3 | The `tsv` parser plugin parses the TSV format. 4 | 5 | ## Parameters 6 | 7 | See [Parse Section Configurations](../configuration/parse-section.md). 8 | 9 | ### `keys` 10 | 11 | | type | default | version | 12 | | :--- | :--- | :--- | 13 | | array of string | required parameter | 0.14.9 | 14 | 15 | The array of names for fields on each line. 16 | 17 | ### `delimiter` 18 | 19 | | type | default | version | 20 | | :--- | :--- | :--- | 21 | | string | `\t` | 0.14.0 | 22 | 23 | The delimiter \(character or string\) separating TSV values. 24 | 25 | ## Example 26 | 27 | With this configuration: 28 | 29 | ```text 30 | 31 | @type tsv 32 | keys time,host,req_id,user 33 | time_key time 34 | 35 | ``` 36 | 37 | This incoming event: 38 | 39 | ```text 40 | 2013/02/28 12:00:00\t192.168.0.1\t111\t- 41 | ``` 42 | 43 | is parsed as: 44 | 45 | ```text 46 | time: 47 | 1362020400 (2013/02/28/ 12:00:00) 48 | 49 | record: 50 | { 51 | "host" : "192.168.0.1", 52 | "req_id" : "111", 53 | "user" : "-" 54 | } 55 | ``` 56 | 57 | If you set `null_value_pattern '-'` in the configuration, `user` field becomes `nil` instead of `"-"`. 58 | 59 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 60 | 61 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-buffer.md: -------------------------------------------------------------------------------- 1 | # How to Write Buffer Plugin 2 | 3 | TODO: Write 4 | 5 | ## Methods 6 | 7 | ### evacuate_chunk 8 | 9 | You can override this method to add feature to evacuate chunks before clearing the queue when reaching the retry limit. 10 | See [Buffer - Handling Successive Failures](../buffer/README.md#handling-successive-failures) for details. 11 | 12 | ```ruby 13 | def evacuate_chunk(chunk) 14 | unless chunk.is_a?(Fluent::Plugin::Buffer::FileChunk) 15 | raise ArgumentError, "The chunk must be FileChunk, but it was #{chunk.class}." 16 | end 17 | 18 | backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id) 19 | FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir) 20 | 21 | FileUtils.copy([chunk.path, chunk.meta_path], backup_dir) 22 | log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id) 23 | rescue => e 24 | log.error "unexpected error while evacuating chunk files.", error: e 25 | end 26 | ``` 27 | 28 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 29 | 30 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-service_discovery.md: -------------------------------------------------------------------------------- 1 | # How to Write Service Discovery Plugin 2 | 3 | TODO: Write 4 | 5 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 6 | 7 | -------------------------------------------------------------------------------- /plugin-development/api-plugin-storage.md: -------------------------------------------------------------------------------- 1 | # How to Write Storage Plugin 2 | 3 | TODO: Write 4 | 5 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 6 | 7 | -------------------------------------------------------------------------------- /plugin-helper-overview/README.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper API 2 | 3 | Fluentd provides plugin helpers to encapsulate and make commonly implemented features available such as timer, threading, formatting, parsing, ensuring configuration syntax's backward compatibility, etc. 4 | 5 | ## How To Use 6 | 7 | To use plugin helpers, call `helpers(*snake_case_symbols)` method to make them available. 8 | 9 | Example: 10 | 11 | ```ruby 12 | helpers :timer, :storage, :compat_parameters 13 | ``` 14 | 15 | It will include `Timer`, `Storage`. and `CompatParameters` plugin helpers. 16 | 17 | ## Built-in Plugin Helpers 18 | 19 | * [`child_process`](api-plugin-helper-child_process.md) 20 | * [`compat_parameters`](api-plugin-helper-compat_parameters.md) 21 | * [`event_emitter`](api-plugin-helper-event_emitter.md) 22 | * [`event_loop`](api-plugin-helper-event_loop.md) 23 | * [`extract`](api-plugin-helper-extract.md) 24 | * [`formatter`](api-plugin-helper-formatter.md) 25 | * [`inject`](api-plugin-helper-inject.md) 26 | * [`metrics`](api-plugin-helper-metrics.md) 27 | * [`parser`](api-plugin-helper-parser.md) 28 | * [`record_accessor`](api-plugin-helper-record_accessor.md) 29 | * [`server`](api-plugin-helper-server.md) 30 | * [`socket`](api-plugin-helper-socket.md) 31 | * [`storage`](api-plugin-helper-storage.md) 32 | * [`thread`](api-plugin-helper-thread.md) 33 | * [`timer`](api-plugin-helper-timer.md) 34 | * [`http_server`](api-plugin-helper-http_server.md) 35 | 36 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 37 | 38 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-child_process.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Child Process 2 | 3 | The `child_process` helper manages the child processes' lifecycle. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load child_process helper 15 | helpers :child_process 16 | 17 | # Omit configure, shutdown and other plugin API 18 | 19 | def start 20 | super 21 | # 2. Execute child process with unique name 22 | child_process_execute(:exec_external_command, 'external_command', immediate: true, mode: [:read]) do 23 | # ... 24 | end 25 | end 26 | end 27 | end 28 | ``` 29 | 30 | The launched child process is managed by the plugin helper which automatically `shutdown`s it. 31 | 32 | ## Methods 33 | 34 | ### `child_process_execute(title, sub_process_name, arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false, mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil, internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil, wait_timeout: nil, on_exit_callback: nil, &block)` 35 | 36 | This method executes `child_process` with the given parameters and routine. 37 | 38 | * `title`: unique symbol value 39 | * `sub_process_name`: sub process name value 40 | * `interval`: Second unit `integer`/`float` value. 41 | * `immediate`: `true`/`false`. Default is `false`. 42 | * `parallel`: `true`/`false`. Default is `false`. 43 | * `mode`: \[`:read`, `:write`\]. Default is \[`:read`, `:write`\]. 44 | * `stderr`: Connect stderr or not. Default is `:discard`. 45 | * `env`: Environment valuables. Default is {}. 46 | * `unsetenv`: `true`/`false`. Default is `false` 47 | * `chdir`: Working directory. Default is `nil`. 48 | * `internal_encoding`: Internal character encoding. Default is `utf-8`. 49 | * `external_encoding`: External character encoding. Default is `ascii-8bit`' 50 | * `scrub`: `true`/`false`. Default is `true`. 51 | * `replace_string`: Replace the invalid code point with the specified character. 52 | 53 | Default is `nil`. 54 | 55 | * `wait_timeout`: Set timeout seconds. Default is `nil`. 56 | * `on_exit_callback`: Set callback function. Default is `nil` 57 | 58 | Code examples: 59 | 60 | ```ruby 61 | # Pass block directly. block is executed with 10 second interval. 62 | child_process_execute(:exec_awesome_command, @command, interval: 10, mode: [:read]) do |io| 63 | # ... 64 | end 65 | 66 | # Pass block with existing method. block is executed once. 67 | child_process_execute(:exec_awesome_command, @command, immediate: true, mode: [:read], &method(:run)) 68 | 69 | def run(io) 70 | # ... 71 | end 72 | ``` 73 | 74 | ## Plugins using `child_process` 75 | 76 | * [`in_exec`](../input/exec.md) 77 | * [`out_exec`](../output/exec.md) 78 | * [`out_exec_filter`](../output/exec_filter.md) 79 | 80 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 81 | 82 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-event_emitter.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Event Emitter 2 | 3 | The `event_emitter` plugin helper introduces the `router` method to the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class RelabelOutput < Output 12 | Fluent::Plugin.register_output('relabel', self) 13 | helpers :event_emitter 14 | 15 | def multi_workers_ready? 16 | true 17 | end 18 | 19 | def process(tag, es) 20 | router.emit_stream(tag, es) 21 | end 22 | end 23 | end 24 | ``` 25 | 26 | ## Methods 27 | 28 | ### `router` 29 | 30 | This method returns a `Fluent::EventRouter` instance. 31 | 32 | Code example: 33 | 34 | ```ruby 35 | # emit event 36 | router.emit(tag, time, record) 37 | 38 | # emit event stream 39 | router.emit_stream(tag, es) 40 | 41 | # emit error event 42 | begin 43 | # do something 44 | rescue => error 45 | # Route event to @ERROR label or 46 | # log error message when @ERROR label is not defined in configuration 47 | router.emit_error_event(tag, time, record, error) 48 | end 49 | ``` 50 | 51 | ## Plugins using `event_emitter` 52 | 53 | * All input plugins 54 | * All filter plugins 55 | * [`out_exec`](../output/exec.md) 56 | * [`out_relabel`](../output/relabel.md) 57 | * [`out_copy`](../output/copy.md) 58 | * [`out_roundrobin`](../output/roundrobin.md) 59 | 60 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 61 | 62 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-event_loop.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Event Loop 2 | 3 | The `event_loop` plugin helper manages event loops. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load `event_loop_helper` 15 | helpers :event_loop 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Attach watcher 23 | watcher = Coolio::TCPServer.new(...) 24 | event_loop_attach(watcher) 25 | end 26 | end 27 | end 28 | ``` 29 | 30 | The attached watcher is managed by the plugin. No need of watcher detach code in plugin's `shutdown`. The plugin shutdowns the attached watchers automatically. 31 | 32 | ## Methods 33 | 34 | ### `event_loop_attach(watcher)` 35 | 36 | This method attaches watcher to the event loop. 37 | 38 | * `watcher`: `Coolio::Watcher` instances 39 | 40 | ## Plugins using `event_loop` 41 | 42 | * [`in_http`](../input/http.md) 43 | * [`in_tail`](../input/tail.md) 44 | 45 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 46 | 47 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-extract.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Extract 2 | 3 | The `extract` plugin helper extracts `tag` or `time` from the event `record` according to the configuration. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput > Output 12 | Fluent::Plugin.register_output('Example') 13 | 14 | def process(tag, es) 15 | es.each do |time, record| 16 | new_tag = extract_tag_from_record(record) 17 | new_time = extract_time_from_record(record) 18 | end 19 | # ... 20 | end 21 | end 22 | ``` 23 | 24 | For more details, see [Extract section](../configuration/extract-section.md). 25 | 26 | ## Methods 27 | 28 | ### `extract_tag_from_record(record)` 29 | 30 | This method extracts `tag` from the given record. 31 | 32 | * `record`: event record 33 | 34 | Example: 35 | 36 | ```ruby 37 | new_tag = extract_tag_from_record(record) 38 | ``` 39 | 40 | ### `extract_time_from_record(record)` 41 | 42 | This method extracts `time` from the given record. 43 | 44 | * `record`: event record 45 | 46 | Example: 47 | 48 | ```ruby 49 | new_time = extract_time_from_record(record) 50 | ``` 51 | 52 | ## Plugins using `extract` 53 | 54 | * [`in_exec`](../input/exec.md) 55 | * [`in_tcp`](../input/tcp.md) 56 | * [`in_udp`](../input/udp.md) 57 | * [`out_exec_filter`](../output/exec_filter.md) 58 | 59 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 60 | 61 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-formatter.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Formatter 2 | 3 | The `formatter` plugin helper manages the lifecycle of the formatter plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load formatter helper 15 | helpers :formatter 16 | 17 | # Omit `shutdown` and other plugin APIs 18 | 19 | def configure(conf) 20 | super 21 | 22 | # 2. Call `formatter_create` to create object 23 | @formatter = formatter_create(usage: 'awesome_formatter') 24 | end 25 | 26 | def format(tag, time, record) 27 | # 3. Call `format` method to format `record` 28 | @formatter.format(tag, time, record) 29 | end 30 | end 31 | end 32 | ``` 33 | 34 | For more details, see the following articles: 35 | 36 | * [Formatter Plugin Overview](../formatter/) 37 | * [Writing Formatter Plugins](../plugin-development/api-plugin-formatter.md) 38 | * [Format Section](../configuration/format-section.md) 39 | 40 | ## Methods 41 | 42 | ### `formatter_create(usage: "", type: nil, conf: nil, default_type: nil)` 43 | 44 | * `usage`: unique name required for multiple formatters 45 | * `type`: formatter type 46 | * `conf`: formatter plugin configuration 47 | * `default_type`: default formatter type 48 | 49 | **Examples**: 50 | 51 | ```ruby 52 | def configure(conf) 53 | super 54 | # Create formatter plugin instance using section in `fluent.conf` during configure phase 55 | @formatter = formatter_create 56 | end 57 | 58 | def format(tag, time, record) 59 | @formatter.format(tag, time, record) 60 | end 61 | ``` 62 | 63 | JSON formatter example: 64 | 65 | ```ruby 66 | # Create JSON formatter 67 | def configure(conf) 68 | super 69 | @json_formatter = formatter_create(usage: 'formatter_in_example_json', type: 'json') 70 | end 71 | 72 | def format(tag, time, record) 73 | @json_formatter.format(tag, time, record) 74 | end 75 | ``` 76 | 77 | MsgPack formatter example: 78 | 79 | ```ruby 80 | # Create MsgPack formatter 81 | def configure(conf) 82 | super 83 | @msgpack_formatter = formatter_create(usage: 'formatter_in_example_msgpack', type: 'msgpack') 84 | end 85 | 86 | def format(tag, time, record) 87 | @msgpack_formatter.format(tag, time, record) 88 | end 89 | ``` 90 | 91 | ## Plugins using `formatter` 92 | 93 | * [`filter_stdout`](../filter/stdout.md) 94 | * [`out_stdout`](../output/stdout.md) 95 | * [`out_exec`](../output/exec.md) 96 | * [`out_exec_filter`](../output/exec_filter.md) 97 | * [`out_file`](../output/file.md) 98 | * [`out_s3`](../output/s3.md) 99 | 100 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 101 | 102 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-inject.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Inject 2 | 3 | The `inject` plugin helper injects values into events according to the configuration. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/filter' 9 | 10 | module Fluent::Plugin 11 | class ExampleFilter < Filter 12 | Fluent::Plugin.register_filter('example', self) 13 | 14 | # 1. Load inject helper 15 | helpers :inject 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def filter(tag, time, record) 20 | # 2. Inject values into `record` 21 | new_record = inject_values_to_record(tag, time, record) 22 | # edit new_record ... 23 | new_record 24 | end 25 | end 26 | end 27 | ``` 28 | 29 | For more details, see [Inject section](../configuration/inject-section.md). 30 | 31 | ## Methods 32 | 33 | ### `inject_values_to_record(tag, time, record)` 34 | 35 | This method injects values to the given record and returns the new record. 36 | 37 | * `tag`: the tag of the event 38 | * `time`: event timestamp 39 | * `record`: event record 40 | 41 | Example: 42 | 43 | ```ruby 44 | def filter(tag, time, record) 45 | new_record = inject_values_to_record(tag, time, record) 46 | # edit new_record ... 47 | new_record 48 | end 49 | ``` 50 | 51 | ### `inject_values_to_event_stream(tag, es)` 52 | 53 | This method injects values into the given event stream and returns the new event stream. 54 | 55 | * `tag`: the tag of the event 56 | * `es`: event stream 57 | 58 | Example: 59 | 60 | ```ruby 61 | def process(tag, es) 62 | new_es = inject_values_to_event_stream(tag, es) 63 | # do something using new_es 64 | end 65 | ``` 66 | 67 | ## Plugins using `inject` 68 | 69 | * [`filter_stdout`](../filter/stdout.md) 70 | * [`out_exec`](../output/exec.md) 71 | * [`out_exec_filter`](../output/exec_filter.md) 72 | * [`out_file`](../output/file.md) 73 | * [`out_stdout`](../output/stdout.md) 74 | 75 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 76 | 77 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-metrics.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Metrics 2 | 3 | The `metrics` plugin helper manages the metrics values in plugins. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load metrics helper 15 | helpers :metrics 16 | 17 | def configure(conf) 18 | super 19 | 20 | # 2. Create parser plugin instance 21 | @metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "example", help_text: "Example metrics") 22 | end 23 | 24 | def start 25 | super 26 | 27 | # 3. Increase metrics value 28 | @metrics.inc 29 | 30 | end 31 | 32 | def statistics 33 | stats = super 34 | 35 | # 4. Retrieve metrics value 36 | stats = { 37 | 'input' => stats["input"].merge({ 'example' => @metrics.get }) 38 | } 39 | stats 40 | end 41 | end 42 | ``` 43 | 44 | For more details, see the following articles: 45 | 46 | * [Metrics Plugins Overview](../metrics/) 47 | 48 | ## Methods 49 | 50 | ### `metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)` 51 | 52 | This method creates a metrics instance. 53 | 54 | * `namespace`: The namespace for the metrics. 55 | * `subsystem`: The names that represent specific functions or components. 56 | * `name`: The metrics name. 57 | * `help_text`: The description for metrics. 58 | * `labels`: The key/value pair for metrics labels. 59 | * `prefer_gauge`: Use gauge instead of counter for the metrics if `true`. 60 | 61 | Since 1.19.0, `metrics_create` method generates a getter method with the specified `name` on the calling instance. 62 | 63 | ## Plugins using `metrics` 64 | 65 | * [`buffer`](../buffer/) 66 | * [`filter`](../filter/) 67 | * [`input`](../input/) 68 | * [`output`](../output/) 69 | 70 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 71 | 72 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-parser.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Parser 2 | 3 | The `parser` plugin helper manages the lifecycle of the parser plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load parser helper 15 | helpers :parser 16 | 17 | # Omit `shutdown` and other plugin APIs 18 | 19 | def configure(conf) 20 | super 21 | 22 | # 2. Create parser plugin instance 23 | @parser = parser_create 24 | end 25 | 26 | def start 27 | super 28 | 29 | # Use parser helper in combination usually with other plugin helpers 30 | timer_execute(:example_timer, 10) do 31 | read_raw_data do |text| 32 | # 3. Call `@parser.parse(text)` to parse raw data 33 | @parser.parse(text) do |time, record| 34 | router.emit(tag, time, record) 35 | end 36 | end 37 | end 38 | end 39 | end 40 | end 41 | ``` 42 | 43 | For more details, see the following articles: 44 | 45 | * [Parser Plugin Overview](../parser/) 46 | * [Writing Parser Plugins](../plugin-development/api-plugin-parser.md) 47 | * [Parser Plugin](../configuration/parse-section.md) 48 | 49 | ## Methods 50 | 51 | ### `parser_create(usage: "", type: nil, conf: nil, default_type: nil)` 52 | 53 | This method creates a parser plugin instance with the given parameters. 54 | 55 | * `usage`: unique name required for multiple parsers 56 | * `type`: parser type 57 | * `conf`: parser plugin configuration 58 | * `default_type`: default parser type 59 | 60 | **Examples** 61 | 62 | ```ruby 63 | # Create parser plugin instance using section in fluent.conf during configure phase 64 | @parser = parser_create 65 | @parser.parse(text) do |time, record| 66 | # ... 67 | end 68 | 69 | # Create JSON parser 70 | @json_parser = parser_create(usage: 'parser_in_example_json', type: 'json') 71 | @json_parser.parse(json) do |time, record| 72 | # ... 73 | end 74 | 75 | # Create MessagePack parser 76 | @msgpack_parser = parser_create(usage: 'parser_in_example_msgpack', type: 'msgpack') 77 | @msgpack_parser.parse(msgpack_binary) do |time, record| 78 | # ... 79 | end 80 | ``` 81 | 82 | ## Plugins using `parser` 83 | 84 | * [`filter_parser`](../filter/parser.md) 85 | * [`in_exec`](../input/exec.md) 86 | * [`in_http`](../input/http.md) 87 | * [`in_syslog`](../input/syslog.md) 88 | * [`in_tail`](../input/tail.md) 89 | * [`in_tcp`](../input/tcp.md) 90 | * [`in_udp`](../input/udp.md) 91 | * [`out_exec_filter`](../output/exec_filter.md) 92 | 93 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 94 | 95 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-record_accessor.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Record Accessor 2 | 3 | The `record_accessor` plugin helper provides unified access to the event record. It uses `jsonpath` like syntax for the target field. With this helper, you can easily access/delete a nested field in the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/filter' 9 | 10 | module Fluent::Plugin 11 | class ExampleFilter < Filter 12 | Fluent::Plugin.register_filter('example', self) 13 | 14 | # 1. Load record_accessor helper 15 | helpers :record_accessor 16 | 17 | def configure(conf) 18 | super 19 | 20 | # 2. Call `record_accessor_create` to create object 21 | @accessor = record_accessor_create('$.user.name') 22 | end 23 | 24 | # Omit `super`, `shutdown` and other plugin APIs 25 | 26 | def filter(tag, time, record) 27 | # 3. Call `call` method to get value 28 | value = @accessor.call(record) # With `$.user.name`, access to record["user"]["name"] 29 | # ... 30 | end 31 | end 32 | end 33 | ``` 34 | 35 | ## Syntax 36 | 37 | * dot notation: `$.` is the starting parameter. Chain fields with dots `.`. 38 | 39 | For example: 40 | 41 | `$.event.level` for `record["event"]["level"]` 42 | 43 | `$.key1[0].key2` for `record["key1"][0]["key2"]` 44 | 45 | * bracket notation: `$[` starting parameter. Chain fields with `[]`. 46 | 47 | Useful for special characters, `.`, whitespace, etc. 48 | 49 | `$['dot.key'][0]['space key']` for `record["dot.key"][0]["space key"]` 50 | 51 | If you set non `$.` or `$[` starting value, e.g. `key log`, it is the same as `record["log"]`. So, using `record_accessor` does not break the existing plugin behavior. 52 | 53 | ## Methods 54 | 55 | ### `record_accessor_create(param)` 56 | 57 | This method returns the accessor object of the event record. 58 | 59 | The `param` is a `String`. 60 | 61 | See the "Syntax" section for more details. 62 | 63 | ```ruby 64 | record_accessor_create("log") 65 | record_accessor_create("$.key1.key2") 66 | record_accessor_create("$['key1'][0]['key2']") 67 | ``` 68 | 69 | After creating an object, call `call`/`delete`/`set` method with the record object. 70 | 71 | ```ruby 72 | accessor.call(record) # get record field 73 | accessor.delete(record) # delete record field 74 | accessor.set(record, value) # set new value to record field 75 | ``` 76 | 77 | NOTE: `set` method is supported since v1.10.3 78 | 79 | ## Plugins using `record_accessor` 80 | 81 | * [`filter_grep`](../filter/grep.md) 82 | * [`filter_parser`](../filter/parser.md) 83 | * [`filter_record_transformer`](../filter/record_transformer.md) 84 | 85 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 86 | 87 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-thread.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Thread 2 | 3 | The `thread` plugin helper manages threads and these threads are integrated with plugins. No need manual run or shutdown in the plugin. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/output' 9 | 10 | module Fluent::Plugin 11 | class ExampleOutput < Output 12 | Fluent::Plugin.register_output('example', self) 13 | 14 | # 1. Load thread helper 15 | helpers :thread 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Create and run thread with unique name. 23 | thread_create(:example_thread_run, &method(:run)) 24 | end 25 | 26 | def run 27 | # ... 28 | end 29 | end 30 | end 31 | ``` 32 | 33 | The launched thread is managed by the plugin helper. No need of thread shutdown code in plugin's `shutdown` method. It shutdowns the launched threads automatically. 34 | 35 | ## Methods 36 | 37 | ### `thread_create(title)` 38 | 39 | This method creates a thread and runs it with the given routine. 40 | 41 | `title` must be unique. 42 | 43 | ```ruby 44 | # Pass block directly 45 | thread_create(:example_plugin_main) { 46 | # ... 47 | } 48 | 49 | # Pass method object with existing method 50 | thread_create(:foo_plugin_body, &method(:run)) 51 | def run 52 | # ... 53 | end 54 | ``` 55 | 56 | ### `thread_current_running?` 57 | 58 | Checks whether the current thread is running or not. This method is available in the running block. 59 | 60 | ```ruby 61 | thread_create(:example_plugin_main) { 62 | while thread_current_running? 63 | # ... 64 | end 65 | } 66 | ``` 67 | 68 | ## Plugins using `thread` 69 | 70 | * [`out_forward`](../output/forward.md) 71 | * [`in_monitor_agent`](../input/monitor_agent.md) 72 | * [`in_sample`](../input/sample.md) 73 | 74 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 75 | 76 | -------------------------------------------------------------------------------- /plugin-helper-overview/api-plugin-helper-timer.md: -------------------------------------------------------------------------------- 1 | # Plugin Helper: Timer 2 | 3 | The `timer` plugin helper manages event timers. 4 | 5 | Here is an example: 6 | 7 | ```ruby 8 | require 'fluent/plugin/input' 9 | 10 | module Fluent::Plugin 11 | class ExampleInput < Input 12 | Fluent::Plugin.register_input('example', self) 13 | 14 | # 1. Load timer helper 15 | helpers :timer 16 | 17 | # Omit `configure`, `shutdown` and other plugin APIs 18 | 19 | def start 20 | super 21 | 22 | # 2. Execute timer with unique name and second unit interval 23 | timer_execute(:example_timer, 10) { 24 | # ... 25 | } 26 | end 27 | end 28 | end 29 | ``` 30 | 31 | The launched timer is managed by the plugin helper. No need of timer shutdown code in plugin's `shutdown` method. The plugin shutdowns the launched timers automatically. 32 | 33 | ## Methods 34 | 35 | ### `timer_execute(title, interval, repeat: true, &block)` 36 | 37 | This method executes the timer with the given parameters and routine. 38 | 39 | * `title`: unique symbol value 40 | * `interval`: second unit `integer`/`float` value. 41 | * `repeat`: `true`/`false` \(default: `true`\). If `false`, timer is one-shot. 42 | 43 | Code examples: 44 | 45 | ```ruby 46 | # Pass block directly. block is executed in 10 second interval. 47 | timer_execute(:example_timer, 10) { 48 | # ... 49 | } 50 | 51 | # Pass block with existing method. block is executed in 5 second and one-shot. 52 | timer_execute(:example_timer_run, 5s, repeat: false, &method(:run)) 53 | def run 54 | # ... 55 | end 56 | ``` 57 | 58 | ## Plugins using `timer` 59 | 60 | * [`out_forward`](../output/forward.md) 61 | * [`in_monitor_agent`](../input/monitor_agent.md) 62 | * [`in_sample`](../input/sample.md) 63 | 64 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 65 | 66 | -------------------------------------------------------------------------------- /plugins/input/dummy.md: -------------------------------------------------------------------------------- 1 | # `dummy` Input Plugin 2 | 3 | `in_dummy` has been renamed `in_sample` since Fluentd v1.11.12. 4 | See [`sample` input plugin](/input/sample.md) article for more details. 5 | 6 | You can keep to use following configuration in v1: 7 | 8 | ```text 9 | 10 | @type dummy 11 | dummy {"hello":"world"} 12 | 13 | ``` 14 | 15 | Fluentd v2 will remove this plugin name. 16 | 17 | 18 | ------------------------------------------------------------------------ 19 | 20 | If this article is incorrect or outdated, or omits critical information, please 21 | [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). 22 | [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native 23 | Computing Foundation (CNCF)](https://cncf.io/). All components are available 24 | under the Apache 2 License. 25 | -------------------------------------------------------------------------------- /quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | Let's get started with **Fluentd**! **Fluentd** is a fully free and fully open-source log collector that instantly enables you to have a '**Log Everything**' architecture with [125+ types of systems](https://www.fluentd.org/plugins). 4 | 5 |  6 | 7 | Fluentd treats logs as JSON, a popular machine-readable format. It is written primarily in C with a thin-Ruby wrapper that gives users flexibility. 8 | 9 | Fluentd's scalability has been proven in the field: its largest user currently collects logs from **50,000+ servers**. 10 | 11 | ## Step 1: Installing Fluentd 12 | 13 | You can install Fluentd as a package, Ruby gem, or from source. 14 | 15 | Please see the following article for details. 16 | 17 | * [Installation](../installation/) 18 | 19 | ## Step 2: Use Cases 20 | 21 | The following articles cover the typical use cases of Fluentd: 22 | 23 | * Use Cases 24 | * [Data Search like Splunk](../how-to-guides/free-alternative-to-splunk-by-fluentd.md) 25 | * [Data Filtering and Alerting](../how-to-guides/splunk-like-grep-and-alert-email.md) 26 | * [Data Analytics with Treasure Data](../how-to-guides/http-to-td.md) 27 | * [Data Collection to MongoDB](../how-to-guides/apache-to-mongodb.md) 28 | * [Data Collection to HDFS](../how-to-guides/http-to-hdfs.md) 29 | * [Data Archiving to Amazon S3](../how-to-guides/apache-to-s3.md) 30 | * Basic Configuration 31 | * [Config File](../configuration/config-file.md) 32 | * Application Logs 33 | * [Ruby](../language-bindings/ruby.md), [Java](../language-bindings/java.md), [Python](../language-bindings/python.md), [PHP](../language-bindings/php.md), 34 | 35 | [Perl](../language-bindings/perl.md), [Node.js](../language-bindings/nodejs.md), [Scala](../language-bindings/scala.md) 36 | * Happy Users :\) 37 | * [Users](https://www.fluentd.org/testimonials) 38 | 39 | ## Step 3: Learn More 40 | 41 | The following articles will provide detailed information about Fluentd: 42 | 43 | * [Architecture Overview](https://www.fluentd.org/architecture) 44 | * [Lifecycle of a Fluentd Event](life-of-a-fluentd-event.md) 45 | * Plugin Overview 46 | * [Input Plugins](../input/) 47 | * [Output Plugins](../output/) 48 | * [Buffer Plugins](../buffer/) 49 | * [Filter Plugins](../filter/) 50 | * [Parser Plugins](../parser/) 51 | * [Formatter Plugins](../formatter/) 52 | * [High Availability Configuration](../deployment/high-availability.md) 53 | * [FAQ](faq.md) 54 | 55 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 56 | 57 | -------------------------------------------------------------------------------- /quickstart/logo.md: -------------------------------------------------------------------------------- 1 | # Logo 2 | 3 | Feel free to use these logos on your slides, blog posts, etc. 4 | 5 | ## Square 6 | 7 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.png) 8 | 9 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_square.svg) 10 | 11 | ## Horizontal 12 | 13 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.png) 14 | 15 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_horizontal.svg) 16 | 17 | ## Icon 18 | 19 | [](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.png) 20 | 21 | [\[PNG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.png) [\[SVG\]](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd_icon.svg) 22 | 23 | ## Links 24 | 25 | * [Fluentd.ai](https://github.com/fluent/fluentd-docs-gitbook/tree/53020426cdcfcb5a5f722031838ee1cb95b5a7a2/images/logo/Fluentd.ai) 26 | * [Complete List of Logos](https://github.com/fluent/fluentd-docs-gitbook/tree/1.0/images/logo) 27 | 28 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 29 | 30 | -------------------------------------------------------------------------------- /quickstart/support.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## Community Forum \(GitHub Discussions\) 4 | 5 | Got a question? Have some cool Fluentd-related projects? The GitHub discussion is the place to be: 6 | 7 | * [Fluentd Discussions](https://github.com/fluent/fluentd/discussions) 8 | 9 | ## Slack 10 | 11 | Join our Slack channel and talk with Fluentd developers and users directly, now. 12 | 13 | * [Slack](https://slack.fluentd.org/) 14 | 15 | ## Enterprise Support 16 | 17 | If you need support for running in production from enterprise companies you can find providers at the following link 18 | 19 | * [Enterprise Support providers](https://www.fluentd.org/enterprise_services) 20 | 21 | ## Community Resources 22 | 23 | Here are other community resources: 24 | 25 | * [StackOverflow](https://stackoverflow.com/questions/tagged/fluentd?sort=newest) 26 | * [Twitter @fluentd](https://www.twitter.com/fluentd) 27 | * [Facebook Page](https://www.facebook.com/pages/Fluentd-Log-Everything-in-JSON/196064987183037) 28 | 29 | ## Change Logs 30 | 31 | If you are interested in what's new in each version, please refer to the following links: 32 | 33 | * [Fluentd's ChangeLog](https://github.com/fluent/fluentd/blob/master/CHANGELOG.md). 34 | * [fluent-package's ChangeLog](https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md) (Successor of td-agent package) 35 | * [td-agent's ChangeLog](https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md) (End of Life in Dec, 2023) 36 | 37 | {% hint style='danger' %} 38 | The series of td-agent had already reached End of Life (EOL). td-agent should not be newly installed because no support, no new release and no security updates anymore. 39 | Use fluent-package instead. See [Upgrade to fluent-package v5](https://www.fluentd.org/blog/upgrade-td-agent-v4-to-v5) for migration. 40 | {% endhint %} 41 | 42 | ## Development 43 | 44 | If you find bugs, please report to the GitHub Issue. 45 | 46 | * [Bug/Feature Tracker](https://github.com/fluent/fluentd/issues) 47 | 48 | The source code is available at GitHub: 49 | 50 | * [Fluentd](https://github.com/fluent/fluentd/) 51 | 52 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under [the Apache License 2.0.](https://www.apache.org/licenses/LICENSE-2.0) 53 | 54 | -------------------------------------------------------------------------------- /service_discovery/README.md: -------------------------------------------------------------------------------- 1 | # Service Discovery Plugins 2 | 3 | Fluentd has nine \(9\) types of plugins: 4 | 5 | * [Input](../input/) 6 | * [Parser](../parser/) 7 | * [Filter](../filter/) 8 | * [Output](../output/) 9 | * [Formatter](../formatter/) 10 | * [Storage](../storage/) 11 | * [Service Discovery](./) 12 | * [Buffer](../buffer/) 13 | * [Metrics](../metrics/) 14 | 15 | This article gives an overview of the Service Discovery Plugin. 16 | 17 | ## Overview 18 | 19 | Some plugins support `` \(e.g. [`out_forward`](../output/forward.md)\). Sometimes, the service discovery for an output plugin does not meet one's needs. Fluentd has a pluggable system called Service Discovery that lets the user extend and reuse custom output service discovery. 20 | 21 | ## How To Use 22 | 23 | Here is a simple example to update target by reading file \(`/etc/fluentd/sd.yaml`\) with `out_forward` and `service_discovery`: 24 | 25 | ```text 26 | 27 | @type forward 28 | 29 | 30 | @type file 31 | path "/etc/fluentd/sd.yaml" 32 | 33 | 34 | ``` 35 | 36 | ## List of Built-in Service Discovery 37 | 38 | * [`static`](static.md) 39 | * [`file`](file.md) 40 | * [`srv`](srv.md) 41 | 42 | ## List of Core Output Plugins with Service Discovery support 43 | 44 | * [`out_forward`](../output/forward.md) 45 | 46 | If this article is incorrect or outdated, or omits critical information, please [let us know](https://github.com/fluent/fluentd-docs-gitbook/issues?state=open). [Fluentd](http://www.fluentd.org/) is an open-source project under [Cloud Native Computing Foundation \(CNCF\)](https://cncf.io/). All components are available under the Apache 2 License. 47 | 48 | -------------------------------------------------------------------------------- /service_discovery/file.md: -------------------------------------------------------------------------------- 1 | # file 2 | 3 | The `file` service discovery plugin updates the targets by reading the local file. YAML and JSON are the allowed file formats. 4 | 5 | ## Example Configuration 6 | 7 | Here is an example with `out_forward` updating targets by sending data: 8 | 9 | ```text 10 | 11 | @type forward 12 | 13 | 14 | @type file 15 | path "/etc/fluentd/sd.yaml" 16 | 17 |