├── test ├── plugin │ ├── data │ │ ├── 2010 │ │ │ └── 01 │ │ │ │ ├── 20100102.log │ │ │ │ ├── 20100102-030405.log │ │ │ │ └── 20100102-030406.log │ │ ├── log │ │ │ ├── bar │ │ │ ├── foo │ │ │ │ ├── bar.log │ │ │ │ └── bar2 │ │ │ └── test.log │ │ ├── log_numeric │ │ │ ├── 01.log │ │ │ ├── 02.log │ │ │ ├── 12.log │ │ │ └── 14.log │ │ └── sd_file │ │ │ ├── invalid_config.yml │ │ │ ├── config │ │ │ ├── config.yaml │ │ │ ├── config.yml │ │ │ └── config.json │ ├── test_formatter_msgpack.rb │ ├── test_out_relabel.rb │ ├── test_string_util.rb │ ├── test_owned_by.rb │ ├── test_formatter_hash.rb │ ├── test_buf_memory.rb │ ├── test_out_buffer.rb │ ├── test_parser_apache.rb │ ├── test_in_gc_stat.rb │ ├── test_formatter_single_value.rb │ ├── test_in_debug_agent.rb │ ├── test_parser_apache_error.rb │ └── test_formatter_json.rb ├── plugin_helper │ ├── data │ │ └── cert │ │ │ ├── empty.pem │ │ │ ├── cert.pem │ │ │ ├── cert-with-no-newline.pem │ │ │ ├── cert-with-CRLF.pem │ │ │ ├── with_ca │ │ │ ├── ca-cert.pem │ │ │ ├── ca-cert-pass.pem │ │ │ ├── cert.pem │ │ │ ├── cert-pass.pem │ │ │ ├── ca-cert-key.pem │ │ │ ├── cert-key.pem │ │ │ └── cert-key-pass.pem │ │ │ ├── cert_chains │ │ │ ├── ca-cert.pem │ │ │ ├── cert-key.pem │ │ │ └── ca-cert-key.pem │ │ │ ├── without_ca │ │ │ ├── cert.pem │ │ │ ├── cert-pass.pem │ │ │ └── cert-key.pem │ │ │ └── cert-key.pem │ ├── http_server │ │ ├── test_request.rb │ │ └── test_route.rb │ ├── service_discovery │ │ └── test_round_robin_balancer.rb │ └── test_event_loop.rb ├── scripts │ ├── fluent │ │ └── plugin │ │ │ ├── parser_known.rb │ │ │ ├── formatter1 │ │ │ └── formatter_test1.rb │ │ │ ├── formatter2 │ │ │ └── formatter_test2.rb │ │ │ └── formatter_known.rb │ └── exec_script.rb ├── test_process.rb ├── test_input.rb ├── helpers │ └── process_extenstion.rb ├── config │ ├── assertions.rb │ └── test_plugin_configuration.rb ├── test_file_wrapper.rb ├── test_unique_id.rb ├── counter │ └── test_error.rb ├── test_msgpack_factory.rb └── command │ └── test_ctl.rb ├── templates ├── new_gem │ ├── Gemfile │ ├── test │ │ ├── helper.rb.erb │ │ └── plugin │ │ │ ├── test_filter.rb.erb │ │ │ ├── test_input.rb.erb │ │ │ ├── test_output.rb.erb │ │ │ ├── test_parser.rb.erb │ │ │ ├── test_storage.rb.erb │ │ │ └── test_formatter.rb.erb │ ├── lib │ │ └── fluent │ │ │ └── plugin │ │ │ ├── input.rb.erb │ │ │ ├── output.rb.erb │ │ │ ├── filter.rb.erb │ │ │ ├── formatter.rb.erb │ │ │ ├── parser.rb.erb │ │ │ └── storage.rb.erb │ ├── Rakefile │ ├── README.md.erb │ └── fluent-plugin.gemspec.erb └── plugin_config_formatter │ ├── section.md.erb │ ├── param.md-table.erb │ ├── param.md-compact.erb │ └── param.md.erb ├── docs └── SECURITY_AUDIT.pdf ├── AUTHORS ├── example ├── suppress_config_dump.conf ├── sd.yaml ├── in_tcp.conf ├── in_udp.conf ├── in_forward.conf ├── in_forward_tls.conf ├── out_file.conf ├── in_tail.conf ├── in_http.conf ├── in_syslog.conf ├── out_forward_heartbeat_none.conf ├── out_forward_sd.conf ├── in_forward_shared_key.conf ├── counter.conf ├── in_out_forward.conf ├── filter_stdout.conf ├── out_forward_tls.conf ├── in_sample_blocks.conf ├── out_copy.conf ├── in_forward_workers.conf ├── out_forward_buf_file.conf ├── in_forward_users.conf ├── logevents.conf ├── in_sample_with_compression.conf ├── out_forward.conf ├── copy_roundrobin.conf ├── secondary_file.conf ├── worker_section.conf ├── out_null.conf ├── out_forward_shared_key.conf ├── in_forward_client.conf ├── out_exec_filter.conf ├── multi_filters.conf ├── v1_literal_example.conf └── out_forward_users.conf ├── .github ├── dependabot.yml ├── PULL_REQUEST_TEMPLATE.md ├── ISSUE_TEMPLATE │ ├── config.yml │ └── feature_request.yml ├── ISSUE_TEMPLATE.md ├── workflows │ ├── backport.yml │ ├── test-ruby-head.yml │ ├── rubocop.yml │ ├── benchmark.yml │ └── stale-actions.yml └── DISCUSSION_TEMPLATE │ ├── q-a-japanese.yml │ └── q-a.yml ├── bin ├── fluent-ca-generate ├── fluent-cat ├── fluent-debug ├── fluent-ctl ├── fluent-plugin-config-format ├── fluent-plugin-generate ├── fluent-cap-ctl ├── fluent-binlog-reader ├── fluent-gem └── fluentd ├── code-of-conduct.md ├── ADOPTERS.md ├── .deepsource.toml ├── Gemfile ├── tasks ├── benchmark │ ├── conf │ │ └── in_tail.conf │ └── patch_in_tail.rb ├── backport.rb └── benchmark.rb ├── lib └── fluent │ ├── daemon.rb │ ├── version.rb │ ├── plugin │ ├── in_dummy.rb │ ├── exec_util.rb │ ├── file_util.rb │ ├── socket_util.rb │ ├── string_util.rb │ ├── out_relabel.rb │ ├── formatter_msgpack.rb │ ├── parser_apache_error.rb │ ├── out_forward │ │ └── error.rb │ ├── buf_memory.rb │ ├── formatter_hash.rb │ ├── parser_apache.rb │ ├── owned_by_mixin.rb │ ├── parser_none.rb │ ├── parser_nginx.rb │ ├── formatter_single_value.rb │ ├── out_buffer.rb │ ├── parser_tsv.rb │ ├── formatter_tsv.rb │ ├── formatter_ltsv.rb │ ├── filter_stdout.rb │ └── parser_ltsv.rb │ ├── filter.rb │ ├── input.rb │ ├── counter.rb │ ├── parser.rb │ ├── load.rb │ ├── formatter.rb │ ├── process.rb │ ├── output_chain.rb │ ├── compat │ ├── structured_format_mixin.rb │ ├── record_filter_mixin.rb │ ├── detach_process_mixin.rb │ ├── string_util.rb │ ├── parser_utils.rb │ ├── input.rb │ ├── set_tag_key_mixin.rb │ └── output_chain.rb │ ├── test │ ├── driver │ │ ├── parser.rb │ │ ├── storage.rb │ │ ├── formatter.rb │ │ ├── input.rb │ │ ├── test_event_router.rb │ │ └── multi_output.rb │ └── startup_shutdown.rb │ ├── ext_monitor_require.rb │ ├── error.rb │ ├── output.rb │ ├── config │ ├── error.rb │ ├── yaml_parser │ │ └── fluent_value.rb │ └── yaml_parser.rb │ ├── plugin_helper │ ├── http_server │ │ ├── methods.rb │ │ ├── ssl_context_builder.rb │ │ ├── request.rb │ │ └── router.rb │ ├── service_discovery │ │ └── round_robin_balancer.rb │ └── counter.rb │ ├── mixin.rb │ ├── unique_id.rb │ ├── variable_store.rb │ ├── win32api.rb │ ├── counter │ └── base_socket.rb │ ├── command │ └── bundler_injection.rb │ ├── label.rb │ ├── oj_options.rb │ └── env.rb ├── .gitignore ├── MAINTAINERS.md └── SECURITY.md /test/plugin/data/log/bar: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/log/foo/bar.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/log/foo/bar2: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/log/test.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/2010/01/20100102.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/log_numeric/01.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/log_numeric/02.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/log_numeric/12.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/log_numeric/14.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin_helper/data/cert/empty.pem: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/2010/01/20100102-030405.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/plugin/data/2010/01/20100102-030406.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /templates/new_gem/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /docs/SECURITY_AUDIT.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluentd/HEAD/docs/SECURITY_AUDIT.pdf -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | FURUHASHI Sadayuki 2 | NAKAGAWA Masahiro 3 | -------------------------------------------------------------------------------- /example/suppress_config_dump.conf: -------------------------------------------------------------------------------- 1 | 2 | suppress_config_dump false 3 | 4 | 5 | 6 | @type stdout 7 | 8 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'github-actions' 4 | directory: '/' 5 | schedule: 6 | interval: 'weekly' 7 | -------------------------------------------------------------------------------- /bin/fluent-ca-generate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | $LOAD_PATH.unshift(File.join(__dir__, 'lib')) 4 | require 'fluent/command/ca_generate' 5 | 6 | Fluent::CaGenerate.new.call 7 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | ## Fluentd Community Code of Conduct 2 | 3 | Fluentd follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | -------------------------------------------------------------------------------- /example/sd.yaml: -------------------------------------------------------------------------------- 1 | - 'host': 127.0.0.1 2 | 'port': 24224 3 | 'weight': 1 4 | 'name': server1 5 | - 'host': 127.0.0.1 6 | 'port': 24225 7 | 'weight': 1 8 | 'name': server2 9 | -------------------------------------------------------------------------------- /bin/fluent-cat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | here = File.dirname(__FILE__) 4 | $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib')) 5 | require 'fluent/command/cat' 6 | -------------------------------------------------------------------------------- /test/scripts/fluent/plugin/parser_known.rb: -------------------------------------------------------------------------------- 1 | module Fluent 2 | TextParser.register_template('known_old', /^(?.*)$/) 3 | Plugin.register_parser('known', /^(?.*)$/) 4 | end 5 | -------------------------------------------------------------------------------- /bin/fluent-debug: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | here = File.dirname(__FILE__) 4 | $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib')) 5 | require 'fluent/command/debug' 6 | -------------------------------------------------------------------------------- /test/plugin/data/sd_file/invalid_config.yml: -------------------------------------------------------------------------------- 1 | - 'host': 127.0.0.1 2 | 'weight': 1 3 | 'name': test1 4 | 'standby': false 5 | 'username': user1 6 | 'password': pass1 7 | 'shared_key': key1 8 | -------------------------------------------------------------------------------- /bin/fluent-ctl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | here = File.dirname(__FILE__) 4 | $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib')) 5 | require 'fluent/command/ctl' 6 | 7 | Fluent::Ctl.new.call 8 | -------------------------------------------------------------------------------- /bin/fluent-plugin-config-format: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH.unshift(File.join(__dir__, 'lib')) 3 | require 'fluent/command/plugin_config_formatter' 4 | 5 | FluentPluginConfigFormatter.new.call 6 | -------------------------------------------------------------------------------- /bin/fluent-plugin-generate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.expand_path(File.join(__dir__, '..', 'lib')) 3 | require 'fluent/command/plugin_generator' 4 | 5 | FluentPluginGenerator.new.call 6 | -------------------------------------------------------------------------------- /ADOPTERS.md: -------------------------------------------------------------------------------- 1 | # Fluentd Adopters 2 | 3 | Fluentd is widely used by hundred of companies, please refer to the testimonial section of our project website to learn more about it: 4 | 5 | https://www.fluentd.org/testimonials 6 | -------------------------------------------------------------------------------- /test/scripts/fluent/plugin/formatter1/formatter_test1.rb: -------------------------------------------------------------------------------- 1 | module Fluent 2 | Plugin.register_formatter( 3 | 'test1', 4 | Proc.new { |tag, time, record| 5 | "#{tag}:#{time}:#{record.size}" 6 | }) 7 | end 8 | -------------------------------------------------------------------------------- /test/scripts/fluent/plugin/formatter2/formatter_test2.rb: -------------------------------------------------------------------------------- 1 | module Fluent 2 | Plugin.register_formatter( 3 | 'test2', 4 | Proc.new { |tag, time, record| 5 | "#{tag}:#{time}:#{record.size}" 6 | }) 7 | end 8 | -------------------------------------------------------------------------------- /bin/fluent-cap-ctl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | here = File.dirname(__FILE__) 4 | $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib')) 5 | require 'fluent/command/cap_ctl' 6 | 7 | Fluent::CapCtl.new.call -------------------------------------------------------------------------------- /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | test_patterns = ["test/**/test_*.rb"] 4 | 5 | exclude_patterns = [ 6 | "bin/**", 7 | "docs/**", 8 | "example/**" 9 | ] 10 | 11 | [[analyzers]] 12 | name = "ruby" 13 | enabled = true 14 | -------------------------------------------------------------------------------- /example/in_tcp.conf: -------------------------------------------------------------------------------- 1 | 2 | @type tcp 3 | format none 4 | bind 0.0.0.0 5 | port 5170 6 | delimiter \n 7 | source_host_key "host" 8 | tag test 9 | 10 | 11 | 12 | @type stdout 13 | 14 | -------------------------------------------------------------------------------- /example/in_udp.conf: -------------------------------------------------------------------------------- 1 | 2 | @type udp 3 | format none 4 | bind 0.0.0.0 5 | port 5160 6 | body_size_limit 4KB 7 | source_host_key "host" 8 | tag test 9 | 10 | 11 | 12 | @type stdout 13 | 14 | -------------------------------------------------------------------------------- /bin/fluent-binlog-reader: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | here = File.dirname(__FILE__) 4 | $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib')) 5 | require 'fluent/command/binlog_reader' 6 | 7 | FluentBinlogReader.new.call 8 | -------------------------------------------------------------------------------- /bin/fluent-gem: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | require 'rubygems/gem_runner' 4 | require 'rubygems/exceptions' 5 | begin 6 | Gem::GemRunner.new.run ARGV 7 | rescue Gem::SystemExitException => e 8 | exit e.exit_code 9 | end 10 | -------------------------------------------------------------------------------- /example/in_forward.conf: -------------------------------------------------------------------------------- 1 | 2 | rpc_endpoint 0.0.0.0:24444 3 | 4 | 5 | 6 | @type forward 7 | 8 | 9 | 10 | @type stdout 11 | # 12 | # flush_interval 10s 13 | # 14 | 15 | -------------------------------------------------------------------------------- /templates/new_gem/test/helper.rb.erb: -------------------------------------------------------------------------------- 1 | require "test-unit" 2 | require "fluent/test" 3 | require "fluent/test/driver/<%= type %>" 4 | require "fluent/test/helpers" 5 | 6 | Test::Unit::TestCase.include(Fluent::Test::Helpers) 7 | Test::Unit::TestCase.extend(Fluent::Test::Helpers) 8 | -------------------------------------------------------------------------------- /test/plugin/data/sd_file/config: -------------------------------------------------------------------------------- 1 | - 'host': 127.0.0.1 2 | 'port': 24224 3 | 'weight': 1 4 | 'name': test1 5 | 'standby': false 6 | 'username': user1 7 | 'password': pass1 8 | 'shared_key': key1 9 | - 'host': 127.0.0.1 10 | 'port': 24225 11 | 'weight': 1 12 | -------------------------------------------------------------------------------- /example/in_forward_tls.conf: -------------------------------------------------------------------------------- 1 | 2 | @type forward 3 | port 24224 4 | 5 | insecure true 6 | 7 | 8 | 9 | 10 | @type stdout 11 | # 12 | # flush_interval 10s 13 | # 14 | 15 | -------------------------------------------------------------------------------- /test/plugin/data/sd_file/config.yaml: -------------------------------------------------------------------------------- 1 | - 'host': 127.0.0.1 2 | 'port': 24224 3 | 'weight': 1 4 | 'name': test1 5 | 'standby': false 6 | 'username': user1 7 | 'password': pass1 8 | 'shared_key': key1 9 | - 'host': 127.0.0.1 10 | 'port': 24225 11 | 'weight': 1 12 | -------------------------------------------------------------------------------- /test/plugin/data/sd_file/config.yml: -------------------------------------------------------------------------------- 1 | - 'host': 127.0.0.1 2 | 'port': 24224 3 | 'weight': 1 4 | 'name': test1 5 | 'standby': false 6 | 'username': user1 7 | 'password': pass1 8 | 'shared_key': key1 9 | - 'host': 127.0.0.1 10 | 'port': 24225 11 | 'weight': 1 12 | -------------------------------------------------------------------------------- /example/out_file.conf: -------------------------------------------------------------------------------- 1 | 2 | @type forward 3 | 4 | 5 | 6 | @type file 7 | path /var/log/fluentd/out_file_test 8 | format json 9 | buffer_type memory 10 | # compress gzip 11 | # symlink_path /path/to/symlink 12 | append false 13 | 14 | -------------------------------------------------------------------------------- /example/in_tail.conf: -------------------------------------------------------------------------------- 1 | 2 | @type tail 3 | format none 4 | path /var/log/fluentd_test.log 5 | pos_file /var/log/fluentd_test.pos 6 | tag test 7 | rotate_wait 5 8 | read_from_head true 9 | refresh_interval 60 10 | 11 | 12 | 13 | @type stdout 14 | 15 | -------------------------------------------------------------------------------- /templates/new_gem/lib/fluent/plugin/input.rb.erb: -------------------------------------------------------------------------------- 1 | <%= preamble %> 2 | 3 | require "fluent/plugin/input" 4 | 5 | module Fluent 6 | module Plugin 7 | class <%= class_name %> < Fluent::Plugin::Input 8 | Fluent::Plugin.register_input("<%= plugin_name %>", self) 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /example/in_http.conf: -------------------------------------------------------------------------------- 1 | 2 | @type http 3 | bind 0.0.0.0 4 | port 9880 5 | body_size_limit 32MB 6 | keepalive_timeout 10 7 | # backlog 0 8 | add_http_headers false 9 | 10 | @type json 11 | 12 | 13 | 14 | 15 | @type stdout 16 | 17 | -------------------------------------------------------------------------------- /templates/new_gem/lib/fluent/plugin/output.rb.erb: -------------------------------------------------------------------------------- 1 | <%= preamble %> 2 | 3 | require "fluent/plugin/output" 4 | 5 | module Fluent 6 | module Plugin 7 | class <%= class_name %> < Fluent::Plugin::Output 8 | Fluent::Plugin.register_output("<%= plugin_name %>", self) 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /example/in_syslog.conf: -------------------------------------------------------------------------------- 1 | 2 | @type syslog 3 | bind 0.0.0.0 4 | port 5140 5 | tag test 6 | protocol_type udp 7 | include_source_host false 8 | source_host_key source_host 9 | # format ... 10 | # with_priority true 11 | 12 | 13 | 14 | @type stdout 15 | 16 | -------------------------------------------------------------------------------- /example/out_forward_heartbeat_none.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag test 4 | 5 | 6 | 7 | @type forward 8 | heartbeat_type none 9 | 10 | host 127.0.0.1 11 | port 24224 12 | 13 | 14 | flush_mode immediate 15 | 16 | 17 | -------------------------------------------------------------------------------- /test/scripts/fluent/plugin/formatter_known.rb: -------------------------------------------------------------------------------- 1 | module Fluent 2 | TextFormatter.register_template('known_old', Proc.new { |tag, time, record| 3 | "#{tag}:#{time}:#{record.size}" 4 | }) 5 | Plugin.register_formatter('known', Proc.new { |tag, time, record| 6 | "#{tag}:#{time}:#{record.size}" 7 | }) 8 | end 9 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org/' 2 | 3 | gemspec 4 | 5 | gem 'benchmark' 6 | 7 | local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local") 8 | if File.exist?(local_gemfile) 9 | puts "Loading Gemfile.local ..." if $DEBUG # `ruby -d` or `bundle -v` 10 | instance_eval File.read(local_gemfile) 11 | end 12 | -------------------------------------------------------------------------------- /example/out_forward_sd.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag test 4 | 5 | 6 | 7 | @type forward 8 | 9 | 10 | @type file 11 | path "#{Dir.pwd}/example/sd.yaml" 12 | 13 | 14 | 15 | flush_interval 1 16 | 17 | 18 | -------------------------------------------------------------------------------- /templates/new_gem/Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler" 2 | Bundler::GemHelper.install_tasks 3 | 4 | require "rake/testtask" 5 | 6 | Rake::TestTask.new(:test) do |t| 7 | t.libs.push("lib", "test") 8 | t.test_files = FileList["test/**/test_*.rb"] 9 | t.verbose = true 10 | t.warning = true 11 | end 12 | 13 | task default: [:test] 14 | -------------------------------------------------------------------------------- /example/in_forward_shared_key.conf: -------------------------------------------------------------------------------- 1 | 2 | rpc_endpoint 0.0.0.0:24444 3 | 4 | 5 | 6 | @type forward 7 | 8 | self_hostname input.testing.local 9 | shared_key secure_communication_is_awesome 10 | 11 | 12 | 13 | 14 | @type stdout 15 | 16 | -------------------------------------------------------------------------------- /tasks/benchmark/conf/in_tail.conf: -------------------------------------------------------------------------------- 1 | 2 | @type tail 3 | tag benchmark 4 | path "#{File.expand_path './tmp/benchmark/data.log'}" 5 | read_from_head true 6 | 7 | @type json 8 | 9 | 10 | 11 | 12 | @type file 13 | path "#{File.expand_path './tmp/benchmark/in_tail'}" 14 | 15 | -------------------------------------------------------------------------------- /example/counter.conf: -------------------------------------------------------------------------------- 1 | 2 | 3 | scope server1 4 | bind 127.0.0.1 5 | port 24321 6 | backup_path tmp/back 7 | 8 | 9 | 10 | 11 | @type sample 12 | tag "test.data" 13 | auto_increment_key number 14 | 15 | 16 | 17 | @type stdout 18 | 19 | -------------------------------------------------------------------------------- /example/in_out_forward.conf: -------------------------------------------------------------------------------- 1 | 2 | @type forward 3 | port 24224 4 | 5 | 6 | 7 | @type forward 8 | buffer_type file 9 | buffer_path /tmp/fluentd.forward.buffer 10 | num_threads 10 11 | flush_interval 1s 12 | 13 | host 127.0.0.1 14 | port 24225 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /example/filter_stdout.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag sample 4 | 5 | 6 | 7 | @type stdout 8 | 9 | 10 | 11 | @type stdout 12 | output_type hash 13 | 14 | 15 | 16 | @type stdout 17 | format ltsv 18 | 19 | 20 | 21 | @type null 22 | 23 | -------------------------------------------------------------------------------- /example/out_forward_tls.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag test 4 | 5 | 6 | 7 | @type forward 8 | transport tls 9 | tls_insecure_mode true 10 | 11 | # first server 12 | host 127.0.0.1 13 | port 24224 14 | 15 | 16 | flush_interval 0 17 | 18 | 19 | -------------------------------------------------------------------------------- /templates/new_gem/lib/fluent/plugin/filter.rb.erb: -------------------------------------------------------------------------------- 1 | <%= preamble %> 2 | 3 | require "fluent/plugin/filter" 4 | 5 | module Fluent 6 | module Plugin 7 | class <%= class_name %> < Fluent::Plugin::Filter 8 | Fluent::Plugin.register_filter("<%= plugin_name %>", self) 9 | 10 | def filter(tag, time, record) 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /templates/new_gem/lib/fluent/plugin/formatter.rb.erb: -------------------------------------------------------------------------------- 1 | <%= preamble %> 2 | 3 | require "fluent/plugin/formatter" 4 | 5 | module Fluent 6 | module Plugin 7 | class <%= class_name %> < Fluent::Plugin::Formatter 8 | Fluent::Plugin.register_formatter("<%= name %>", self) 9 | 10 | def format(tag, time, record) 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /templates/new_gem/lib/fluent/plugin/parser.rb.erb: -------------------------------------------------------------------------------- 1 | <%= preamble %> 2 | 3 | require "fluent/plugin/parser" 4 | 5 | module Fluent 6 | module Plugin 7 | class <%= class_name %> < Fluent::Plugin::Parser 8 | Fluent::Plugin.register_parser("<%= plugin_name %>", self) 9 | 10 | def parse(text) 11 | yield time, record 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /example/in_sample_blocks.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag sample 4 | rate 100 5 | sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"} 6 | 7 | 8 | 9 | @type null 10 | never_flush true 11 | 12 | @type memory 13 | overflow_action block 14 | chunk_limit_size 1k 15 | total_limit_size 2k 16 | 17 | 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | **Which issue(s) this PR fixes**: 8 | Fixes # 9 | 10 | **What this PR does / why we need it**: 11 | 12 | **Docs Changes**: 13 | 14 | **Release Note**: 15 | -------------------------------------------------------------------------------- /example/out_copy.conf: -------------------------------------------------------------------------------- 1 | 2 | @type forward 3 | 4 | 5 | 6 | @type copy 7 | deep_copy false 8 | 9 | @type stdout 10 | 11 | 12 | @type file 13 | path /var/log/fluentd/out_file_test 14 | format json 15 | buffer_type memory 16 | # compress gzip 17 | # symlink_path /path/to/symlink 18 | append false 19 | 20 | 21 | -------------------------------------------------------------------------------- /test/plugin/data/sd_file/config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "host": "127.0.0.1", 4 | "port": 24224, 5 | "weight": 1, 6 | "name": "test1", 7 | "standby": false, 8 | "username": "user1", 9 | "password": "pass1", 10 | "shared_key": "key1" 11 | }, 12 | { 13 | "host": "127.0.0.1", 14 | "port": 24225, 15 | "weight": 1 16 | } 17 | ] 18 | -------------------------------------------------------------------------------- /example/in_forward_workers.conf: -------------------------------------------------------------------------------- 1 | 2 | workers 3 3 | root_dir "#{File.join(Dir.pwd, 'test', 'tmp', 'root')}" 4 | 5 | 6 | 7 | @type forward 8 | @id forward_in_1 9 | 10 | 11 | 12 | @type stdout 13 | @id stdout_out_1 14 | 15 | worker_id_key worker_id 16 | 17 | 18 | @type file 19 | flush_interval 1s 20 | 21 | 22 | -------------------------------------------------------------------------------- /templates/new_gem/test/plugin/test_filter.rb.erb: -------------------------------------------------------------------------------- 1 | require "helper" 2 | require "fluent/plugin/<%= plugin_filename %>" 3 | 4 | class <%= class_name %>Test < Test::Unit::TestCase 5 | setup do 6 | Fluent::Test.setup 7 | end 8 | 9 | test "failure" do 10 | flunk 11 | end 12 | 13 | private 14 | 15 | def create_driver(conf) 16 | Fluent::Test::Driver::Filter.new(Fluent::Plugin::<%= class_name %>).configure(conf) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /templates/new_gem/test/plugin/test_input.rb.erb: -------------------------------------------------------------------------------- 1 | require "helper" 2 | require "fluent/plugin/<%= plugin_filename %>" 3 | 4 | class <%= class_name %>Test < Test::Unit::TestCase 5 | setup do 6 | Fluent::Test.setup 7 | end 8 | 9 | test "failure" do 10 | flunk 11 | end 12 | 13 | private 14 | 15 | def create_driver(conf) 16 | Fluent::Test::Driver::Input.new(Fluent::Plugin::<%= class_name %>).configure(conf) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /templates/new_gem/test/plugin/test_output.rb.erb: -------------------------------------------------------------------------------- 1 | require "helper" 2 | require "fluent/plugin/<%= plugin_filename %>" 3 | 4 | class <%= class_name %>Test < Test::Unit::TestCase 5 | setup do 6 | Fluent::Test.setup 7 | end 8 | 9 | test "failure" do 10 | flunk 11 | end 12 | 13 | private 14 | 15 | def create_driver(conf) 16 | Fluent::Test::Driver::Output.new(Fluent::Plugin::<%= class_name %>).configure(conf) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /templates/new_gem/test/plugin/test_parser.rb.erb: -------------------------------------------------------------------------------- 1 | require "helper" 2 | require "fluent/plugin/<%= plugin_filename %>" 3 | 4 | class <%= class_name %>Test < Test::Unit::TestCase 5 | setup do 6 | Fluent::Test.setup 7 | end 8 | 9 | test "failure" do 10 | flunk 11 | end 12 | 13 | private 14 | 15 | def create_driver(conf) 16 | Fluent::Test::Driver::Parser.new(Fluent::Plugin::<%= class_name %>).configure(conf) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /templates/plugin_config_formatter/section.md.erb: -------------------------------------------------------------------------------- 1 | <%- 2 | section_header = "#" * (3 + level) 3 | required_label = required ? "required" : "optional" 4 | multiple_label = multi ? "multiple" : "single" 5 | -%> 6 | <%= section_header %> \<<%= section_name %>\> section (<%= required_label %>) (<%= multiple_label %>) 7 | <%- if alias_name -%> 8 | 9 | Alias: <%= alias_name %> 10 | <%- end -%> 11 | 12 | <%= dump_section_markdown(sub_section, level + 1) %> 13 | -------------------------------------------------------------------------------- /templates/new_gem/test/plugin/test_storage.rb.erb: -------------------------------------------------------------------------------- 1 | require "helper" 2 | require "fluent/plugin/<%= plugin_filename %>" 3 | 4 | class <%= class_name %>Test < Test::Unit::TestCase 5 | setup do 6 | Fluent::Test.setup 7 | end 8 | 9 | test "failure" do 10 | flunk 11 | end 12 | 13 | private 14 | 15 | def create_driver(conf) 16 | Fluent::Test::Driver::Storage.new(Fluent::Plugin::<%= class_name %>).configure(conf) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /templates/new_gem/test/plugin/test_formatter.rb.erb: -------------------------------------------------------------------------------- 1 | require "helper" 2 | require "fluent/plugin/<%= plugin_filename %>" 3 | 4 | class <%= class_name %>Test < Test::Unit::TestCase 5 | setup do 6 | Fluent::Test.setup 7 | end 8 | 9 | test "failure" do 10 | flunk 11 | end 12 | 13 | private 14 | 15 | def create_driver(conf) 16 | Fluent::Test::Driver::Formatter.new(Fluent::Plugin::<%= class_name %>).configure(conf) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/fluent/daemon.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | 4 | here = File.dirname(__FILE__) 5 | $LOAD_PATH << File.expand_path(File.join(here, '..')) 6 | 7 | require 'serverengine' 8 | require 'fluent/supervisor' 9 | 10 | server_module = Fluent.const_get(ARGV[0]) 11 | worker_module = Fluent.const_get(ARGV[1]) 12 | params = JSON.parse(ARGV[2]) 13 | ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) } 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | INSTALL 3 | NEWS 4 | Makefile 5 | Makefile.in 6 | README 7 | ac 8 | aclocal.m4 9 | autom4te.cache 10 | confdefs.h 11 | config.log 12 | config.status 13 | configure 14 | deps/ 15 | fluent-cat 16 | fluent-gem 17 | fluentd 18 | pkg/* 19 | tmp/* 20 | test/tmp/* 21 | test/config/tmp/* 22 | make_dist.sh 23 | Gemfile.local 24 | .ruby-version 25 | *.swp 26 | coverage/* 27 | .vagrant/ 28 | cov-int/ 29 | cov-fluentd.tar.gz 30 | .vscode 31 | .idea/ 32 | .bundle/ 33 | /vendor/ 34 | -------------------------------------------------------------------------------- /example/out_forward_buf_file.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag test 4 | 5 | 6 | 7 | @type monitor_agent 8 | emit_interval 5 9 | 10 | 11 | 12 | @type forward 13 | buffer_path /tmp/fluentd.forward 14 | buffer_type file 15 | flush_interval 5 16 | send_timeout 60 17 | heartbeat_type tcp 18 | heartbeat_interval 1 19 | 20 | host 127.0.0.1 21 | port 24224 22 | 23 | 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Ask a Question 4 | url: https://github.com/fluent/fluentd/discussions 5 | about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions 6 | - name: Feedback a Fluentd Use-Case/Testimonial 7 | url: https://github.com/fluent/fluentd-website/issues/new?template=testimonials.yml 8 | about: Feedback your Fluentd use-case/testimonial, How do you use Fluentd in your service? 9 | -------------------------------------------------------------------------------- /example/in_forward_users.conf: -------------------------------------------------------------------------------- 1 | 2 | rpc_endpoint 0.0.0.0:24444 3 | 4 | 5 | 6 | @type forward 7 | 8 | self_hostname input.testing.local 9 | shared_key secure_communication_is_awesome 10 | user_auth yes 11 | 12 | username user1 13 | password yes_this_is_user1 14 | 15 | 16 | username user2 17 | password yes_this_is_really_user2 18 | 19 | 20 | 21 | 22 | 23 | @type stdout 24 | 25 | -------------------------------------------------------------------------------- /test/test_process.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helper' 2 | require 'fluent/process' 3 | 4 | class ProcessCompatibilityTest < ::Test::Unit::TestCase 5 | test 'DetachProcessMixin is defined' do 6 | assert defined?(::Fluent::DetachProcessMixin) 7 | assert_equal ::Fluent::DetachProcessMixin, ::Fluent::Compat::DetachProcessMixin 8 | end 9 | 10 | test 'DetachMultiProcessMixin is defined' do 11 | assert defined?(::Fluent::DetachMultiProcessMixin) 12 | assert_equal ::Fluent::DetachMultiProcessMixin, ::Fluent::Compat::DetachMultiProcessMixin 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /tasks/benchmark/patch_in_tail.rb: -------------------------------------------------------------------------------- 1 | require 'benchmark' 2 | require 'fluent/plugin/in_tail' 3 | 4 | class Fluent::Plugin::TailInput::TailWatcher::IOHandler 5 | alias_method :original_with_io, :with_io 6 | 7 | def with_io(&block) 8 | @benchmark_measured_in_tail ||= false 9 | # Measure the benchmark only once. 10 | return original_with_io(&block) if @benchmark_measured_in_tail 11 | 12 | Benchmark.bm do |x| 13 | x.report { 14 | original_with_io(&block) 15 | @benchmark_measured_in_tail = true 16 | } 17 | end 18 | 19 | exit 0 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /example/logevents.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | @label @samplelog 4 | tag "data" 5 | sample {"message":"yay"} 6 | 7 | 12 | 26 | -------------------------------------------------------------------------------- /example/in_sample_with_compression.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | @label @main 4 | tag "test.data" 5 | size 2 6 | rate 10 7 | sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"} 8 | auto_increment_key number 9 | 10 | 11 | 24 | -------------------------------------------------------------------------------- /bin/fluentd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | 4 | # In recent CRuby, RubyVM::InstructionSequence is always defined. 5 | # This optimization aims to reduce memory (avg 0.5 megabytes) at startup. 6 | # ref: http://atdot.net/~ko1/activities/2017_fukuoka_rubykaigi_02.pdf 7 | # But other implementation such as truffleruby does not provide such 8 | # class. 9 | if defined?(RubyVM::InstructionSequence) 10 | RubyVM::InstructionSequence.compile_option = {trace_instruction: false} 11 | end 12 | 13 | here = File.dirname(__FILE__) 14 | $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib')) 15 | require 'fluent/command/fluentd' 16 | -------------------------------------------------------------------------------- /test/scripts/exec_script.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | require 'msgpack' 3 | 4 | def gen_tsv(time) 5 | "#{time}\ttag1\tok" 6 | end 7 | 8 | def gen_json(time) 9 | {'tag' => 'tag1', 'time' => time, 'k1' => 'ok'}.to_json 10 | end 11 | 12 | def gen_msgpack(time) 13 | {'tagger' => 'tag1', 'datetime' => time, 'k1' => 'ok'}.to_msgpack 14 | end 15 | 16 | def gen_raw_string(time) 17 | "#{time} hello" 18 | end 19 | 20 | time = ARGV.first 21 | time = Integer(time) rescue time 22 | 23 | case ARGV.last.to_i 24 | when 0 25 | puts gen_tsv(time) 26 | when 1 27 | puts gen_json(time) 28 | when 2 29 | print gen_msgpack(time) 30 | when 3 31 | print gen_raw_string(time) 32 | end 33 | -------------------------------------------------------------------------------- /templates/plugin_config_formatter/param.md-table.erb: -------------------------------------------------------------------------------- 1 | <%- 2 | type = config[:type] 3 | required_label = config[:required] ? "required" : "optional" 4 | default = config[:default] 5 | alias_name = config[:alias] 6 | deprecated = config[:deprecated] 7 | obsoleted = config[:obsoleted] 8 | description = config[:desc] 9 | -%> 10 | |<%= name %>|<%= type %> (<%= required_label %>)|<%= description %><%- if type == :enum -%> (<%= config[:list].map{|x| "`#{x}`"}.join(", ") %>)<%- end -%><%- if alias_name -%>
Alias: <%= alias_name %><%- end -%><%- if deprecated -%>
Deprecated: <%= deprecated %><%- end -%><%- if obsoleted -%>
Obsoleted: <%= :obsoleted %><%- end -%>|<%- if default -%>`<%= default %>`<%- end -%>| 11 | -------------------------------------------------------------------------------- /example/out_forward.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag test 4 | 5 | 6 | 7 | @type forward 8 | 9 | 10 | # first server 11 | host 127.0.0.1 12 | port 24224 13 | 14 | # 15 | # # second server 16 | # host localhost 17 | # port 24225 18 | # 19 | # 20 | # # second server 21 | # host localhost 22 | # port 24226 23 | # standby 24 | # 25 | 26 | flush_interval 0 27 | send_timeout 60 28 | heartbeat_type udp 29 | heartbeat_interval 1 30 | recover_wait 10 31 | hard_timeout 60 32 | expire_dns_cache nil 33 | phi_threshold 16 34 | phi_failure_detector true 35 | 36 | -------------------------------------------------------------------------------- /test/plugin/test_formatter_msgpack.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helper' 2 | require 'fluent/test/driver/formatter' 3 | require 'fluent/plugin/formatter_msgpack' 4 | 5 | class MessagePackFormatterTest < ::Test::Unit::TestCase 6 | def setup 7 | @time = event_time 8 | end 9 | 10 | def create_driver(conf = "") 11 | Fluent::Test::Driver::Formatter.new(Fluent::Plugin::MessagePackFormatter).configure(conf) 12 | end 13 | 14 | def tag 15 | "tag" 16 | end 17 | 18 | def record 19 | {'message' => 'awesome'} 20 | end 21 | 22 | def test_format 23 | d = create_driver({}) 24 | formatted = d.instance.format(tag, @time, record) 25 | 26 | assert_equal(record.to_msgpack, formatted) 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/fluent/version.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | module Fluent 18 | 19 | VERSION = '1.19.0' 20 | 21 | end 22 | -------------------------------------------------------------------------------- /lib/fluent/plugin/in_dummy.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Remove this file in fluentd v2 18 | require_relative 'in_sample' 19 | -------------------------------------------------------------------------------- /example/copy_roundrobin.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | @label @test 4 | tag test.copy 5 | auto_increment_key id 6 | 7 | 8 | 9 | @type sample 10 | @label @test 11 | tag test.rr 12 | auto_increment_key id 13 | 14 | 15 | 40 | -------------------------------------------------------------------------------- /lib/fluent/filter.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/filter' 18 | 19 | module Fluent 20 | Filter = Fluent::Compat::Filter 21 | end 22 | -------------------------------------------------------------------------------- /lib/fluent/input.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/input' 18 | 19 | module Fluent 20 | Input = Fluent::Compat::Input 21 | end 22 | -------------------------------------------------------------------------------- /example/secondary_file.conf: -------------------------------------------------------------------------------- 1 | 2 | rpc_endpoint 0.0.0.0:24444 3 | 4 | 5 | 6 | @type sample 7 | tag test 8 | 9 | 10 | 11 | @type forward 12 | @label @raw 13 | 14 | 15 | 20 | 21 | 22 | @type forward 23 | 24 | 25 | @type memory 26 | timekey 2s 27 | timekey_wait 1s 28 | flush_mode interval 29 | flush_interval 1s 30 | 31 | 32 | 33 | host 0.0.0.0 34 | port 24224 35 | 36 | 37 | 38 | @type secondary_file 39 | directory log/secondary/ 40 | basename ${tag}_%Y%m%d%L_${message} 41 | 42 | 43 | -------------------------------------------------------------------------------- /templates/new_gem/lib/fluent/plugin/storage.rb.erb: -------------------------------------------------------------------------------- 1 | <%= preamble %> 2 | 3 | require "fluent/plugin/storage" 4 | 5 | module Fluent 6 | module Plugin 7 | class <%= class_name %> < Fluent::Plugin::Storage 8 | Fluent::Plugin.register_storage("<%= plugin_name %>", self) 9 | 10 | def initialize 11 | super 12 | end 13 | 14 | def configure(conf) 15 | super 16 | end 17 | 18 | def load 19 | end 20 | 21 | def save 22 | end 23 | 24 | def get(key) 25 | end 26 | 27 | def fetch(key, defval) 28 | end 29 | 30 | def put(key, value) 31 | end 32 | 33 | def delete(key) 34 | end 35 | 36 | def update(key, &block) 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/fluent/counter.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/counter/client' 18 | require 'fluent/counter/server' 19 | 20 | module Fluent 21 | module Counter 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/fluent/plugin/exec_util.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/exec_util' 18 | 19 | module Fluent 20 | # obsolete 21 | ExecUtil = Fluent::Compat::ExecUtil 22 | end 23 | -------------------------------------------------------------------------------- /lib/fluent/plugin/file_util.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/file_util' 18 | 19 | module Fluent 20 | # obsolete 21 | FileUtil = Fluent::Compat::FileUtil 22 | end 23 | -------------------------------------------------------------------------------- /lib/fluent/plugin/socket_util.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/socket_util' 18 | 19 | module Fluent 20 | # obsolete 21 | SocketUtil = Fluent::Compat::SocketUtil 22 | end 23 | -------------------------------------------------------------------------------- /lib/fluent/plugin/string_util.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/string_util' 18 | 19 | module Fluent 20 | # obsolete 21 | StringUtil = Fluent::Compat::StringUtil 22 | end 23 | -------------------------------------------------------------------------------- /test/plugin_helper/http_server/test_request.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | require 'flexmock/test_unit' 3 | require 'fluent/plugin_helper/http_server/request' 4 | 5 | class HttpHelperRequestTest < Test::Unit::TestCase 6 | def test_request 7 | headers = Protocol::HTTP::Headers.new({ 'Content-Type' => 'text/html', 'Content-Encoding' => 'gzip' }) 8 | req = flexmock('request', path: '/path?foo=42', headers: headers) 9 | 10 | request = Fluent::PluginHelper::HttpServer::Request.new(req) 11 | 12 | assert_equal('/path', request.path) 13 | assert_equal('foo=42', request.query_string) 14 | assert_equal({'foo'=>['42']}, request.query) 15 | assert_equal('text/html', request.headers['content-type']) 16 | assert_equal(['gzip'], request.headers['content-encoding']) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /templates/new_gem/README.md.erb: -------------------------------------------------------------------------------- 1 | # <%= gem_name %> 2 | 3 | [Fluentd](https://fluentd.org/) <%= type %> plugin to do something. 4 | 5 | TODO: write description for you plugin. 6 | 7 | ## Installation 8 | 9 | ### RubyGems 10 | 11 | ``` 12 | $ gem install <%= gem_name %> 13 | ``` 14 | 15 | ### Bundler 16 | 17 | Add following line to your Gemfile: 18 | 19 | ```ruby 20 | gem "<%= gem_name %>" 21 | ``` 22 | 23 | And then execute: 24 | 25 | ``` 26 | $ bundle 27 | ``` 28 | 29 | ## Configuration 30 | 31 | You can generate configuration template: 32 | 33 | ``` 34 | $ fluent-plugin-config-format <%= type %> <%= name %> 35 | ``` 36 | 37 | You can copy and paste generated documents here. 38 | 39 | ## Copyright 40 | 41 | * Copyright(c) <%= Date.today.year %>- <%= user_name %> 42 | * License 43 | * <%= @license.full_name %> 44 | -------------------------------------------------------------------------------- /templates/plugin_config_formatter/param.md-compact.erb: -------------------------------------------------------------------------------- 1 | <%- 2 | type = config[:type] 3 | required_label = config[:required] ? "required" : "optional" 4 | default = config[:default] 5 | alias_name = config[:alias] 6 | deprecated = config[:deprecated] 7 | obsoleted = config[:obsoleted] 8 | description = config[:desc] 9 | -%> 10 | * **<%= name %>** (<%= type %>) (<%= required_label %>): <%= description %> 11 | <%- if type == :enum -%> 12 | * Available values: <%= config[:list].join(", ") %> 13 | <%- end -%> 14 | <%- if default -%> 15 | * Default value: `<%= default %>`. 16 | <%- end -%> 17 | <%- if alias_name -%> 18 | * Alias: <%= alias_name %> 19 | <%- end -%> 20 | <%- if deprecated -%> 21 | * Deprecated: <%= deprecated %> 22 | <%- end -%> 23 | <%- if obsoleted -%> 24 | * Obsoleted: <%= :obsoleted %> 25 | <%- end -%> 26 | -------------------------------------------------------------------------------- /test/plugin/test_out_relabel.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helper' 2 | require 'fluent/test/driver/output' 3 | require 'fluent/plugin/out_relabel' 4 | 5 | class RelabelOutputTest < Test::Unit::TestCase 6 | def setup 7 | Fluent::Test.setup 8 | end 9 | 10 | def default_config 11 | config_element('ROOT', '', {"@type"=>"relabel", "@label"=>"@RELABELED"}) 12 | end 13 | 14 | def create_driver(conf = default_config) 15 | Fluent::Test::Driver::Output.new(Fluent::Plugin::RelabelOutput).configure(conf) 16 | end 17 | 18 | def test_process 19 | d = create_driver 20 | 21 | time = event_time("2011-01-02 13:14:15 UTC") 22 | d.run(default_tag: 'test') do 23 | d.feed(time, {"a"=>1}) 24 | d.feed(time, {"a"=>2}) 25 | end 26 | assert_equal [["test", time, {"a"=>1}], ["test", time, {"a"=>2}]], d.events 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /example/worker_section.conf: -------------------------------------------------------------------------------- 1 | 2 | workers 4 3 | root_dir /path/fluentd/root 4 | 5 | 6 | # top-level sections works on all workers in parallel 7 | @type forward 8 | port 24224 9 | 10 | 11 | # this sections also works on all workers in parallel 12 | @type stdout 13 | 14 | worker_id_key worker_id 15 | 16 | 17 | 18 | # this section works only on first worker process 19 | 20 | @type tail 21 | format none 22 | path /var/log/fluentd_test.log 23 | pos_file /var/log/fluentd_test.pos 24 | tag tail 25 | rotate_wait 5 26 | read_from_head true 27 | refresh_interval 60 28 | 29 | 30 | 31 | @type stdout 32 | 33 | worker_id_key worker_id 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /lib/fluent/parser.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/parser' 18 | 19 | module Fluent 20 | ParserError = Fluent::Compat::Parser::ParserError 21 | Parser = Fluent::Compat::Parser 22 | TextParser = Fluent::Compat::TextParser 23 | end 24 | -------------------------------------------------------------------------------- /lib/fluent/load.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | require 'fcntl' 3 | require 'time' 4 | require 'stringio' 5 | require 'fileutils' 6 | require 'json' 7 | require 'yajl' 8 | require 'uri' 9 | require 'msgpack' 10 | require 'strptime' 11 | begin 12 | require 'sigdump/setup' 13 | rescue 14 | # ignore setup error on Win or similar platform which doesn't support signal 15 | end 16 | require 'cool.io' 17 | 18 | require 'fluent/time' 19 | require 'fluent/env' 20 | require 'fluent/version' 21 | require 'fluent/log' 22 | require 'fluent/config' 23 | require 'fluent/engine' 24 | require 'fluent/rpc' 25 | require 'fluent/mixin' 26 | require 'fluent/plugin' 27 | require 'fluent/parser' 28 | require 'fluent/formatter' 29 | require 'fluent/event' 30 | require 'fluent/input' 31 | require 'fluent/output' 32 | require 'fluent/filter' 33 | require 'fluent/match' 34 | require 'fluent/ext_monitor_require' 35 | -------------------------------------------------------------------------------- /lib/fluent/formatter.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/formatter' 18 | 19 | module Fluent 20 | Formatter = Fluent::Compat::Formatter 21 | TextFormatter = Fluent::Compat::TextFormatter 22 | # deprecate_constant is ruby 2.3 feature 23 | end 24 | -------------------------------------------------------------------------------- /lib/fluent/process.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/detach_process_mixin' 18 | 19 | module Fluent 20 | DetachProcessMixin = Fluent::Compat::DetachProcessMixin 21 | DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin 22 | end 23 | -------------------------------------------------------------------------------- /templates/plugin_config_formatter/param.md.erb: -------------------------------------------------------------------------------- 1 | <%- 2 | type = config[:type] 3 | required_label = config[:required] ? "required" : "optional" 4 | default = config[:default] 5 | alias_name = config[:alias] 6 | deprecated = config[:deprecated] 7 | obsoleted = config[:obsoleted] 8 | description = config[:desc] 9 | param_header = "#" * (3 + level) 10 | -%> 11 | <%= param_header %> <%= name %> (<%= type %>) (<%= required_label %>) 12 | 13 | <%= description %> 14 | <%- if type == :enum -%> 15 | 16 | Available values: <%= config[:list].join(", ") %> 17 | <%- end -%> 18 | <%- if default -%> 19 | 20 | Default value: `<%= default %>`. 21 | <%- end -%> 22 | <%- if alias_name -%> 23 | 24 | Alias: <%= alias_name %> 25 | <%- end -%> 26 | <%- if deprecated -%> 27 | 28 | Deprecated: <%= deprecated %> 29 | <%- end -%> 30 | <%- if obsoleted -%> 31 | 32 | Obsoleted: <%= :obsoleted %> 33 | <%- end -%> 34 | 35 | -------------------------------------------------------------------------------- /example/out_null.conf: -------------------------------------------------------------------------------- 1 | # 2 | # bundle exec bin/fluentd -c example/out_buffered_null.conf 3 | # (+ --emit-error-log-interval 10) 4 | 5 | @type sample 6 | tag sample 7 | rate 500000000 8 | sample [ 9 | {"message": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, 10 | {"message": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, 11 | {"message": "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"} 12 | ] 13 | 14 | 15 | 16 | @type null 17 | 18 | flush_interval 60s 19 | chunk_limit_size 1k 20 | total_limit_size 4k 21 | 22 | 23 | 24 | 32 | 33 | 34 | @type relabel 35 | @label error_log 36 | 37 | -------------------------------------------------------------------------------- /lib/fluent/output_chain.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/compat/output_chain' 18 | 19 | module Fluent 20 | OutputChain = Fluent::Compat::OutputChain 21 | CopyOutputChain = Fluent::Compat::CopyOutputChain 22 | NullOutputChain = Fluent::Compat::NullOutputChain 23 | end 24 | -------------------------------------------------------------------------------- /example/out_forward_shared_key.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | tag test 4 | 5 | 6 | @type sample 7 | tag test2 8 | 9 | 10 | 11 | @type forward 12 | flush_interval 0 13 | 14 | self_hostname output.testing.local 15 | shared_key secure_communication_is_awesome 16 | 17 | 18 | host 127.0.0.1 19 | port 24224 20 | 21 | 22 | 23 | 24 | @type forward 25 | flush_interval 0 26 | 27 | self_hostname output-fail.testing.local 28 | shared_key secure_communication_is_not_awesome 29 | # input plugin shows warning for wrong shared_key 30 | # 2016-08-08 16:27:00 +0900 [warn]: Shared key mismatch address="127.0.0.1" hostname="output-fail.testing.local" 31 | 32 | 33 | host 127.0.0.1 34 | port 24224 35 | 36 | 37 | -------------------------------------------------------------------------------- /example/in_forward_client.conf: -------------------------------------------------------------------------------- 1 | 2 | rpc_endpoint 0.0.0.0:24444 3 | 4 | 5 | 6 | @type forward 7 | port 24224 8 | bind 0.0.0.0 9 | 10 | self_hostname input.testing.local 11 | shared_key secure_communication_is_awesome 12 | user_auth yes 13 | allow_anonymous_source no 14 | 15 | username user1 16 | password yes_this_is_user1 17 | 18 | 19 | username user2 20 | password yes_this_is_really_user2 21 | 22 | 23 | username user3 24 | password noooooo_this_may_not_be_user3 25 | 26 | 27 | # host 127.0.0.1 28 | network 127.0.0.0/24 29 | shared_key using_different_key_makes_us_secure 30 | users user1,user2 31 | 32 | 33 | 34 | 35 | 36 | @type stdout 37 | 38 | -------------------------------------------------------------------------------- /lib/fluent/compat/structured_format_mixin.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | module Fluent 18 | module Compat 19 | module StructuredFormatMixin 20 | def format(tag, time, record) 21 | filter_record(tag, time, record) 22 | format_record(record) 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/plugin/test_string_util.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helper' 2 | require 'fluent/plugin/string_util' 3 | 4 | class StringUtilTest < Test::Unit::TestCase 5 | def setup 6 | @null_value_pattern = Regexp.new("^(-|null|NULL)$") 7 | end 8 | 9 | sub_test_case 'valid string' do 10 | test 'null string' do 11 | assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "null").to_s, "null" 12 | assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "NULL").to_s, "NULL" 13 | assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "-").to_s, "-" 14 | end 15 | 16 | test 'normal string' do 17 | assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "fluentd"), nil 18 | end 19 | end 20 | 21 | sub_test_case 'invalid string' do 22 | test 'normal string' do 23 | assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "\xff"), nil 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/test_input.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helper' 2 | require 'fluent/input' 3 | 4 | class FluentInputTest < ::Test::Unit::TestCase 5 | include Fluent 6 | 7 | def setup 8 | Fluent::Test.setup 9 | end 10 | 11 | def create_driver(conf = '') 12 | Fluent::Test::InputTestDriver.new(Fluent::Input).configure(conf, true) 13 | end 14 | 15 | def test_router 16 | d = create_driver 17 | assert_equal Engine.root_agent.event_router, d.instance.router 18 | 19 | d = nil 20 | assert_nothing_raised { 21 | d = create_driver('@label @known') 22 | } 23 | expected = Engine.root_agent.find_label('@known').event_router 24 | assert_equal expected, d.instance.router 25 | 26 | # TestDriver helps to create a label instance automatically, so directly test here 27 | assert_raise(ArgumentError) { 28 | Fluent::Input.new.configure(Config.parse('@label @unknown', '(test)', '(test_dir)', true)) 29 | } 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /test/plugin_helper/service_discovery/test_round_robin_balancer.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | require 'fluent/plugin_helper/service_discovery/round_robin_balancer' 3 | 4 | class TestRoundRobinBalancer < ::Test::Unit::TestCase 5 | test 'select_service' do 6 | rrb = Fluent::PluginHelper::ServiceDiscovery::RoundRobinBalancer.new 7 | rrb.rebalance([1, 2, 3]) 8 | 9 | rrb.select_service { |n| assert_equal 1, n } 10 | rrb.select_service { |n| assert_equal 2, n } 11 | rrb.select_service { |n| assert_equal 3, n } 12 | rrb.select_service { |n| assert_equal 1, n } 13 | rrb.select_service { |n| assert_equal 2, n } 14 | rrb.select_service { |n| assert_equal 3, n } 15 | rrb.rebalance([1, 2, 3, 4]) 16 | rrb.select_service { |n| assert_equal 1, n } 17 | rrb.select_service { |n| assert_equal 2, n } 18 | rrb.select_service { |n| assert_equal 3, n } 19 | rrb.select_service { |n| assert_equal 4, n } 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /example/out_exec_filter.conf: -------------------------------------------------------------------------------- 1 | 2 | @type sample 3 | @label @exec 4 | tag exec_input 5 | rate 10 6 | auto_increment_key num 7 | sample {"data":"mydata"} 8 | 9 | 10 | 36 | 37 | 42 | 43 | -------------------------------------------------------------------------------- /lib/fluent/test/driver/parser.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/test/driver/base_owned' 18 | 19 | module Fluent 20 | module Test 21 | module Driver 22 | class Parser < BaseOwned 23 | def initialize(klass, **kwargs, &block) 24 | super 25 | @section_name = "parse" 26 | end 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/fluent/test/driver/storage.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/test/driver/base_owned' 18 | 19 | module Fluent 20 | module Test 21 | module Driver 22 | class Storage < BaseOwned 23 | def initialize(klass, **kwargs, &block) 24 | super 25 | @section_name = "storage" 26 | end 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/fluent/test/driver/formatter.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/test/driver/base_owned' 18 | 19 | module Fluent 20 | module Test 21 | module Driver 22 | class Formatter < BaseOwned 23 | def initialize(klass, **kwargs, &block) 24 | super 25 | @section_name = "format" 26 | end 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem. 2 | 3 | **Got a question or problem?** 4 | 5 | RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you. 6 | 7 | If you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd). 8 | Don't use Github issue for asking questions. Here are examples: 9 | 10 | - I installed xxx plugin but it doesn't work. Why? 11 | - Fluentd starts but logs are not sent to xxx. Am I wrong? 12 | - I want to do xxx. How to realize it with plugins? 13 | 14 | We may close such questions to keep clear repository for developers and users. 15 | Github issue is mainly for submitting a bug report or feature request. See below. 16 | 17 | If you can't judge your case is a bug or not, use mailing list or slack first. 18 | -------------------------------------------------------------------------------- /lib/fluent/ext_monitor_require.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # To avoid duplicated requirements, extract this logic as file. 18 | 19 | if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.7.0') 20 | require 'monitor' 21 | else 22 | begin 23 | # monitor_ext is bundled since ruby 2.7.0 24 | require 'ext_monitor' 25 | rescue LoadError => _ 26 | require 'monitor' 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_relabel.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/plugin/output' 18 | 19 | module Fluent::Plugin 20 | class RelabelOutput < Output 21 | Fluent::Plugin.register_output('relabel', self) 22 | helpers :event_emitter 23 | 24 | def multi_workers_ready? 25 | true 26 | end 27 | 28 | def process(tag, es) 29 | router.emit_stream(tag, es) 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /test/helpers/process_extenstion.rb: -------------------------------------------------------------------------------- 1 | require 'timecop' 2 | 3 | module Process 4 | class << self 5 | alias_method :clock_gettime_original, :clock_gettime 6 | 7 | def clock_gettime(clock_id, unit = :float_second) 8 | # now only support CLOCK_REALTIME 9 | if Process::CLOCK_REALTIME == clock_id 10 | t = Time.now 11 | 12 | case unit 13 | when :float_second 14 | t.to_i + t.nsec / 1_000_000_000.0 15 | when :float_millisecond 16 | t.to_i * 1_000 + t.nsec / 1_000_000.0 17 | when :float_microsecond 18 | t.to_i * 1_000_000 + t.nsec / 1_000.0 19 | when :second 20 | t.to_i 21 | when :millisecond 22 | t.to_i * 1000 + t.nsec / 1_000_000 23 | when :microsecond 24 | t.to_i * 1_000_000 + t.nsec / 1_000 25 | when :nanosecond 26 | t.to_i * 1_000_000_000 + t.nsec 27 | end 28 | else 29 | Process.clock_gettime_original(clock_id, unit) 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /example/multi_filters.conf: -------------------------------------------------------------------------------- 1 | # This example is to measure optimized filter pipeline performance. 2 | 3 | 4 | @type sample 5 | tag test 6 | size 1000 7 | 8 | 9 | 10 | @type grep 11 | exclude1 hello . 12 | 13 | 14 | 15 | @type grep 16 | exclude1 hello . 17 | 18 | 19 | 20 | @type grep 21 | exclude1 hello . 22 | 23 | 24 | 25 | @type grep 26 | exclude1 hello . 27 | 28 | 29 | 30 | @type grep 31 | exclude1 hello . 32 | 33 | 34 | 35 | @type grep 36 | exclude1 hello . 37 | 38 | 39 | 40 | @type grep 41 | exclude1 hello . 42 | 43 | 44 | 45 | @type grep 46 | exclude1 hello . 47 | 48 | 49 | 50 | @type grep 51 | exclude1 hello . 52 | 53 | 54 | 55 | @type grep 56 | exclude1 hello . 57 | 58 | 59 | 60 | @type buffered_null 61 | 62 | -------------------------------------------------------------------------------- /lib/fluent/plugin/formatter_msgpack.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/plugin/formatter' 18 | 19 | module Fluent 20 | module Plugin 21 | class MessagePackFormatter < Formatter 22 | Plugin.register_formatter('msgpack', self) 23 | 24 | def formatter_type 25 | :binary 26 | end 27 | 28 | def format(tag, time, record) 29 | record.to_msgpack 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /.github/workflows/backport.yml: -------------------------------------------------------------------------------- 1 | name: Backport 2 | 3 | on: 4 | schedule: 5 | # Sun 10:00 (JST) 6 | - cron: '0 1 * * 0' 7 | workflow_dispatch: 8 | 9 | permissions: read-all 10 | 11 | concurrency: 12 | group: ${{ github.head_ref || github.sha }}-${{ github.workflow }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | test: 17 | runs-on: ubuntu-latest 18 | continue-on-error: false 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | ruby-version: ['3.4'] 23 | task: ['backport:v1_16', 'backport:v1_19'] 24 | 25 | name: Backport PR on ${{ matrix.os }} 26 | steps: 27 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 28 | - name: Set up Ruby 29 | uses: ruby/setup-ruby@ac793fdd38cc468a4dd57246fa9d0e868aba9085 # v1.270.0 30 | with: 31 | ruby-version: ${{ matrix.ruby-version }} 32 | bundler-cache: true 33 | - name: Run backport task 34 | shell: bash 35 | run: | 36 | bundle exec rake ${{ matrix.task }} 37 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Fluentd Maintainers 2 | 3 | - [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/) 4 | - [Okkez](https://github.com/okkez) 5 | - [Hiroshi Hatake](https://github.com/cosmo0920), [Chronosphere](https://chronosphere.io/) 6 | - [Masahiro Nakagawa](https://github.com/repeatedly) 7 | - [Satoshi Tagomori](https://github.com/tagomoris) 8 | - [Toru Takahashi](https://github.com/toru-takahashi), [Treasure Data](https://www.treasuredata.com/) 9 | - [Eduardo Silva](https://github.com/edsiper), [Chronosphere](https://chronosphere.io/) 10 | - [Fujimoto Seiji](https://github.com/fujimots) 11 | - [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/) 12 | - [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/) 13 | - [Daijiro Fukuda](https://github.com/daipom), [ClearCode](https://www.clear-code.com/) 14 | 15 | ## Sub Projects: Fluent Bit 16 | 17 | Fluent Bit maintainers list can be found here: 18 | 19 | - https://github.com/fluent/fluent-bit/blob/master/MAINTAINERS.md 20 | -------------------------------------------------------------------------------- /lib/fluent/plugin/parser_apache_error.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluentd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'fluent/plugin/parser_regexp' 18 | 19 | module Fluent 20 | module Plugin 21 | class ApacheErrorParser < RegexpParser 22 | Plugin.register_parser("apache_error", self) 23 | config_set_default :expression, /^\[[^ ]* (?