├── .gitignore
├── .rspec
├── .travis.yml
├── CHANGELOG.md
├── Gemfile
├── Gemfile.fluentd.0.10
├── Gemfile.fluentd.0.12
├── LICENSE
├── README.md
├── Rakefile
├── example
├── example.conf
└── workers.conf
├── fluent-plugin-record-reformer.gemspec
├── lib
└── fluent
│ └── plugin
│ ├── out_record_reformer.rb
│ └── out_record_reformer
│ ├── core.rb
│ ├── v12.rb
│ └── v14.rb
└── test
├── bench_out_record_reformer.rb
├── helper.rb
├── output_test_driver.rb
└── test_out_record_reformer.rb
/.gitignore:
--------------------------------------------------------------------------------
1 | /*.gem
2 | ~*
3 | #*
4 | *~
5 | .bundle
6 | Gemfile.lock
7 | .rbenv-version
8 | vendor
9 | doc/*
10 | tmp/*
11 | coverage
12 | .yardoc
13 | pkg/
14 | .ruby-version
15 |
--------------------------------------------------------------------------------
/.rspec:
--------------------------------------------------------------------------------
1 | --colour
2 | --format documentation
3 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | rvm:
2 | - 2.1.*
3 | - 2.2.*
4 | - 2.3.0
5 | - 2.4.0
6 | gemfile:
7 | - Gemfile
8 | - Gemfile.fluentd.0.12
9 | - Gemfile.fluentd.0.10
10 | matrix:
11 | exclude:
12 | - rvm: 2.4.0
13 | gemfile: Gemfile.fluentd.0.10
14 | before_install: gem update bundler
15 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | ## 0.9.1 (2017/07/26)
2 |
3 | Enhancements:
4 |
5 | * Support multi process workers of v0.14.12.
6 |
7 | ## 0.9.0 (2017/02/21)
8 |
9 | Enhancements:
10 |
11 | * Use v0.14 API for fluentd v0.14
12 |
13 | ## 0.8.3 (2017/01/26)
14 |
15 | Fixes
16 |
17 | * Apply `remove_keys` last, otherwise, `renew_time_key` could be removed before generating new time
18 |
19 | ## 0.8.2 (2016/08/21)
20 |
21 | Fixes
22 |
23 | * Prevent overwriting reserved placeholder keys such as tag, time, etc with `enable_ruby false` (thanks to @kimamula)
24 |
25 | ## 0.8.1 (2016/03/09)
26 |
27 | Fixes
28 |
29 | * Fix to be thread-safe
30 |
31 | Changes
32 |
33 | * Relax conditions which auto_typecast is applied for enable_ruby yes
34 |
35 | ## 0.8.0 (2016/01/28)
36 |
37 | Enhancements
38 |
39 | * Support `${record["key"]}` placeholder
40 | * Speed up `enable_ruby true`
41 |
42 | ## 0.7.2 (2015/12/29)
43 |
44 | Enhancements
45 |
46 | * Add desc to options (thanks to cosmo0920)
47 |
48 | ## 0.7.1 (2015/12/16)
49 |
50 | Enhancements
51 |
52 | * Add @id, @type, @label to BUILTIN_CONFIGURATIONS not to map into records (thanks to TrickyMonkey)
53 |
54 | ## 0.7.0 (2015/06/19)
55 |
56 | Enhancements
57 |
58 | * Add `auto_typecast` option (thanks to @piroor)
59 |
60 | ## 0.6.3 (2015/05/27)
61 |
62 | Fixes:
63 |
64 | * Fix not to include `renew_time_key` in records
65 |
66 | ## 0.6.2 (2015/05/27)
67 |
68 | Enhancements:
69 |
70 | * Add `renew_time_key` option (thanks to @tagomoris)
71 |
72 | ## 0.6.1 (2015/05/10)
73 |
74 | Enhancements:
75 |
76 | * Support label routing of Fluentd v0.12
77 |
78 | ## 0.6.0 (2015/04/11)
79 |
80 | Changes:
81 |
82 | * Accept field names starting with `@` (and any field names) in `enable_ruby false`
83 |
84 | ## 0.5.0 (2015/03/06)
85 |
86 | Enhancements:
87 |
88 | * Support JSON Array/Hash
89 | * Support placeholders for keys
90 |
91 | ## 0.4.0 (2014/10/31)
92 |
93 | Changes:
94 |
95 | * accept numbers as a record key
96 | * rescue if ruby code expansion failed, and log.warn
97 | * use newly test-unit gem instead of rspec
98 |
99 | ## 0.3.0 (2014/10/01)
100 |
101 | Fixes:
102 |
103 | * Fix to support camelCase record key name with `enable_ruby false`
104 |
105 | ## 0.2.10 (2014/09/22)
106 |
107 | Changes:
108 |
109 | * Remove fluentd version constraint
110 |
111 | ## 0.2.9 (2014/05/14)
112 |
113 | Enhancements:
114 |
115 | * Add `keep_keys` option
116 |
117 | ## 0.2.8 (2014/04/12)
118 |
119 | Changes:
120 |
121 | * Deprecate `output_tag` option. Use `tag` option instead.
122 |
123 | ## 0.2.7 (2014/03/26)
124 |
125 | Fixes:
126 |
127 | * Fix `log` method was not available in the inner class #5.
128 |
129 | ## 0.2.6 (2014/02/24)
130 |
131 | Enhancement:
132 |
133 | * Add debug log
134 |
135 | ## 0.2.5 (2014/02/04)
136 |
137 | Enhancement:
138 |
139 | * Support `log_level` option of Fleuntd v0.10.43
140 |
141 | ## 0.2.4 (2014/01/30)
142 |
143 | Fixes:
144 |
145 | * Fix `unitialized constant OpenStruct` error (thanks to emcgee)
146 |
147 | ## 0.2.3 (2014/01/25)
148 |
149 | Changes:
150 |
151 | * Change ${time} placeholder from integer to string when `enable_ruby false`
152 |
153 | ## 0.2.2 (2014/01/20)
154 |
155 | Enhancement:
156 |
157 | * Add `tag_prefix` and `tag_suffix` placeholders. Thanks to [xthexder](https://github.com/xthexder).
158 |
159 | ## 0.2.1 (2014/01/15)
160 |
161 | Enhancement:
162 |
163 | * Speed up
164 |
165 | ## 0.2.0 (2014/01/15)
166 |
167 | Enhancement:
168 |
169 | * Support a `record` directive
170 | * Add `remove_keys` option
171 | * Add `renew_record` option
172 | * Add `enable_ruby` option
173 |
174 | ## 0.1.1 (2013/11/21)
175 |
176 | Changes:
177 |
178 | * change the name of `tags` placeholder to `tag_parts`. `tags` is still available for old version compatibility, though
179 |
180 | ## 0.1.0 (2013/09/09)
181 |
182 | Enhancement:
183 |
184 | * require 'pathname', 'uri', 'cgi' to use these utilities in a placeholder
185 |
186 | ## 0.0.3 (2013/08/07)
187 |
188 | Enhancement:
189 |
190 | * Enable to reform tag
191 |
192 | ## 0.0.2 (2013/08/07)
193 |
194 | Enhancement:
195 |
196 | * Increase possible placeholders more such as `method`.
197 |
198 | ## 0.0.1 (2013/05/02)
199 |
200 | First release
201 |
--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source "http://rubygems.org"
2 |
3 | gemspec
4 |
--------------------------------------------------------------------------------
/Gemfile.fluentd.0.10:
--------------------------------------------------------------------------------
1 | source "http://rubygems.org"
2 |
3 | gem 'fluentd', '~> 0.10.43'
4 | gemspec
5 |
--------------------------------------------------------------------------------
/Gemfile.fluentd.0.12:
--------------------------------------------------------------------------------
1 | source "http://rubygems.org"
2 |
3 | gem 'fluentd', '~> 0.12.0'
4 | gemspec
5 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2013 - 2015 Naotoshi Seo
2 |
3 | MIT License
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining
6 | a copy of this software and associated documentation files (the
7 | "Software"), to deal in the Software without restriction, including
8 | without limitation the rights to use, copy, modify, merge, publish,
9 | distribute, sublicense, and/or sell copies of the Software, and to
10 | permit persons to whom the Software is furnished to do so, subject to
11 | the following conditions:
12 |
13 | The above copyright notice and this permission notice shall be
14 | included in all copies or substantial portions of the Software.
15 |
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # fluent-plugin-record-reformer
2 |
3 | [](http://travis-ci.org/sonots/fluent-plugin-record-reformer)
4 |
5 | Fluentd plugin to add or replace fields of a event record
6 |
7 | ## Requirements
8 |
9 | See [.travis.yml](.travis.yml)
10 |
11 | Note that `fluent-plugin-record-reformer` supports both v0.14 API and v0.12 API in one gem.
12 |
13 | ## Installation
14 |
15 | Use RubyGems:
16 |
17 | gem install fluent-plugin-record-reformer
18 |
19 | ## Configuration
20 |
21 | Example:
22 |
23 |
24 | type record_reformer
25 | remove_keys remove_me
26 | renew_record false
27 | enable_ruby false
28 |
29 | tag reformed.${tag_prefix[-2]}
30 |
31 | hostname ${hostname}
32 | input_tag ${tag}
33 | last_tag ${tag_parts[-1]}
34 | message ${record['message']}, yay!
35 |
36 |
37 |
38 | Assume following input is coming (indented):
39 |
40 | ```js
41 | foo.bar {
42 | "remove_me":"bar",
43 | "not_remove_me":"bar",
44 | "message":"Hello world!"
45 | }
46 | ```
47 |
48 | then output becomes as below (indented):
49 |
50 | ```js
51 | reformed.foo {
52 | "not_remove_me":"bar",
53 | "hostname":"YOUR_HOSTNAME",
54 | "input_tag":"foo.bar",
55 | "last_tag":"bar",
56 | "message":"Hello world!, yay!",
57 | }
58 | ```
59 |
60 | ## Configuration (Classic Style)
61 |
62 | Example:
63 |
64 |
65 | type record_reformer
66 | remove_keys remove_me
67 | renew_record false
68 | enable_ruby false
69 | tag reformed.${tag_prefix[-2]}
70 |
71 | hostname ${hostname}
72 | input_tag ${tag}
73 | last_tag ${tag_parts[-1]}
74 | message ${record['message']}, yay!
75 |
76 |
77 | This results in same, but please note that following option parameters are reserved, so can not be used as a record key.
78 |
79 | ## Option Parameters
80 |
81 | - output_tag (obsolete)
82 |
83 | The output tag name. This option is deprecated. Use `tag` option instead
84 |
85 | - tag
86 |
87 | The output tag name.
88 |
89 | - remove_keys
90 |
91 | Specify record keys to be removed by a string separated by , (comma) like
92 |
93 | remove_keys message,foo
94 |
95 | - renew_record *bool*
96 |
97 | `renew_record true` creates an output record newly without extending (merging) the input record fields. Default is `false`.
98 |
99 | - renew\_time\_key *string*
100 |
101 | `renew_time_key foo` overwrites the time of events with a value of the record field `foo` if exists. The value of `foo` must be a unix time.
102 |
103 | - keep_keys
104 |
105 | You may want to remain some record fields although you specify `renew_record true`. Then, specify record keys to be kept by a string separated by , (comma) like
106 |
107 | keep_keys message,foo
108 |
109 | - enable_ruby *bool*
110 |
111 | Enable to use ruby codes in placeholders. See `Placeholders` section.
112 | Default is `true` (just for lower version compatibility).
113 |
114 | - auto_typecast *bool*
115 |
116 | Automatically cast the field types. Default is false.
117 | NOTE: This option is effective only for field values comprised of a single placeholder.
118 |
119 | Effective Examples:
120 |
121 | foo ${foo}
122 |
123 | Non-Effective Examples:
124 |
125 | foo ${foo}${bar}
126 | foo ${foo}bar
127 | foo 1
128 |
129 | Internally, this **keeps** the type of value if the value text is comprised of a single placeholder, otherwise, values are treated as strings.
130 |
131 | When you need to cast field types manually, [out_typecast](https://github.com/tarom/fluent-plugin-typecast) and [filter_typecast](https://github.com/sonots/fluent-plugin-filter_typecast) are available.
132 |
133 | ## Placeholders
134 |
135 | Following placeholders are available:
136 |
137 | * ${record["key"]} Record value of `key` such as `${record["message"]}` in the above example (available from v0.8.0).
138 | * Originally, record placeholders were available as `${key}` such as `${message}`. This is still kept for the backward compatibility, but would be removed in the future.
139 | * ${hostname} Hostname of the running machine
140 | * ${tag} Input tag
141 | * ${time} Time of the event
142 | * ${tags[N]} (Obsolete. Use tag\_parts) Input tag splitted by '.'
143 | * ${tag\_parts[N]} Input tag splitted by '.' indexed with N such as `${tag_parts[0]}`, `${tag_parts[-1]}`.
144 | * ${tag\_prefix[N]} Tag parts before and on the index N. For example,
145 |
146 | Input tag: prefix.test.tag.suffix
147 |
148 | ${tag_prefix[0]} => prefix
149 | ${tag_prefix[1]} => prefix.test
150 | ${tag_prefix[-2]} => prefix.test.tag
151 | ${tag_prefix[-1]} => prefix.test.tag.suffix
152 |
153 | * ${tag\_suffix[N]} Tag parts after and on the index N. For example,
154 |
155 | Input tag: prefix.test.tag.suffix
156 |
157 | ${tag_suffix[0]} => prefix.test.tag.suffix
158 | ${tag_suffix[1]} => test.tag.suffix
159 | ${tag_suffix[-2]} => tag.suffix
160 | ${tag_suffix[-1]} => suffix
161 |
162 | It is also possible to write a ruby code in placeholders if you set `enable_ruby true` option, so you may write some codes as
163 |
164 | * ${time.strftime('%Y-%m-%dT%H:%M:%S%z')}
165 | * ${tag\_parts.last}
166 |
167 | but, please note that enabling ruby codes is not encouraged by security reasons and also in terms of the performance.
168 |
169 | ## Relatives
170 |
171 | Following plugins look similar:
172 |
173 | * [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier)
174 | * [fluent-plugin-format](https://github.com/mach/fluent-plugin-format)
175 | * [fluent-plugin-add](https://github.com/yu-yamada/fluent-plugin-add)
176 | * [filter_record_transformer](http://docs.fluentd.org/v0.12/articles/filter_record_transformer) is a Fluentd v0.12 built-in plugin which is based on record-reformer.
177 |
178 | ## ChangeLog
179 |
180 | See [CHANGELOG.md](CHANGELOG.md) for details.
181 |
182 | ## Contributing
183 |
184 | 1. Fork it
185 | 2. Create your feature branch (`git checkout -b my-new-feature`)
186 | 3. Commit your changes (`git commit -am 'Add some feature'`)
187 | 4. Push to the branch (`git push origin my-new-feature`)
188 | 5. Create new [Pull Request](../../pull/new/master)
189 |
190 | ## Copyright
191 |
192 | Copyright (c) 2013 - 2015 Naotoshi Seo. See [LICENSE](LICENSE) for details.
193 |
--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | # encoding: utf-8
2 | require "bundler/gem_tasks"
3 |
4 | require 'rake/testtask'
5 | desc 'Run test_unit based test'
6 | Rake::TestTask.new(:test) do |t|
7 | t.libs << "test"
8 | t.test_files = Dir["test/**/test_*.rb"].sort
9 | t.verbose = true
10 | #t.warning = true
11 | end
12 | task :default => :test
13 |
14 | desc 'Open an irb session preloaded with the gem library'
15 | task :console do
16 | sh 'irb -rubygems -I lib'
17 | end
18 | task :c => :console
19 |
--------------------------------------------------------------------------------
/example/example.conf:
--------------------------------------------------------------------------------
1 |
2 | @type dummy
3 | tag dummy
4 | dummy {"message":"foo","time":1432732710,"members":["Alice"]}
5 |
6 |
7 |
8 | @type record_reformer
9 | renew_time_key time
10 | tag reformed.${tag}
11 | enable_ruby true
12 | auto_typecast true
13 |
14 | members ${members + ["Bob"]}
15 |
16 |
17 |
18 |
19 | @type stdout
20 |
21 |
--------------------------------------------------------------------------------
/example/workers.conf:
--------------------------------------------------------------------------------
1 |
2 | workers 2 # v0.4.12
3 |
4 |
5 | @include example.conf
6 |
--------------------------------------------------------------------------------
/fluent-plugin-record-reformer.gemspec:
--------------------------------------------------------------------------------
1 | # encoding: utf-8
2 | $:.push File.expand_path('../lib', __FILE__)
3 |
4 | Gem::Specification.new do |gem|
5 | gem.name = "fluent-plugin-record-reformer"
6 | gem.version = "0.9.1"
7 | gem.authors = ["Naotoshi Seo"]
8 | gem.email = "sonots@gmail.com"
9 | gem.homepage = "https://github.com/sonots/fluent-plugin-record-reformer"
10 | gem.description = "Fluentd plugin to add or replace fields of a event record"
11 | gem.summary = gem.description
12 | gem.licenses = ["MIT"]
13 | gem.has_rdoc = false
14 |
15 | gem.files = `git ls-files`.split("\n")
16 | gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17 | gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18 | gem.require_paths = ['lib']
19 |
20 | gem.add_dependency "fluentd"
21 | gem.add_development_dependency "rake"
22 | gem.add_development_dependency "pry"
23 | gem.add_development_dependency "pry-nav"
24 | gem.add_development_dependency "test-unit"
25 | gem.add_development_dependency "test-unit-rr"
26 | gem.add_development_dependency "timecop"
27 | end
28 |
--------------------------------------------------------------------------------
/lib/fluent/plugin/out_record_reformer.rb:
--------------------------------------------------------------------------------
1 | require 'fluent/version'
2 | major, minor, patch = Fluent::VERSION.split('.').map(&:to_i)
3 | if major > 0 || (major == 0 && minor >= 14)
4 | require_relative 'out_record_reformer/v14'
5 | else
6 | require_relative 'out_record_reformer/v12'
7 | end
8 |
--------------------------------------------------------------------------------
/lib/fluent/plugin/out_record_reformer/core.rb:
--------------------------------------------------------------------------------
1 | require 'ostruct'
2 | require 'socket'
3 |
4 | module Fluent
5 | module RecordReformerOutputCore
6 | def initialize
7 | super
8 | end
9 |
10 | def self.included(klass)
11 | klass.config_param :output_tag, :string, :default => nil, # obsolete
12 | :desc => 'The output tag name. This option is deprecated. Use `tag` option instead.'
13 | klass.config_param :tag, :string, :default => nil,
14 | :desc => 'The output tag name.'
15 | klass.config_param :remove_keys, :string, :default => nil,
16 | :desc => 'Specify record keys to be removed by a string separated by , (comma).'
17 | klass.config_param :keep_keys, :string, :default => nil,
18 | :desc => 'Specify record keys to be kept by a string separated by , (comma).'
19 | klass.config_param :renew_record, :bool, :default => false,
20 | :desc => 'Creates an output record newly without extending (merging) the input record fields.'
21 | klass.config_param :renew_time_key, :string, :default => nil,
22 | :desc => 'Overwrites the time of events with a value of the record field.'
23 | klass.config_param :enable_ruby, :bool, :default => true, # true for lower version compatibility
24 | :desc => 'Enable to use ruby codes in placeholders.'
25 | klass.config_param :auto_typecast, :bool, :default => false, # false for lower version compatibility
26 | :desc => 'Automatically cast the field types.'
27 | end
28 |
29 | BUILTIN_CONFIGURATIONS = %W(@id @type @label type tag output_tag remove_keys renew_record keep_keys enable_ruby renew_time_key auto_typecast)
30 |
31 | def configure(conf)
32 | super
33 |
34 | map = {}
35 | conf.each_pair { |k, v|
36 | next if BUILTIN_CONFIGURATIONS.include?(k)
37 | conf.has_key?(k) # to suppress unread configuration warning
38 | map[k] = parse_value(v)
39 | }
40 | # directive
41 | conf.elements.select { |element| element.name == 'record' }.each { |element|
42 | element.each_pair { |k, v|
43 | element.has_key?(k) # to suppress unread configuration warning
44 | map[k] = parse_value(v)
45 | }
46 | }
47 |
48 | if @remove_keys
49 | @remove_keys = @remove_keys.split(',')
50 | end
51 |
52 | if @keep_keys
53 | raise Fluent::ConfigError, "out_record_reformer: `renew_record` must be true to use `keep_keys`" unless @renew_record
54 | @keep_keys = @keep_keys.split(',')
55 | end
56 |
57 | if @output_tag and @tag.nil? # for lower version compatibility
58 | log.warn "out_record_reformer: `output_tag` is deprecated. Use `tag` option instead."
59 | @tag = @output_tag
60 | end
61 | if @tag.nil?
62 | raise Fluent::ConfigError, "out_record_reformer: `tag` must be specified"
63 | end
64 |
65 | placeholder_expander_params = {
66 | :log => log,
67 | :auto_typecast => @auto_typecast,
68 | }
69 | @placeholder_expander =
70 | if @enable_ruby
71 | # require utilities which would be used in ruby placeholders
72 | require 'pathname'
73 | require 'uri'
74 | require 'cgi'
75 | RubyPlaceholderExpander.new(placeholder_expander_params)
76 | else
77 | PlaceholderExpander.new(placeholder_expander_params)
78 | end
79 | @map = @placeholder_expander.preprocess_map(map)
80 | @tag = @placeholder_expander.preprocess_map(@tag)
81 |
82 | @hostname = Socket.gethostname
83 | end
84 |
85 | def process(tag, es)
86 | tag_parts = tag.split('.')
87 | tag_prefix = tag_prefix(tag_parts)
88 | tag_suffix = tag_suffix(tag_parts)
89 | placeholder_values = {
90 | 'tag' => tag,
91 | 'tags' => tag_parts, # for old version compatibility
92 | 'tag_parts' => tag_parts,
93 | 'tag_prefix' => tag_prefix,
94 | 'tag_suffix' => tag_suffix,
95 | 'hostname' => @hostname,
96 | }
97 | last_record = nil
98 | es.each {|time, record|
99 | last_record = record # for debug log
100 | placeholder_values.merge!({
101 | 'time' => @placeholder_expander.time_value(time),
102 | 'record' => record,
103 | })
104 | new_tag, new_record = reform(@tag, record, placeholder_values)
105 | if new_tag
106 | if @renew_time_key && new_record.has_key?(@renew_time_key)
107 | time = new_record[@renew_time_key].to_i
108 | end
109 | @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
110 | router.emit(new_tag, time, new_record)
111 | end
112 | }
113 | rescue => e
114 | log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.first}"
115 | log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
116 | end
117 |
118 | private
119 |
120 | def parse_value(value_str)
121 | if value_str.start_with?('{', '[')
122 | JSON.parse(value_str)
123 | else
124 | value_str
125 | end
126 | rescue => e
127 | log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", :error_class => e.class, :error => e.message
128 | value_str # emit as string
129 | end
130 |
131 | def reform(tag, record, placeholder_values)
132 | placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
133 |
134 | new_tag = expand_placeholders(tag, placeholders)
135 |
136 | new_record = @renew_record ? {} : record.dup
137 | @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
138 | new_record.merge!(expand_placeholders(@map, placeholders))
139 |
140 | [new_tag, new_record]
141 | end
142 |
143 | def expand_placeholders(value, placeholders)
144 | if value.is_a?(String)
145 | new_value = @placeholder_expander.expand(value, placeholders)
146 | elsif value.is_a?(Hash)
147 | new_value = {}
148 | value.each_pair do |k, v|
149 | new_key = @placeholder_expander.expand(k, placeholders, true)
150 | new_value[new_key] = expand_placeholders(v, placeholders)
151 | end
152 | elsif value.is_a?(Array)
153 | new_value = []
154 | value.each_with_index do |v, i|
155 | new_value[i] = expand_placeholders(v, placeholders)
156 | end
157 | else
158 | new_value = value
159 | end
160 | new_value
161 | end
162 |
163 | def tag_prefix(tag_parts)
164 | return [] if tag_parts.empty?
165 | tag_prefix = [tag_parts.first]
166 | 1.upto(tag_parts.size-1).each do |i|
167 | tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
168 | end
169 | tag_prefix
170 | end
171 |
172 | def tag_suffix(tag_parts)
173 | return [] if tag_parts.empty?
174 | rev_tag_parts = tag_parts.reverse
175 | rev_tag_suffix = [rev_tag_parts.first]
176 | 1.upto(tag_parts.size-1).each do |i|
177 | rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
178 | end
179 | rev_tag_suffix.reverse!
180 | end
181 |
182 | # THIS CLASS MUST BE THREAD-SAFE
183 | class PlaceholderExpander
184 | attr_reader :placeholders, :log
185 |
186 | def initialize(params)
187 | @log = params[:log]
188 | @auto_typecast = params[:auto_typecast]
189 | end
190 |
191 | def time_value(time)
192 | Time.at(time).to_s
193 | end
194 |
195 | def preprocess_map(value, force_stringify = false)
196 | value
197 | end
198 |
199 | def prepare_placeholders(placeholder_values)
200 | placeholders = {}
201 |
202 | placeholder_values.each do |key, value|
203 | if value.kind_of?(Array) # tag_parts, etc
204 | size = value.size
205 | value.each_with_index do |v, idx|
206 | placeholders.store("${#{key}[#{idx}]}", v)
207 | placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
208 | end
209 | elsif value.kind_of?(Hash) # record, etc
210 | value.each do |k, v|
211 | unless placeholder_values.has_key?(k) # prevent overwriting the reserved keys such as tag
212 | placeholders.store("${#{k}}", v)
213 | end
214 | placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
215 | end
216 | else # string, interger, float, and others?
217 | placeholders.store("${#{key}}", value)
218 | end
219 | end
220 |
221 | placeholders
222 | end
223 |
224 | # Expand string with placeholders
225 | #
226 | # @param [String] str
227 | # @param [Boolean] force_stringify the value must be string, used for hash key
228 | def expand(str, placeholders, force_stringify = false)
229 | if @auto_typecast and !force_stringify
230 | single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
231 | if single_placeholder_matched
232 | log_if_unknown_placeholder($1, placeholders)
233 | return placeholders[single_placeholder_matched[1]]
234 | end
235 | end
236 | str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
237 | log_if_unknown_placeholder($1, placeholders)
238 | placeholders[$1]
239 | }
240 | end
241 |
242 | private
243 |
244 | def log_if_unknown_placeholder(placeholder, placeholders)
245 | unless placeholders.include?(placeholder)
246 | log.warn "record_reformer: unknown placeholder `#{placeholder}` found"
247 | end
248 | end
249 | end
250 |
251 | # THIS CLASS MUST BE THREAD-SAFE
252 | class RubyPlaceholderExpander
253 | attr_reader :log
254 |
255 | def initialize(params)
256 | @log = params[:log]
257 | @auto_typecast = params[:auto_typecast]
258 | @cleanroom_expander = CleanroomExpander.new
259 | end
260 |
261 | def time_value(time)
262 | Time.at(time)
263 | end
264 |
265 | # Preprocess record map to convert into ruby string expansion
266 | #
267 | # @param [Hash|String|Array] value record map config
268 | # @param [Boolean] force_stringify the value must be string, used for hash key
269 | def preprocess_map(value, force_stringify = false)
270 | new_value = nil
271 | if value.is_a?(String)
272 | if @auto_typecast and !force_stringify
273 | num_placeholders = value.scan('${').size
274 | if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
275 | new_value = value[2..-2] # ${..} => ..
276 | end
277 | end
278 | unless new_value
279 | new_value = "%Q[#{value.gsub('${', '#{')}]" # xx${..}xx => %Q[xx#{..}xx]
280 | end
281 | elsif value.is_a?(Hash)
282 | new_value = {}
283 | value.each_pair do |k, v|
284 | new_value[preprocess_map(k, true)] = preprocess_map(v)
285 | end
286 | elsif value.is_a?(Array)
287 | new_value = []
288 | value.each_with_index do |v, i|
289 | new_value[i] = preprocess_map(v)
290 | end
291 | else
292 | new_value = value
293 | end
294 | new_value
295 | end
296 |
297 | def prepare_placeholders(placeholder_values)
298 | placeholder_values
299 | end
300 |
301 | # Expand string with placeholders
302 | #
303 | # @param [String] str
304 | def expand(str, placeholders, force_stringify = false)
305 | @cleanroom_expander.expand(
306 | str,
307 | placeholders['tag'],
308 | placeholders['time'],
309 | placeholders['record'],
310 | placeholders['tag_parts'],
311 | placeholders['tag_prefix'],
312 | placeholders['tag_suffix'],
313 | placeholders['hostname'],
314 | )
315 | rescue => e
316 | log.warn "record_reformer: failed to expand `#{str}`", :error_class => e.class, :error => e.message
317 | log.warn_backtrace
318 | nil
319 | end
320 |
321 | class CleanroomExpander
322 | def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
323 | tags = tag_parts # for old version compatibility
324 | Thread.current[:record_reformer_record] = record # for old version compatibility
325 | instance_eval(__str_to_eval__)
326 | end
327 |
328 | # for old version compatibility
329 | def method_missing(name)
330 | key = name.to_s
331 | record = Thread.current[:record_reformer_record]
332 | if record.has_key?(key)
333 | record[key]
334 | else
335 | raise NameError, "undefined local variable or method `#{key}'"
336 | end
337 | end
338 |
339 | (Object.instance_methods).each do |m|
340 | undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
341 | end
342 | end
343 | end
344 | end
345 | end
346 |
--------------------------------------------------------------------------------
/lib/fluent/plugin/out_record_reformer/v12.rb:
--------------------------------------------------------------------------------
1 | require_relative 'core'
2 |
3 | module Fluent
4 | class RecordReformerOutput < Output
5 | Fluent::Plugin.register_output('record_reformer', self)
6 |
7 | include ::Fluent::RecordReformerOutputCore
8 |
9 | def initialize
10 | super
11 | end
12 |
13 | # To support log_level option implemented by Fluentd v0.10.43
14 | unless method_defined?(:log)
15 | define_method("log") { $log }
16 | end
17 |
18 | # Define `router` method of v0.12 to support v0.10 or earlier
19 | unless method_defined?(:router)
20 | define_method("router") { Fluent::Engine }
21 | end
22 |
23 | def configure(conf)
24 | super
25 | end
26 |
27 | def emit(tag, es, chain)
28 | process(tag, es)
29 | chain.next
30 | end
31 | end
32 | end
33 |
--------------------------------------------------------------------------------
/lib/fluent/plugin/out_record_reformer/v14.rb:
--------------------------------------------------------------------------------
1 | require_relative 'core'
2 |
3 | module Fluent
4 | class Plugin::RecordReformerOutput < Plugin::Output
5 | Fluent::Plugin.register_output('record_reformer', self)
6 |
7 | helpers :event_emitter
8 | include ::Fluent::RecordReformerOutputCore
9 |
10 | def initialize
11 | super
12 | end
13 |
14 | def configure(conf)
15 | super
16 | end
17 |
18 | def multi_workers_ready?
19 | true
20 | end
21 |
22 | def process(tag, es)
23 | super
24 | end
25 | end
26 | end
27 |
--------------------------------------------------------------------------------
/test/bench_out_record_reformer.rb:
--------------------------------------------------------------------------------
1 | # encoding: UTF-8
2 | require_relative 'helper'
3 | require 'fluent/plugin/out_record_reformer'
4 | require 'benchmark'
5 | Fluent::Test.setup
6 |
7 | def create_driver(config, tag = 'foo.bar')
8 | Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, tag).configure(config)
9 | end
10 |
11 | # setup
12 | message = {'message' => "2013/01/13T07:02:11.124202 INFO GET /ping"}
13 | time = Time.now.to_i
14 |
15 | enable_ruby_driver = create_driver(%[
16 | enable_ruby true
17 | output_tag reformed.${tag}
18 | message ${tag_parts[0]}
19 | ])
20 | disable_ruby_driver = create_driver(%[
21 | enable_ruby false
22 | output_tag reformed.${tag}
23 | message ${tag_parts[0]}
24 | ])
25 |
26 | # bench
27 | n = 1000
28 | Benchmark.bm(7) do |x|
29 | x.report("enable_ruby") { enable_ruby_driver.run { n.times { enable_ruby_driver.emit(message, time) } } }
30 | x.report("disable_ruby") { disable_ruby_driver.run { n.times { disable_ruby_driver.emit(message, time) } } }
31 | end
32 |
33 | #BEFORE REFACTORING
34 | # user system total real
35 | #enable_ruby 0.310000 0.000000 0.310000 ( 0.835560)
36 | #disable_ruby 0.150000 0.000000 0.150000 ( 0.679239)
37 |
38 | #AFTER REFACTORING (PlaceholderParser and RubyPlaceholderParser)
39 | # user system total real
40 | #enable_ruby 0.290000 0.010000 0.300000 ( 0.815281)
41 | #disable_ruby 0.060000 0.000000 0.060000 ( 0.588556)
42 |
--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
1 | require 'test/unit'
2 | require 'test/unit/rr'
3 | require 'timecop'
4 | require 'fluent/log'
5 | require 'fluent/test'
6 |
7 | unless defined?(Test::Unit::AssertionFailedError)
8 | class Test::Unit::AssertionFailedError < StandardError
9 | end
10 | end
11 |
12 | # Reduce sleep period at
13 | # https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81
14 | module Fluent
15 | module Test
16 | class TestDriver
17 | def run(num_waits = 10, &block)
18 | @instance.start
19 | begin
20 | # wait until thread starts
21 | # num_waits.times { sleep 0.05 }
22 | sleep 0.05
23 | return yield
24 | ensure
25 | @instance.shutdown
26 | end
27 | end
28 | end
29 | end
30 | end
31 |
32 | require_relative 'output_test_driver'
33 |
--------------------------------------------------------------------------------
/test/output_test_driver.rb:
--------------------------------------------------------------------------------
1 | # This test driver makes a compatible layer for v0.14 as of v0.12
2 |
3 | # d = create_driver(conf, use_v1, default_tag: @tag)
4 | # time = event_time("2010-05-04 03:02:01")
5 | # d.run do
6 | # d.emit(record, time)
7 | # end
8 | # d.emits
9 |
10 | require 'fluent/version'
11 | major, minor, patch = Fluent::VERSION.split('.').map(&:to_i)
12 | if major > 0 || (major == 0 && minor >= 14)
13 | require 'fluent/test/driver/output'
14 | require 'fluent/test/helpers'
15 | include Fluent::Test::Helpers
16 |
17 | class OutputTestDriver < Fluent::Test::Driver::Output
18 | def initialize(klass, tag)
19 | super(klass)
20 | @tag = tag
21 | end
22 |
23 | def configure(conf, use_v1)
24 | super(conf, syntax: use_v1 ? :v1 : :v0)
25 | end
26 |
27 | def run(&block)
28 | super(default_tag: @tag, &block)
29 | end
30 |
31 | def emit(record, time)
32 | feed(time, record)
33 | end
34 |
35 | def emits
36 | events
37 | end
38 | end
39 |
40 | def create_driver(conf, use_v1, default_tag: @tag)
41 | OutputTestDriver.new(Fluent::Plugin::RecordReformerOutput, default_tag).configure(conf, use_v1)
42 | end
43 | else
44 | def event_time(str)
45 | Time.parse(str)
46 | end
47 |
48 | def create_driver(conf, use_v1, default_tag: @tag)
49 | Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, default_tag).configure(conf, use_v1)
50 | end
51 | end
52 |
--------------------------------------------------------------------------------
/test/test_out_record_reformer.rb:
--------------------------------------------------------------------------------
1 | require_relative 'helper'
2 | require 'time'
3 | require 'fluent/plugin/out_record_reformer'
4 |
5 | Fluent::Test.setup
6 |
7 | class RecordReformerOutputTest < Test::Unit::TestCase
8 | def emit(config, use_v1, msgs = [''])
9 | d = create_driver(config, use_v1)
10 | d.run do
11 | records = msgs.map do |msg|
12 | next msg if msg.is_a?(Hash)
13 | { 'eventType0' => 'bar', 'message' => msg }
14 | end
15 | records.each do |record|
16 | d.emit(record, @time)
17 | end
18 | end
19 |
20 | @instance = d.instance
21 | d.emits
22 | end
23 |
24 | setup do
25 | @hostname = Socket.gethostname.chomp
26 | @tag = 'test.tag'
27 | @tag_parts = @tag.split('.')
28 | @time = event_time("2010-05-04 03:02:01")
29 | Timecop.freeze(@time)
30 | end
31 |
32 | teardown do
33 | Timecop.return
34 | end
35 |
36 | CONFIG = %[
37 | tag reformed.${tag}
38 |
39 | hostname ${hostname}
40 | input_tag ${tag}
41 | time ${time.to_s}
42 | message ${hostname} ${tag_parts.last} ${URI.escape(message)}
43 | ]
44 |
45 | [true, false].each do |use_v1|
46 | sub_test_case 'configure' do
47 | test 'typical usage' do
48 | assert_nothing_raised do
49 | create_driver(CONFIG, use_v1)
50 | end
51 | end
52 |
53 | test "tag is not specified" do
54 | assert_raise(Fluent::ConfigError) do
55 | create_driver('', use_v1)
56 | end
57 | end
58 |
59 | test "keep_keys must be specified together with renew_record true" do
60 | assert_raise(Fluent::ConfigError) do
61 | create_driver(%[keep_keys a], use_v1)
62 | end
63 | end
64 | end
65 |
66 | sub_test_case "test options" do
67 | test 'typical usage' do
68 | msgs = ['1', '2']
69 | emits = emit(CONFIG, use_v1, msgs)
70 | assert_equal 2, emits.size
71 | emits.each_with_index do |(tag, time, record), i|
72 | assert_equal("reformed.#{@tag}", tag)
73 | assert_equal('bar', record['eventType0'])
74 | assert_equal(@hostname, record['hostname'])
75 | assert_equal(@tag, record['input_tag'])
76 | assert_equal(Time.at(@time).localtime.to_s, record['time'])
77 | assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", record['message'])
78 | end
79 | end
80 |
81 | test '(obsolete) output_tag' do
82 | config = %[output_tag reformed.${tag}]
83 | msgs = ['1']
84 | emits = emit(config, use_v1, msgs)
85 | emits.each_with_index do |(tag, time, record), i|
86 | assert_equal("reformed.#{@tag}", tag)
87 | end
88 | end
89 |
90 | test 'record directive' do
91 | config = %[
92 | tag reformed.${tag}
93 |
94 |
95 | hostname ${hostname}
96 | tag ${tag}
97 | time ${time.to_s}
98 | message ${hostname} ${tag_parts.last} ${message}
99 |
100 | ]
101 | msgs = ['1', '2']
102 | emits = emit(config, use_v1, msgs)
103 | emits.each_with_index do |(tag, time, record), i|
104 | assert_equal("reformed.#{@tag}", tag)
105 | assert_equal('bar', record['eventType0'])
106 | assert_equal(@hostname, record['hostname'])
107 | assert_equal(@tag, record['tag'])
108 | assert_equal(Time.at(@time).localtime.to_s, record['time'])
109 | assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", record['message'])
110 | end
111 | end
112 |
113 | test 'remove_keys' do
114 | config = CONFIG + %[remove_keys eventType0,message]
115 | emits = emit(config, use_v1)
116 | emits.each_with_index do |(tag, time, record), i|
117 | assert_equal("reformed.#{@tag}", tag)
118 | assert_not_include(record, 'eventType0')
119 | assert_equal(@hostname, record['hostname'])
120 | assert_equal(@tag, record['input_tag'])
121 | assert_equal(Time.at(@time).localtime.to_s, record['time'])
122 | assert_not_include(record, 'message')
123 | end
124 | end
125 |
126 | test 'renew_record' do
127 | config = CONFIG + %[renew_record true]
128 | msgs = ['1', '2']
129 | emits = emit(config, use_v1, msgs)
130 | emits.each_with_index do |(tag, time, record), i|
131 | assert_equal("reformed.#{@tag}", tag)
132 | assert_not_include(record, 'eventType0')
133 | assert_equal(@hostname, record['hostname'])
134 | assert_equal(@tag, record['input_tag'])
135 | assert_equal(Time.at(@time).localtime.to_s, record['time'])
136 | assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", record['message'])
137 | end
138 | end
139 |
140 | test 'renew_time_key' do
141 | times = [ Time.at(event_time("2010-05-04 03:02:02")), Time.at(event_time("2010-05-04 03:02:03")) ]
142 | config = <
147 | event_time_key ${Time.parse(record["message"]).to_i}
148 |
149 | EOC
150 | msgs = times.map{|t| t.to_s }
151 | emits = emit(config, use_v1, msgs)
152 | emits.each_with_index do |(tag, time, record), i|
153 | assert_equal("reformed.#{@tag}", tag)
154 | assert_equal(times[i].to_i, time)
155 | assert_true(record.has_key?('event_time_key'))
156 | end
157 | end
158 |
159 | test 'renew_time_key and remove_keys' do
160 | config = <
166 | event_time_key ${Time.parse(record["message"]).to_i}
167 |
168 | EOC
169 | times = [ Time.at(event_time("2010-05-04 03:02:02")), Time.at(event_time("2010-05-04 03:02:03")) ]
170 | msgs = times.map{|t| t.to_s }
171 | emits = emit(config, use_v1, msgs)
172 | emits.each_with_index do |(tag, time, record), i|
173 | assert_equal("reformed.#{@tag}", tag)
174 | assert_equal(times[i].to_i, time)
175 | assert_false(record.has_key?('event_time_key'))
176 | end
177 | end
178 |
179 | test 'keep_keys' do
180 | config = %[tag reformed.${tag}\nrenew_record true\nkeep_keys eventType0,message]
181 | msgs = ['1', '2']
182 | emits = emit(config, use_v1, msgs)
183 | emits.each_with_index do |(tag, time, record), i|
184 | assert_equal("reformed.#{@tag}", tag)
185 | assert_equal('bar', record['eventType0'])
186 | assert_equal(msgs[i], record['message'])
187 | end
188 | end
189 |
190 | test 'enable_ruby no' do
191 | config = %[
192 | tag reformed.${tag}
193 | enable_ruby no
194 |
195 | message ${hostname} ${tag_parts.last} ${URI.encode(message)}
196 |
197 | ]
198 | msgs = ['1', '2']
199 | emits = emit(config, use_v1, msgs)
200 | emits.each_with_index do |(tag, time, record), i|
201 | assert_equal("reformed.#{@tag}", tag)
202 | assert_equal("#{@hostname} ", record['message'])
203 | end
204 | end
205 | end
206 |
207 | sub_test_case 'test placeholders' do
208 | %w[yes no].each do |enable_ruby|
209 |
210 | test "hostname with enble_ruby #{enable_ruby}" do
211 | config = %[
212 | tag tag
213 | enable_ruby #{enable_ruby}
214 |
215 | message ${hostname}
216 |
217 | ]
218 | emits = emit(config, use_v1)
219 | emits.each do |(tag, time, record)|
220 | assert_equal(@hostname, record['message'])
221 | end
222 | end
223 |
224 | test "tag with enable_ruby #{enable_ruby}" do
225 | config = %[
226 | tag tag
227 | enable_ruby #{enable_ruby}
228 |
229 | message ${tag}
230 |
231 | ]
232 | emits = emit(config, use_v1)
233 | emits.each do |(tag, time, record)|
234 | assert_equal(@tag, record['message'])
235 | end
236 | end
237 |
238 | test "tag_parts with enable_ruby #{enable_ruby}" do
239 | config = %[
240 | tag tag
241 | enable_ruby #{enable_ruby}
242 |
243 | message ${tag_parts[0]} ${tag_parts[-1]}
244 |
245 | ]
246 | expected = "#{@tag.split('.').first} #{@tag.split('.').last}"
247 | emits = emit(config, use_v1)
248 | emits.each do |(tag, time, record)|
249 | assert_equal(expected, record['message'])
250 | end
251 | end
252 |
253 | test "(obsolete) tags with enable_ruby #{enable_ruby}" do
254 | config = %[
255 | tag tag
256 | enable_ruby #{enable_ruby}
257 |
258 | message ${tags[0]} ${tags[-1]}
259 |
260 | ]
261 | expected = "#{@tag.split('.').first} #{@tag.split('.').last}"
262 | emits = emit(config, use_v1)
263 | emits.each do |(tag, time, record)|
264 | assert_equal(expected, record['message'])
265 | end
266 | end
267 |
268 | test "${tag_prefix[N]} and ${tag_suffix[N]} with enable_ruby #{enable_ruby}" do
269 | config = %[
270 | tag tag
271 | enable_ruby #{enable_ruby}
272 |
273 | message ${tag_prefix[1]} ${tag_prefix[-2]} ${tag_suffix[2]} ${tag_suffix[-3]}
274 |
275 | ]
276 | @tag = 'prefix.test.tag.suffix'
277 | expected = "prefix.test prefix.test.tag tag.suffix test.tag.suffix"
278 | emits = emit(config, use_v1)
279 | emits.each do |(tag, time, record)|
280 | assert_equal(expected, record['message'])
281 | end
282 | end
283 |
284 | test "time with enable_ruby #{enable_ruby}" do
285 | config = %[
286 | tag tag
287 | enable_ruby #{enable_ruby}
288 |
289 | message ${time}
290 |
291 | ]
292 | emits = emit(config, use_v1)
293 | emits.each do |(tag, time, record)|
294 | assert_equal(Time.at(time).localtime.to_s, record['message'])
295 | end
296 | end
297 |
298 | test "record keys with enable_ruby #{enable_ruby}" do
299 | config = %[
300 | tag tag
301 | enable_ruby #{enable_ruby}
302 | remove_keys eventType0
303 |
304 | message bar ${message}
305 | eventtype ${eventType0}
306 |
307 | ]
308 | msgs = ['1', '2']
309 | emits = emit(config, use_v1, msgs)
310 | emits.each_with_index do |(tag, time, record), i|
311 | assert_not_include(record, 'eventType0')
312 | assert_equal("bar", record['eventtype'])
313 | assert_equal("bar #{msgs[i]}", record['message'])
314 | end
315 | end
316 |
317 | test "Prevent overriting reserved keys (such as tag, etc) #40 with enable_ruby #{enable_ruby}" do
318 | config = %[
319 | tag tag
320 | enable_ruby #{enable_ruby}
321 |
322 | new_tag ${tag}
323 | new_time ${time}
324 | new_record_tag ${record["tag"]}
325 | new_record_time ${record["time"]}
326 |
327 | ]
328 | records = [{'tag' => 'tag', 'time' => 'time'}]
329 | emits = emit(config, use_v1, records)
330 | emits.each do |(tag, time, record)|
331 | assert_not_equal('tag', record['new_tag'])
332 | assert_equal(@tag, record['new_tag'])
333 | assert_not_equal('time', record['new_time'])
334 | assert_equal(Time.at(@time).localtime.to_s, record['new_time'])
335 | assert_equal('tag', record['new_record_tag'])
336 | assert_equal('time', record['new_record_time'])
337 | end
338 | end
339 |
340 | test "hash values with placeholders with enable_ruby #{enable_ruby}" do
341 | config = %[
342 | tag tag
343 | enable_ruby #{enable_ruby}
344 |
345 | hash_field {"hostname":"${hostname}", "tag":"${tag}", "${tag}":100}
346 |
347 | ]
348 | msgs = ['1', '2']
349 | es = emit(config, use_v1, msgs)
350 | es.each_with_index do |(tag, time, record), i|
351 | assert_equal({"hostname" => @hostname, "tag" => @tag, "#{@tag}" => 100}, record['hash_field'])
352 | end
353 | end
354 |
355 | test "array values with placeholders with enable_ruby #{enable_ruby}" do
356 | config = %[
357 | tag tag
358 | enable_ruby #{enable_ruby}
359 |
360 | array_field ["${hostname}", "${tag}"]
361 |
362 | ]
363 | msgs = ['1', '2']
364 | es = emit(config, use_v1, msgs)
365 | es.each_with_index do |(tag, time, record), i|
366 | assert_equal([@hostname, @tag], record['array_field'])
367 | end
368 | end
369 |
370 | test "array and hash values with placeholders with enable_ruby #{enable_ruby}" do
371 | config = %[
372 | tag tag
373 | enable_ruby #{enable_ruby}
374 |
375 | mixed_field [{"tag":"${tag}"}]
376 |
377 | ]
378 | msgs = ['1', '2']
379 | es = emit(config, use_v1, msgs)
380 | es.each_with_index do |(tag, time, record), i|
381 | assert_equal([{"tag" => @tag}], record['mixed_field'])
382 | end
383 | end
384 |
385 | if use_v1 == true
386 | # works with only v1 config
387 | test "keys with placeholders with enable_ruby #{enable_ruby}" do
388 | config = %[
389 | tag tag
390 | enable_ruby #{enable_ruby}
391 | renew_record true
392 |
393 | ${hostname} hostname
394 | foo.${tag} tag
395 |
396 | ]
397 | msgs = ['1', '2']
398 | es = emit(config, use_v1, msgs)
399 | es.each_with_index do |(tag, time, record), i|
400 | assert_equal({@hostname=>'hostname',"foo.#{@tag}"=>'tag'}, record)
401 | end
402 | end
403 | end
404 |
405 | test "disabled autodetectction of value type with enable_ruby #{enable_ruby}" do
406 | config = %[
407 | tag tag
408 | enable_ruby #{enable_ruby}
409 | auto_typecast false
410 |
411 | single ${source}
412 | multiple ${source}${source}
413 | with_prefix prefix-${source}
414 | with_suffix ${source}-suffix
415 |
416 | ]
417 | msgs = [
418 | { "source" => "string" },
419 | { "source" => 123 },
420 | { "source" => [1, 2] },
421 | { "source" => {a:1, b:2} },
422 | { "source" => nil },
423 | ]
424 | expected_results = [
425 | { :single => "string",
426 | :multiple => "stringstring",
427 | :with_prefix => "prefix-string",
428 | :with_suffix => "string-suffix" },
429 | { :single => 123.to_s,
430 | :multiple => "#{123.to_s}#{123.to_s}",
431 | :with_prefix => "prefix-#{123.to_s}",
432 | :with_suffix => "#{123.to_s}-suffix" },
433 | { :single => [1, 2].to_s,
434 | :multiple => "#{[1, 2].to_s}#{[1, 2].to_s}",
435 | :with_prefix => "prefix-#{[1, 2].to_s}",
436 | :with_suffix => "#{[1, 2].to_s}-suffix" },
437 | { :single => {a:1, b:2}.to_s,
438 | :multiple => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
439 | :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
440 | :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
441 | { :single => nil.to_s,
442 | :multiple => "#{nil.to_s}#{nil.to_s}",
443 | :with_prefix => "prefix-#{nil.to_s}",
444 | :with_suffix => "#{nil.to_s}-suffix" },
445 | ]
446 | actual_results = []
447 | es = emit(config, use_v1, msgs)
448 | es.each_with_index do |(tag, time, record), i|
449 | actual_results << {
450 | :single => record["single"],
451 | :multiple => record["multiple"],
452 | :with_prefix => record["with_prefix"],
453 | :with_suffix => record["with_suffix"],
454 | }
455 | end
456 | assert_equal(expected_results, actual_results)
457 | end
458 |
459 | test "enabled autodetectction of value type with enable_ruby #{enable_ruby}" do
460 | config = %[
461 | tag tag
462 | enable_ruby #{enable_ruby}
463 | auto_typecast true
464 |
465 | single ${source}
466 | multiple ${source}${source}
467 | with_prefix prefix-${source}
468 | with_suffix ${source}-suffix
469 |
470 | ]
471 | msgs = [
472 | { "source" => "string" },
473 | { "source" => 123 },
474 | { "source" => [1, 2] },
475 | { "source" => {a:1, b:2} },
476 | { "source" => nil },
477 | ]
478 | expected_results = [
479 | { :single => "string",
480 | :multiple => "stringstring",
481 | :with_prefix => "prefix-string",
482 | :with_suffix => "string-suffix" },
483 | { :single => 123,
484 | :multiple => "#{123.to_s}#{123.to_s}",
485 | :with_prefix => "prefix-#{123.to_s}",
486 | :with_suffix => "#{123.to_s}-suffix" },
487 | { :single => [1, 2],
488 | :multiple => "#{[1, 2].to_s}#{[1, 2].to_s}",
489 | :with_prefix => "prefix-#{[1, 2].to_s}",
490 | :with_suffix => "#{[1, 2].to_s}-suffix" },
491 | { :single => {a:1, b:2},
492 | :multiple => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
493 | :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
494 | :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
495 | { :single => nil,
496 | :multiple => "#{nil.to_s}#{nil.to_s}",
497 | :with_prefix => "prefix-#{nil.to_s}",
498 | :with_suffix => "#{nil.to_s}-suffix" },
499 | ]
500 | actual_results = []
501 | es = emit(config, use_v1, msgs)
502 | es.each_with_index do |(tag, time, record), i|
503 | actual_results << {
504 | :single => record["single"],
505 | :multiple => record["multiple"],
506 | :with_prefix => record["with_prefix"],
507 | :with_suffix => record["with_suffix"],
508 | }
509 | end
510 | assert_equal(expected_results, actual_results)
511 | end
512 |
513 | test %Q[record["key"] with enable_ruby #{enable_ruby}] do
514 | config = %[
515 | tag tag
516 | enable_ruby #{enable_ruby}
517 | auto_typecast true
518 |
519 | _timestamp ${record["@timestamp"]}
520 | _foo_bar ${record["foo.bar"]}
521 |
522 | ]
523 | d = create_driver(config, use_v1)
524 | record = {
525 | "foo.bar" => "foo.bar",
526 | "@timestamp" => 10,
527 | }
528 | es = emit(config, use_v1, [record])
529 | es.each_with_index do |(tag, time, r), i|
530 | assert { r['_timestamp'] == record['@timestamp'] }
531 | assert { r['_foo_bar'] == record['foo.bar'] }
532 | end
533 | end
534 | end
535 |
536 | test 'unknown placeholder (enable_ruby no)' do
537 | config = %[
538 | tag tag
539 | enable_ruby no
540 |
541 | message ${unknown}
542 |
543 | ]
544 | d = create_driver(config, use_v1)
545 | mock(d.instance.log).warn("record_reformer: unknown placeholder `${unknown}` found")
546 | d.run { d.emit({}, @time) }
547 | assert_equal 1, d.emits.size
548 | end
549 |
550 | test 'failed to expand record field (enable_ruby yes)' do
551 | config = %[
552 | tag tag
553 | enable_ruby yes
554 |
555 | message ${unknown['bar']}
556 |
557 | ]
558 | d = create_driver(config, use_v1)
559 | mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
560 | d.run { d.emit({}, @time) }
561 | # emit, but nil value
562 | assert_equal 1, d.emits.size
563 | d.emits.each do |(tag, time, record)|
564 | assert_nil(record['message'])
565 | end
566 | end
567 |
568 | test 'failed to expand tag (enable_ruby yes)' do
569 | config = %[
570 | tag ${unknown['bar']}
571 | enable_ruby yes
572 | ]
573 | d = create_driver(config, use_v1)
574 | mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
575 | d.run { d.emit({}, @time) }
576 | # nil tag message should not be emitted
577 | assert_equal 0, d.emits.size
578 | end
579 |
580 | test 'expand fields starting with @ (enable_ruby no)' do
581 | config = %[
582 | tag tag
583 | enable_ruby no
584 |
585 | foo ${@timestamp}
586 |
587 | ]
588 | d = create_driver(config, use_v1)
589 | message = {"@timestamp" => "foo"}
590 | d.run { d.emit(message, @time) }
591 | d.emits.each do |(tag, time, record)|
592 | assert_equal message["@timestamp"], record["foo"]
593 | end
594 | end
595 |
596 | # https://github.com/sonots/fluent-plugin-record-reformer/issues/35
597 | test 'auto_typecast placeholder containing {} (enable_ruby yes)' do
598 | config = %[
599 | tag tag
600 | enable_ruby yes
601 | auto_typecast yes
602 |
603 | foo ${record.map{|k,v|v}}
604 |
605 | ]
606 | d = create_driver(config, use_v1)
607 | message = {"@timestamp" => "foo"}
608 | d.run { d.emit(message, @time) }
609 | d.emits.each do |(tag, time, record)|
610 | assert_equal [message["@timestamp"]], record["foo"]
611 | end
612 | end
613 |
614 | test 'expand fields starting with @ (enable_ruby yes)' do
615 | config = %[
616 | tag tag
617 | enable_ruby yes
618 |
619 | foo ${__send__("@timestamp")}
620 |
621 | ]
622 | d = create_driver(config, use_v1)
623 | message = {"@timestamp" => "foo"}
624 | d.run { d.emit(message, @time) }
625 | d.emits.each do |(tag, time, record)|
626 | assert_equal message["@timestamp"], record["foo"]
627 | end
628 | end
629 | end
630 |
631 | test "compatibility test (enable_ruby yes) (use_v1 #{use_v1})" do
632 | config = %[
633 | tag tag
634 | enable_ruby yes
635 | auto_typecast yes
636 |
637 | _message prefix-${message}-suffix
638 | _time ${Time.at(time)}
639 | _number ${number == '-' ? 0 : number}
640 | _match ${/0x[0-9a-f]+/.match(hex)[0]}
641 | _timestamp ${__send__("@timestamp")}
642 | _foo_bar ${__send__('foo.bar')}
643 |
644 | ]
645 | d = create_driver(config, use_v1)
646 | record = {
647 | "number" => "-",
648 | "hex" => "0x10",
649 | "foo.bar" => "foo.bar",
650 | "@timestamp" => 10,
651 | "message" => "10",
652 | }
653 | es = emit(config, use_v1, [record])
654 | es.each_with_index do |(tag, time, r), i|
655 | assert { r['_message'] == "prefix-#{record['message']}-suffix" }
656 | assert { r['_time'] == Time.at(@time) }
657 | assert { r['_number'] == 0 }
658 | assert { r['_match'] == record['hex'] }
659 | assert { r['_timestamp'] == record['@timestamp'] }
660 | assert { r['_foo_bar'] == record['foo.bar'] }
661 | end
662 | end
663 | end
664 | end
665 |
--------------------------------------------------------------------------------