├── .github
    └── workflows
    │   └── linux.yml
├── .travis.yml
├── ChangeLog
├── Gemfile
├── README.md
├── Rakefile
├── VERSION
├── fluent-plugin-record-modifier.gemspec
├── lib
    └── fluent
    │   └── plugin
    │       ├── filter_record_modifier.rb
    │       └── out_record_modifier.rb
└── test
    ├── test_filter_record_modifier.rb
    └── test_out_record_modifier.rb


/.github/workflows/linux.yml:
--------------------------------------------------------------------------------
 1 | name: linux
 2 | on:
 3 |   - push
 4 |   - pull_request
 5 | jobs:
 6 |   build:
 7 |     runs-on: ${{ matrix.os }}
 8 |     continue-on-error: ${{ matrix.experimental }}
 9 |     strategy:
10 |       fail-fast: false
11 |       matrix:
12 |         ruby: ['3.1', '3.2', '3.3', '3.4']
13 |         os:
14 |           - ubuntu-latest
15 |         experimental: [false]
16 |         include:
17 |           - ruby-version: head
18 |             os: ubuntu-latest
19 |             experimental: true
20 | 
21 |     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
22 |     steps:
23 |     - uses: actions/checkout@v2
24 |     - uses: ruby/setup-ruby@v1
25 |       with:
26 |         ruby-version: ${{ matrix.ruby }}
27 |     - name: unit testing
28 |       env:
29 |         CI: true
30 |       run: |
31 |         gem install bundler rake
32 |         bundle install --jobs 4 --retry 3
33 |         bundle exec rake test
34 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: ruby
 2 | 
 3 | rvm:
 4 |   - 2.1
 5 |   - 2.2.5
 6 |   - 2.3.1
 7 |   - 2.4.2
 8 |   - ruby-head
 9 | 
10 | branches:
11 |   only:
12 |     - master
13 |     - v0.12
14 | 
15 | gemfile:
16 |   - Gemfile
17 | 
18 | matrix:
19 |   allow_failures:
20 |     - rvm: ruby-head
21 | 
22 | before_install: gem update bundler
23 | script: bundle exec rake test
24 | 


--------------------------------------------------------------------------------
/ChangeLog:
--------------------------------------------------------------------------------
 1 | Release 2.1.1 - 2022/08/25
 2 | 
 3 |   * Improve whitelist_keys performance
 4 |   * Fix documentation for named capture references
 5 | 
 6 | Release 2.1.0 - 2019/12/18
 7 | 
 8 |   * Add <record> keys to whitelist_keys
 9 | 
10 | Release 2.0.1 - 2019/02/06
11 | 
12 |   * Update output plugin to support multi process environment
13 | 
14 | Release 2.0.0 - 2019/01/23
15 | 
16 |   * Update output plugin to follow filter plugin
17 |   * Remove backward compatibility check for 0.5.0 or earlier version
18 | 
19 | Release 1.1.0 - 2018/06/01
20 | 
21 |   * Add `replace` config sections
22 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/33
23 | 
24 | Release 1.0.2 - 2017/12/27
25 | 
26 |   * Add prepare_value parameter
27 |   * Fix non-string object handling
28 |   * Remove deprecated code
29 | 
30 | Release 1.0.1 - 2017/12/16
31 | 
32 |   * Fix fluentd dependency
33 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/28
34 | 
35 | Release 1.0.0 - 2017/12/12
36 | 
37 |   * Use v0.14 API
38 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/14
39 |   * Support frozen object
40 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/26
41 | 
42 | Release 0.5.0 - 2016/08/23
43 | 
44 |   * Remove fluent-plugin-config-placeholder dependency
45 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/13
46 | 
47 | Release 0.4.1 - 2016/02/26
48 | 
49 |   * Set/Convert encoding recursively
50 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/9
51 |   * Add whitelist_keys option
52 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/11
53 | 
54 | Release 0.4.0 - 2016/01/08
55 | 
56 |   * Introduce <record> directive
57 |   * Support ${xxx} placeholders
58 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/6
59 | 
60 | 
61 | Release 0.3.0 - 2015/06/09
62 | 
63 |   * Add record_modifier filter
64 |   * Change fluentd dependency version
65 | 
66 | 
67 | Release 0.2.0 - 2014/09/29
68 | 
69 |   * Update fluent-mixin-config-placeholders to v0.3.0
70 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/6
71 | 
72 | 
73 | Release 0.1.3 - 2014/04/16
74 | 
75 |   * Add remove_keys option to remove needless fields from record
76 |     https://github.com/repeatedly/fluent-plugin-record-modifier#remove_keys
77 | 
78 | 
79 | Release 0.1.2 - 2013/11/22
80 | 
81 |   * Add char_encoding parameter to handle charactor encoding in event record
82 |     https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding
83 | 
84 | 
85 | Release 0.1.1 - 2013/04/24
86 | 
87 |   * Fix SetTagKeyMixin
88 |     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/1
89 | 
90 | Release 0.1.0 - 2013/04/7
91 | 
92 |   * First release
93 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source "http://rubygems.org"
2 | 
3 | gemspec
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Filter plugin to modify event record for [Fluentd](http://fluentd.org)
  2 | 
  3 | Adding arbitary field to event record without customizing existence plugin.
  4 | 
  5 | For example, generated event from *in_tail* doesn't contain "hostname" of running machine.
  6 | In this case, you can use *record_modifier* to add "hostname" field to event record.
  7 | 
  8 | ## Requirements
  9 | 
 10 | | fluent-plugin-record-modifier  | fluentd | ruby |
 11 | |--------------------------------|---------|------|
 12 | | >= 2.0.0 | >= v1.0.0  | >= 2.1 |
 13 | | >= 1.0.0 | >= v0.14.0 | >= 2.1 |
 14 | |  < 1.0.0 | >= v0.12.0 | >= 1.9 |
 15 | 
 16 | ## Installation
 17 | 
 18 | Use RubyGems:
 19 | 
 20 |     fluent-gem install fluent-plugin-record-modifier --no-document
 21 | 
 22 | ## Configuration
 23 | 
 24 | Use `record_modifier` filter.
 25 | 
 26 |     <filter pattern>
 27 |       @type record_modifier
 28 | 
 29 |       <record>
 30 |         gen_host "#{Socket.gethostname}"
 31 |         foo bar
 32 |       </record>
 33 |     </filter>
 34 | 
 35 | If following record is passed:
 36 | 
 37 | ```js
 38 | {"message":"hello world!"}
 39 | ```
 40 | 
 41 | then you got new record like below:
 42 | 
 43 | ```js
 44 | {"message":"hello world!", "gen_host":"oreore-mac.local", "foo":"bar"}
 45 | ```
 46 | 
 47 | You can also use `record_transformer` like `${xxx}` placeholders and access `tag`, `time`, `record` and `tag_parts` values by Ruby code.
 48 | 
 49 |     <filter pattern>
 50 |       @type record_modifier
 51 | 
 52 |       <record>
 53 |         tag ${tag}
 54 |         tag_extract ${tag_parts[0]}-${tag_parts[1]}-foo
 55 |         formatted_time ${Time.at(time).to_s}
 56 |         new_field foo:${record['key1'] + record['dict']['key']} 
 57 |       </record>
 58 |     </filter>
 59 | 
 60 | `record_modifier` is faster than `record_transformer`. See [this comment](https://github.com/repeatedly/fluent-plugin-record-modifier/pull/7#issuecomment-169843012).
 61 | But unlike `record_transformer`, `record_modifier` doesn't support following features for now.
 62 | 
 63 | - tag_suffix and tag_prefix
 64 | - dynamic key placeholder
 65 | 
 66 | ### prepare_value
 67 | 
 68 | Prepare values for filtering. This ruby code is evaluated in `configure` phase and prepared values can be used in `<record>`. Here is an example:
 69 | 
 70 |     <filter pattern>
 71 |       @type record_modifier
 72 |       prepare_value require 'foo'; @foo = Foo.new
 73 |       <record>
 74 |         key ${@foo.method1}
 75 |       </record>
 76 |     </filter>
 77 | 
 78 | This feature is useful for using external library.
 79 | 
 80 | ### char_encoding
 81 | 
 82 | Fluentd including some plugins treats logs as a BINARY by default to forward.
 83 | But a user sometimes wants to process logs depends on their requirements, e.g. handling char encoding correctly.
 84 | 
 85 | `char_encoding` parameter is useful for this case.
 86 | 
 87 | ```conf
 88 | <filter pattern>
 89 |   @type record_modifier
 90 | 
 91 |   # set UTF-8 encoding information to string.
 92 |   char_encoding utf-8
 93 | 
 94 |   # change char encoding from 'UTF-8' to 'EUC-JP'
 95 |   char_encoding utf-8:euc-jp
 96 | </filter>
 97 | ```
 98 | 
 99 | In `char_encoding from:to` case, it replaces invalid character with safe character.
100 | 
101 | ### remove_keys
102 | 
103 | The logs include needless record keys in some cases.
104 | You can remove it by using `remove_keys` parameter.
105 | 
106 | ```conf
107 | <filter pattern>
108 |   @type record_modifier
109 | 
110 |   # remove key1 and key2 keys from record
111 |   remove_keys key1,key2
112 | </filter>
113 | ```
114 | 
115 | If following record is passed:
116 | 
117 | ```js
118 | {"key1":"hoge", "key2":"foo", "key3":"bar"}
119 | ```
120 | 
121 | then you got new record like below:
122 | 
123 | ```js
124 | {"key3":"bar"}
125 | ```
126 | 
127 | Since v2.2.0, `remove_keys` supports nested key delete via [`record_accessor` syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor).
128 | 
129 | ### whitelist_keys
130 | 
131 | If you want to handle the set of explicitly specified keys, you can use `whitelist_keys` of this plugin. It's exclusive with `remove_keys`.
132 | 
133 | ```conf
134 | <filter pattern>
135 |   @type record_modifier
136 | 
137 |   # remove all keys except for key1 and key2
138 |   whitelist_keys key1,key2
139 | </filter>
140 | ```
141 | 
142 | If following record is passed:
143 | 
144 | ```js
145 | {"key1":"hoge", "key2":"foo", "key3":"bar"}
146 | ```
147 | 
148 | then you got new record like below:
149 | 
150 | ```js
151 | {"key1":"hoge", "key2":"foo"}
152 | ```
153 | 
154 | ### replace_keys_value
155 | 
156 | If you want to replace specific value for keys you can use `replace` section.
157 | 
158 | ```conf
159 | <filter pattern>
160 |   @type record_modifier
161 | 
162 |   # replace key key1
163 |   <replace>
164 |     # your key name
165 |     key key1
166 |     # your regexp
167 |     expression /^(?<start>.+).{2}(?<end>.+)$/
168 |     # replace string
169 |     replace \k<start>ors\k<end>
170 |   </replace>
171 |   # replace key key2
172 |   <replace>
173 |     # your key name
174 |     key key2
175 |     # your regexp
176 |     expression /^(.{1}).{2}(.{1})$/
177 |     # replace string
178 |     replace \1ors\2
179 |   </replace>
180 | </filter>
181 | ```
182 | 
183 | If following record is passed:
184 | 
185 | ```js
186 | {"key1":"hoge", "key2":"hoge", "key3":"bar"}
187 | ```
188 | 
189 | then you got new record like below:
190 | 
191 | ```js
192 | {"key1":"horse", "key2":"horse", "key3":"bar"}
193 | ```
194 | 
195 | 
196 | ### Ruby code trick for complex logic
197 | 
198 | If you need own complex logic in filter, writing filter plugin is better. But if you don't want to write new plugin, you can use temporal key trick like below:
199 | 
200 | ```
201 | <filter reform.**>
202 |   @type record_modifier
203 |   remove_keys _dummy_
204 |   <record>
205 |     _dummy_ ${if record.has_key?('foo'); record['bar'] = 'Hi!'; end; nil}
206 |   </record>
207 | </filter>
208 | ```
209 | 
210 | ### record_modifier output
211 | 
212 | Output plugin version of `record_modifier` filter. If you want to process events and change tag at the same time, this plugin is useful.
213 | 
214 |     <match pattern>
215 |       @type record_modifier
216 |       tag foo.${record["field1"]}
217 | 
218 |       <record>
219 |         gen_host "#{Socket.gethostname}"
220 |         foo bar
221 |       </record>
222 |     </match>
223 | 
224 | ## Copyright
225 | 
226 | <table>
227 |   <tr>
228 |     <td>Author</td><td>Masahiro Nakagawa <repeatedly@gmail.com></td>
229 |   </tr>
230 |   <tr>
231 |     <td>Copyright</td><td>Copyright (c) 2013- Masahiro Nakagawa</td>
232 |   </tr>
233 |   <tr>
234 |     <td>License</td><td>MIT License</td>
235 |   </tr>
236 | </table>
237 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | require 'bundler'
 2 | require 'bundler/gem_tasks'
 3 | 
 4 | require 'rake/testtask'
 5 | 
 6 | Rake::TestTask.new(:test) do |test|
 7 |   test.libs << 'lib' << 'test'
 8 |   test.test_files = FileList['test/test_*.rb']
 9 |   test.verbose = true
10 | end
11 | 
12 | task :default => [:build]
13 | 
14 | 


--------------------------------------------------------------------------------
/VERSION:
--------------------------------------------------------------------------------
1 | 2.2.0


--------------------------------------------------------------------------------
/fluent-plugin-record-modifier.gemspec:
--------------------------------------------------------------------------------
 1 | # encoding: utf-8
 2 | $:.push File.expand_path('../lib', __FILE__)
 3 | 
 4 | Gem::Specification.new do |gem|
 5 |   gem.name        = "fluent-plugin-record-modifier"
 6 |   gem.description = "Filter plugin for modifying event record"
 7 |   gem.homepage    = "https://github.com/repeatedly/fluent-plugin-record-modifier"
 8 |   gem.summary     = gem.description
 9 |   gem.version     = File.read("VERSION").strip
10 |   gem.authors     = ["Masahiro Nakagawa"]
11 |   gem.email       = "repeatedly@gmail.com"
12 |   #gem.platform    = Gem::Platform::RUBY
13 |   gem.license     = 'MIT'
14 |   gem.files       = `git ls-files`.split("\n")
15 |   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
16 |   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17 |   gem.require_paths = ['lib']
18 | 
19 |   gem.add_dependency "fluentd", [">= 1.1", "< 2"]
20 |   gem.add_development_dependency "rake", ">= 0.9.2"
21 |   gem.add_development_dependency("test-unit", ["~> 3.4.0"])
22 | end
23 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/filter_record_modifier.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/plugin/filter'
  2 | 
  3 | module Fluent
  4 |   class Plugin::RecordModifierFilter < Plugin::Filter
  5 |     Fluent::Plugin.register_filter('record_modifier', self)
  6 | 
  7 |     helpers :record_accessor
  8 | 
  9 |     config_param :prepare_value, :string, default: nil,
 10 |                  desc: <<-DESC
 11 | Prepare values for filtering in configure phase. Prepared values can be used in <record>.
 12 | You can write any ruby code.
 13 | DESC
 14 |     config_param :char_encoding, :string, default: nil,
 15 |                  desc: <<-DESC
 16 | Fluentd including some plugins treats the logs as a BINARY by default to forward.
 17 | But an user sometimes processes the logs depends on their requirements,
 18 | e.g. handling char encoding correctly.
 19 | In more detail, please refer this section:
 20 | https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
 21 | DESC
 22 |     config_param :remove_keys, :array, default: nil,
 23 |                  desc: <<-DESC
 24 | The logs include needless record keys in some cases.
 25 | You can remove it by using `remove_keys` parameter.
 26 | This option is exclusive with `whitelist_keys`.
 27 | DESC
 28 | 
 29 |     config_param :whitelist_keys, :string, default: nil,
 30 |                  desc: <<-DESC
 31 | Specify `whitelist_keys` to remove all unexpected keys and values from events.
 32 | Modified events will have only specified keys (if exist in original events).
 33 | This option is exclusive with `remove_keys`.
 34 | DESC
 35 | 
 36 |     config_section :replace, param_name: :replaces, multi: true do
 37 |       desc "The field name to which the regular expression is applied"
 38 |       config_param :key, :string
 39 |       desc "The regular expression"
 40 |       config_param :expression do |value|
 41 |         if value.start_with?("/") && value.end_with?("/")
 42 |           Regexp.compile(value[1..-2])
 43 |         else
 44 |           $log.warn "You should use \"pattern /#{value}/\" instead of \"pattern #{value}\""
 45 |           Regexp.compile(value)
 46 |         end
 47 |       end
 48 |       desc "The replacement string"
 49 |       config_param :replace, :string
 50 |     end
 51 | 
 52 |     def configure(conf)
 53 |       super
 54 | 
 55 |       @map = {}
 56 |       @to_enc = nil
 57 |       if @char_encoding
 58 |         from, to = @char_encoding.split(':', 2)
 59 |         @from_enc = Encoding.find(from)
 60 |         @to_enc = Encoding.find(to) if to
 61 | 
 62 |         m = if @to_enc
 63 |               method(:convert_encoding)
 64 |             else
 65 |               method(:set_encoding)
 66 |             end
 67 | 
 68 |         (class << self; self; end).module_eval do
 69 |           define_method(:change_encoding, m)
 70 |         end
 71 |       end
 72 | 
 73 |       @has_tag_parts = false
 74 |       conf.elements.select { |element| element.name == 'record' }.each do |element|
 75 |         element.each_pair do |k, v|
 76 |           element.has_key?(k) # to suppress unread configuration warning
 77 |           @has_tag_parts = true if v.include?('tag_parts')
 78 |           @map[k] = DynamicExpander.new(k, v, @prepare_value)
 79 |         end
 80 |       end
 81 | 
 82 |       if @remove_keys and @whitelist_keys
 83 |         raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
 84 |       elsif @remove_keys
 85 |         @remove_keys = @remove_keys.map { |key|
 86 |           record_accessor_create(key)
 87 |         }
 88 |       elsif @whitelist_keys
 89 |         @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
 90 |         @whitelist_keys.concat(@map.keys).uniq!
 91 |       end
 92 | 
 93 |       # Collect DynamicExpander related garbage instructions
 94 |       GC.start
 95 |     end
 96 | 
 97 |     def filter(tag, time, record)
 98 |       tag_parts = @has_tag_parts ? tag.split('.') : nil
 99 | 
100 |       @map.each_pair { |k, v|
101 |         record[k] = v.expand(tag, time, record, tag_parts)
102 |       }
103 | 
104 |       if @remove_keys
105 |         @remove_keys.each { |ra|
106 |           ra.delete(record)
107 |         }
108 |       elsif @whitelist_keys
109 |         modified = {}
110 |         @whitelist_keys.each do |key|
111 |           modified[key] = record[key] if record.has_key?(key)
112 |         end
113 |         record = modified
114 |       end
115 | 
116 |       unless @replaces.empty?
117 |         @replaces.each { |replace|
118 |           target_key = replace.key
119 |           if record.include?(target_key) && replace.expression.match(record[target_key])
120 |             record[target_key] = record[target_key].gsub(replace.expression, replace.replace)
121 |           end
122 |         }
123 |       end
124 | 
125 |       record = change_encoding(record) if @char_encoding
126 |       record
127 |     end
128 | 
129 |     private
130 | 
131 |     def set_encoding(value)
132 |       if value.is_a?(String)
133 |         value.force_encoding(@from_enc)
134 |       elsif value.is_a?(Hash)
135 |         value.each_pair { |k, v|
136 |           if v.frozen? && v.is_a?(String)
137 |             value[k] = set_encoding(v.dup)
138 |           else
139 |             set_encoding(v)
140 |           end
141 |         }
142 |       elsif value.is_a?(Array)
143 |         value.each { |v| set_encoding(v) }
144 |       else
145 |         value
146 |       end
147 |     end
148 | 
149 |     def convert_encoding(value)
150 |       if value.is_a?(String)
151 |         value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
152 |         value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
153 |       elsif value.is_a?(Hash)
154 |         value.each_pair { |k, v|
155 |           if v.frozen? && v.is_a?(String)
156 |             value[k] = convert_encoding(v.dup)
157 |           else
158 |             convert_encoding(v)
159 |           end
160 |         }
161 |       elsif value.is_a?(Array)
162 |         value.each { |v| convert_encoding(v) }
163 |       else
164 |         value
165 |       end
166 |     end
167 | 
168 |     class DynamicExpander
169 |       def initialize(param_key, param_value, prepare_value)
170 |         if param_value.include?('${')
171 |           __str_eval_code__ = parse_parameter(param_value)
172 | 
173 |           # Use class_eval with string instead of define_method for performance.
174 |           # It can't share instructions but this is 2x+ faster than define_method in filter case.
175 |           # Refer: http://tenderlovemaking.com/2013/03/03/dynamic_method_definitions.html
176 |           (class << self; self; end).class_eval <<-EORUBY,  __FILE__, __LINE__ + 1
177 |             def expand(tag, time, record, tag_parts)
178 |               #{__str_eval_code__}
179 |             end
180 |           EORUBY
181 |         else
182 |           @param_value = param_value
183 |         end
184 | 
185 |         begin
186 |           eval prepare_value if prepare_value
187 |         rescue SyntaxError
188 |           raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
189 |         end
190 | 
191 |         begin
192 |           # check eval genarates wrong code or not
193 |           expand(nil, nil, nil, nil)
194 |         rescue SyntaxError
195 |           raise ConfigError, "Pass invalid syntax parameter : key = #{param_key}, value = #{param_value}"
196 |         rescue
197 |           # Ignore other runtime errors
198 |         end
199 |       end
200 | 
201 |       # Default implementation for fixed value. This is overwritten when parameter contains '${xxx}' placeholder
202 |       def expand(tag, time, record, tag_parts)
203 |         @param_value
204 |       end
205 | 
206 |       private
207 | 
208 |       def parse_parameter(value)
209 |         num_placeholders = value.scan('${').size
210 |         if num_placeholders == 1
211 |           if value.start_with?('${') && value.end_with?('}')
212 |             return value[2..-2]
213 |           else
214 |             "\"#{value.gsub('${', '#{')}\""
215 |           end
216 |         else
217 |           "\"#{value.gsub('${', '#{')}\""
218 |         end
219 |       end
220 |     end
221 |   end
222 | end
223 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_record_modifier.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/plugin/output'
  2 | 
  3 | module Fluent
  4 |   class Plugin::RecordModifierOutput < Plugin::Output
  5 |     Fluent::Plugin.register_output('record_modifier', self)
  6 | 
  7 |     helpers :event_emitter
  8 | 
  9 |     config_param :tag, :string,
 10 |                  desc: "The output record tag name."
 11 | 
 12 |     config_param :prepare_value, :string, default: nil,
 13 |                  desc: <<-DESC
 14 | Prepare values for filtering in configure phase. Prepared values can be used in <record>.
 15 | You can write any ruby code.
 16 | DESC
 17 |     config_param :char_encoding, :string, default: nil,
 18 |                  desc: <<-DESC
 19 | Fluentd including some plugins treats the logs as a BINARY by default to forward.
 20 | But an user sometimes processes the logs depends on their requirements,
 21 | e.g. handling char encoding correctly.
 22 | In more detail, please refer this section:
 23 | https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
 24 | DESC
 25 |     config_param :remove_keys, :string, default: nil,
 26 |                  desc: <<-DESC
 27 | The logs include needless record keys in some cases.
 28 | You can remove it by using `remove_keys` parameter.
 29 | This option is exclusive with `whitelist_keys`.
 30 | DESC
 31 | 
 32 |     config_param :whitelist_keys, :string, default: nil,
 33 |                  desc: <<-DESC
 34 | Specify `whitelist_keys` to remove all unexpected keys and values from events.
 35 | Modified events will have only specified keys (if exist in original events).
 36 | This option is exclusive with `remove_keys`.
 37 | DESC
 38 | 
 39 |     config_section :replace, param_name: :replaces, multi: true do
 40 |       desc "The field name to which the regular expression is applied"
 41 |       config_param :key, :string
 42 |       desc "The regular expression"
 43 |       config_param :expression do |value|
 44 |         if value.start_with?("/") && value.end_with?("/")
 45 |           Regexp.compile(value[1..-2])
 46 |         else
 47 |           $log.warn "You should use \"pattern /#{value}/\" instead of \"pattern #{value}\""
 48 |           Regexp.compile(value)
 49 |         end
 50 |       end
 51 |       desc "The replacement string"
 52 |       config_param :replace, :string
 53 |     end
 54 | 
 55 |     def configure(conf)
 56 |       super
 57 | 
 58 |       @map = {}
 59 |       @to_enc = nil
 60 |       if @char_encoding
 61 |         from, to = @char_encoding.split(':', 2)
 62 |         @from_enc = Encoding.find(from)
 63 |         @to_enc = Encoding.find(to) if to
 64 | 
 65 |         m = if @to_enc
 66 |               method(:convert_encoding)
 67 |             else
 68 |               method(:set_encoding)
 69 |             end
 70 | 
 71 |         (class << self; self; end).module_eval do
 72 |           define_method(:change_encoding, m)
 73 |         end
 74 |       end
 75 | 
 76 |       @has_tag_parts = false
 77 |       conf.elements.select { |element| element.name == 'record' }.each do |element|
 78 |         element.each_pair do |k, v|
 79 |           element.has_key?(k) # to suppress unread configuration warning
 80 |           @has_tag_parts = true if v.include?('tag_parts')
 81 |           @map[k] = DynamicExpander.new(k, v, @prepare_value)
 82 |         end
 83 |       end
 84 | 
 85 |       if @remove_keys and @whitelist_keys
 86 |         raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
 87 |       elsif @remove_keys
 88 |         @remove_keys = @remove_keys.split(',').map(&:strip)
 89 |       elsif @whitelist_keys
 90 |         @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
 91 |         @whitelist_keys.concat(@map.keys).uniq!
 92 |       end
 93 | 
 94 |       @has_tag_parts = true if @tag.include?('tag_parts')
 95 |       @tag_ex = DynamicExpander.new('tag', @tag, @prepare_value)
 96 | 
 97 |         # Collect DynamicExpander related garbage instructions
 98 |       GC.start
 99 |     end
100 | 
101 |     def multi_workers_ready?
102 |       true
103 |     end
104 | 
105 |     def process(tag, es)
106 |       tag_parts = @has_tag_parts ? tag.split('.') : nil
107 |       if @tag_ex.param_value.nil?
108 |         result = {}
109 |         es.each { |time, record|
110 |           new_record = modify_record(tag, time, record, tag_parts)
111 |           new_tag = @tag_ex.expand(tag, time, new_record, tag_parts)
112 |           result[new_tag] ||= MultiEventStream.new
113 |           result[new_tag].add(time, new_record)
114 |         }
115 |         result.each { |tag, stream|
116 |           router.emit_stream(tag, stream)
117 |         }
118 |       else
119 |         stream = MultiEventStream.new
120 |         es.each { |time, record|
121 |           new_record = modify_record(tag, time, record, tag_parts)
122 |           stream.add(time, new_record)
123 |         }
124 |         router.emit_stream(@tag, stream)
125 |       end
126 |     end
127 | 
128 |     private
129 | 
130 |     def modify_record(tag, time, record, tag_parts)
131 |       @map.each_pair { |k, v|
132 |         record[k] = v.expand(tag, time, record, tag_parts)
133 |       }
134 | 
135 |       if @remove_keys
136 |         @remove_keys.each { |v|
137 |           record.delete(v)
138 |         }
139 |       elsif @whitelist_keys
140 |         modified = {}
141 |         @whitelist_keys.each do |key|
142 |           modified[key] = record[key] if record.has_key?(key)
143 |         end
144 |         record = modified
145 |       end
146 | 
147 |       unless @replaces.empty?
148 |         @replaces.each { |replace|
149 |           target_key = replace.key
150 |           if record.include?(target_key) && replace.expression.match(record[target_key])
151 |             record[target_key] = record[target_key].gsub(replace.expression, replace.replace)
152 |           end
153 |         }
154 |       end
155 | 
156 |       record = change_encoding(record) if @char_encoding
157 |       record
158 |     end
159 | 
160 |     def set_encoding(value)
161 |       if value.is_a?(String)
162 |         value.force_encoding(@from_enc)
163 |       elsif value.is_a?(Hash)
164 |         value.each_pair { |k, v|
165 |           if v.frozen? && v.is_a?(String)
166 |             value[k] = set_encoding(v.dup)
167 |           else
168 |             set_encoding(v)
169 |           end
170 |         }
171 |       elsif value.is_a?(Array)
172 |         value.each { |v| set_encoding(v) }
173 |       else
174 |         value
175 |       end
176 |     end
177 | 
178 |     def convert_encoding(value)
179 |       if value.is_a?(String)
180 |         value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
181 |         value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
182 |       elsif value.is_a?(Hash)
183 |         value.each_pair { |k, v|
184 |           if v.frozen? && v.is_a?(String)
185 |             value[k] = convert_encoding(v.dup)
186 |           else
187 |             convert_encoding(v)
188 |           end
189 |         }
190 |       elsif value.is_a?(Array)
191 |         value.each { |v| convert_encoding(v) }
192 |       else
193 |         value
194 |       end
195 |     end
196 | 
197 |     class DynamicExpander
198 |       attr_reader :param_value
199 | 
200 |       def initialize(param_key, param_value, prepare_value)
201 |         if param_value.include?('${')
202 |           __str_eval_code__ = parse_parameter(param_value)
203 | 
204 |           # Use class_eval with string instead of define_method for performance.
205 |           # It can't share instructions but this is 2x+ faster than define_method in filter case.
206 |           # Refer: http://tenderlovemaking.com/2013/03/03/dynamic_method_definitions.html
207 |           (class << self; self; end).class_eval <<-EORUBY,  __FILE__, __LINE__ + 1
208 |             def expand(tag, time, record, tag_parts)
209 |               #{__str_eval_code__}
210 |             end
211 |           EORUBY
212 |         else
213 |           @param_value = param_value
214 |         end
215 | 
216 |         begin
217 |           eval prepare_value if prepare_value
218 |         rescue SyntaxError
219 |           raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
220 |         end
221 | 
222 |         begin
223 |           # check eval genarates wrong code or not
224 |           expand(nil, nil, nil, nil)
225 |         rescue SyntaxError
226 |           raise ConfigError, "Pass invalid syntax parameter : key = #{param_key}, value = #{param_value}"
227 |         rescue
228 |           # Ignore other runtime errors
229 |         end
230 |       end
231 | 
232 |       # Default implementation for fixed value. This is overwritten when parameter contains '${xxx}' placeholder
233 |       def expand(tag, time, record, tag_parts)
234 |         @param_value
235 |       end
236 | 
237 |       private
238 | 
239 |       def parse_parameter(value)
240 |         num_placeholders = value.scan('${').size
241 |         if num_placeholders == 1
242 |           if value.start_with?('${') && value.end_with?('}')
243 |             return value[2..-2]
244 |           else
245 |             "\"#{value.gsub('${', '#{')}\""
246 |           end
247 |         else
248 |           "\"#{value.gsub('${', '#{')}\""
249 |         end
250 |       end
251 |     end
252 |   end
253 | end
254 | 


--------------------------------------------------------------------------------
/test/test_filter_record_modifier.rb:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | require 'fluent/test'
  3 | require 'fluent/test/driver/filter'
  4 | require 'fluent/plugin/filter_record_modifier'
  5 | require 'test/unit'
  6 | 
  7 | class RecordModifierFilterTest < Test::Unit::TestCase
  8 |   def setup
  9 |     Fluent::Test.setup
 10 |     @tag = 'test.tag'
 11 |   end
 12 | 
 13 |   CONFIG = %q!
 14 |     remove_keys hoge
 15 | 
 16 |     <record>
 17 |       gen_host "#{Socket.gethostname}"
 18 |       foo bar
 19 |       included_tag ${tag}
 20 |       tag_wrap -${tag_parts[0]}-${tag_parts[1]}-
 21 |     </record>
 22 |   !
 23 | 
 24 |   def create_driver(conf = CONFIG)
 25 |     Fluent::Test::Driver::Filter.new(Fluent::Plugin::RecordModifierFilter).configure(conf)
 26 |   end
 27 | 
 28 |   def get_hostname
 29 |     require 'socket'
 30 |     Socket.gethostname.chomp
 31 |   end
 32 | 
 33 |   def test_configure
 34 |     d = create_driver
 35 |     map = d.instance.instance_variable_get(:@map)
 36 | 
 37 |     map.each_pair { |k, v|
 38 |       assert v.is_a?(Fluent::Plugin::RecordModifierFilter::DynamicExpander)
 39 |     }
 40 |   end
 41 | 
 42 |   def test_format
 43 |     d = create_driver
 44 | 
 45 |     d.run(default_tag: @tag) do
 46 |       d.feed("a" => 1)
 47 |       d.feed("a" => 2)
 48 |     end
 49 | 
 50 |     mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => @tag, 'tag_wrap' => "-#{@tag.split('.')[0]}-#{@tag.split('.')[1]}-"}
 51 |     assert_equal [
 52 |       {"a" => 1}.merge(mapped),
 53 |       {"a" => 2}.merge(mapped),
 54 |     ], d.filtered.map { |e| e.last }
 55 |   end
 56 | 
 57 |   def test_set_char_encoding
 58 |     d = create_driver %[
 59 |       char_encoding utf-8
 60 |     ]
 61 | 
 62 |     d.run(default_tag: @tag) do
 63 |       d.feed("k" => 'v'.force_encoding('BINARY'))
 64 |       d.feed("k" => %w(v ビ).map{|v| v.force_encoding('BINARY')})
 65 |       d.feed("k" => {"l" => 'ビ'.force_encoding('BINARY')})
 66 |     end
 67 | 
 68 |     assert_equal [
 69 |       {"k" => 'v'.force_encoding('UTF-8')},
 70 |       {"k" => %w(v ビ).map{|v| v.force_encoding('UTF-8')}},
 71 |       {"k" => {"l" => 'ビ'.force_encoding('UTF-8')}},
 72 |     ], d.filtered.map { |e| e.last }
 73 |   end
 74 | 
 75 |   def test_convert_char_encoding
 76 |     d = create_driver %[
 77 |       char_encoding utf-8:cp932
 78 |     ]
 79 | 
 80 |     d.run(default_tag: @tag) do
 81 |       d.feed("k" => 'v'.force_encoding('utf-8'))
 82 |       d.feed("k" => %w(v ビ).map{|v| v.force_encoding('utf-8')})
 83 |       d.feed("k" => {"l" => 'ビ'.force_encoding('utf-8')})
 84 |     end
 85 | 
 86 |     assert_equal [
 87 |       {"k" => 'v'.force_encoding('cp932')},
 88 |       {"k" => %w(v ビ).map{|v| v.encode!('cp932')}},
 89 |       {"k" => {"l" => 'ビ'.encode!('cp932')}},
 90 |     ], d.filtered.map { |e| e.last }
 91 |   end
 92 | 
 93 |   def test_remove_one_key
 94 |     d = create_driver %[
 95 |       remove_keys k1
 96 |     ]
 97 | 
 98 |     d.run(default_tag: @tag) do
 99 |       d.feed("k1" => 'v', "k2" => 'v')
100 |     end
101 | 
102 |     assert_equal [{"k2" => 'v'}], d.filtered.map { |e| e.last }
103 |   end
104 | 
105 |   def test_remove_multiple_keys
106 |     d = create_driver %[
107 |       remove_keys k1, k2, k3
108 |     ]
109 | 
110 |     d.run(default_tag: @tag) do
111 |       d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v'})
112 |     end
113 | 
114 |     assert_equal [{"k4" => 'v'}], d.filtered.map { |e| e.last }
115 |   end
116 | 
117 |   def test_remove_nested_keys
118 |     d = create_driver %[
119 |       remove_keys k1, $.kubernetes.test
120 |     ]
121 | 
122 |     d.run(default_tag: @tag) do
123 |       d.feed("k1" => 'v', "kubernetes" => {"test" => 'v', "prod" => 'v'}, "k2" => 'v')
124 |     end
125 | 
126 |     assert_equal [{"kubernetes" => {"prod" => 'v'}, "k2" => 'v'}], d.filtered.map { |e| e.last }
127 |   end
128 | 
129 |   def test_remove_non_whitelist_keys
130 |     d = create_driver %[
131 |       <record>
132 |         foo bar
133 |       </record>
134 |       whitelist_keys k1, k2, k3
135 |     ]
136 | 
137 |     d.run(default_tag: @tag) do
138 |       d.feed("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
139 |     end
140 | 
141 |     assert_equal [{"k1" => 'v', "k2" => 'v', 'foo' => 'bar'}], d.filtered.map(&:last)
142 |   end
143 | 
144 |   def test_prepare_values
145 |     d = create_driver %[
146 |       prepare_value @foo = 'foo'
147 |       <record>
148 |         test_key ${@foo}
149 |       </record>
150 |     ]
151 | 
152 |     d.run(default_tag: @tag) do
153 |       d.feed("k1" => 'v')
154 |     end
155 | 
156 |     assert_equal [{"k1" => 'v', "test_key" => 'foo'}], d.filtered.map(&:last)
157 |   end
158 | 
159 |   def test_replace_values
160 |     d = create_driver <<'CONFIG'
161 |         <replace>
162 |           key k1
163 |           expression /^(?<start>.+).{2}(?<end>.+)$/
164 |           replace \k<start>ors\k<end>
165 |         </replace>
166 |         <replace>
167 |           key k2
168 |           expression /^(.{1}).{2}(.{1})$/
169 |           replace \1ors\2
170 |         </replace>
171 | CONFIG
172 | 
173 |     d.run(default_tag: @tag) do
174 |       d.feed("k1" => 'hoge', "k2" => 'hoge', "k3" => 'bar')
175 |     end
176 | 
177 |     assert_equal [{"k1" => 'horse', "k2" => 'horse', "k3" => 'bar'}], d.filtered.map(&:last)
178 |   end
179 | 
180 |   def test_does_not_replace
181 |     d = create_driver <<'CONFIG'
182 |         <replace>
183 |           key k1
184 |           expression /^(?<start>.+).{2}(?<end>.+)$/
185 |           replace \k<start>ors\k<end>
186 |         </replace>
187 | CONFIG
188 | 
189 |     d.run(default_tag: @tag) do
190 |       d.feed("k1" => 'hog')
191 |     end
192 | 
193 |     assert_equal [{"k1" => 'hog'}], d.filtered.map(&:last)
194 |   end
195 | 
196 |   sub_test_case 'frozen check' do
197 |     def test_set_char_encoding
198 |       d = create_driver %[
199 |         char_encoding utf-8
200 |       ]
201 | 
202 |       d.run(default_tag: @tag) do
203 |         d.feed("k" => 'v'.force_encoding('BINARY').freeze, 'n' => 1)
204 |         d.feed("k" => {"l" => 'v'.force_encoding('BINARY').freeze, 'n' => 1})
205 |       end
206 | 
207 |       assert_equal [
208 |         {"k" => 'v'.force_encoding('UTF-8'), 'n' => 1},
209 |         {"k" => {"l" => 'v'.force_encoding('UTF-8'), 'n' => 1}},
210 |       ], d.filtered.map { |e| e.last }
211 |     end
212 | 
213 |     def test_convert_char_encoding
214 |       d = create_driver %[
215 |         char_encoding utf-8:cp932
216 |       ]
217 | 
218 |       d.run(default_tag: @tag) do
219 |         d.feed("k" => 'v'.force_encoding('utf-8').freeze, 'n' => 1)
220 |         d.feed("k" => {"l" => 'v'.force_encoding('utf-8').freeze, 'n' => 1})
221 |       end
222 | 
223 |       assert_equal [
224 |         {"k" => 'v'.force_encoding('cp932'), 'n' => 1},
225 |         {"k" => {"l" => 'v'.force_encoding('cp932'), 'n' => 1}},
226 |       ], d.filtered.map { |e| e.last }
227 |     end
228 |   end
229 | end
230 | 


--------------------------------------------------------------------------------
/test/test_out_record_modifier.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/test/driver/output'
  2 | require 'fluent/plugin/out_record_modifier'
  3 | 
  4 | 
  5 | class RecordModifierOutputTest < Test::Unit::TestCase
  6 |   def setup
  7 |     Fluent::Test.setup
  8 |   end
  9 | 
 10 |   CONFIG = %q!
 11 |     tag foo.filtered
 12 |     <record>
 13 |       gen_host "#{Socket.gethostname}"
 14 |       foo bar
 15 |     </record>
 16 |     remove_keys hoge
 17 |   !
 18 | 
 19 |   def create_driver(conf = CONFIG)
 20 |     Fluent::Test::Driver::Output.new(Fluent::Plugin::RecordModifierOutput).configure(conf)
 21 |   end
 22 | 
 23 |   def get_hostname
 24 |     require 'socket'
 25 |     Socket.gethostname.chomp
 26 |   end
 27 | 
 28 |   def test_configure
 29 |     d = create_driver
 30 |     map = d.instance.instance_variable_get(:@map)
 31 | 
 32 |     assert_equal get_hostname, map['gen_host'].param_value
 33 |     assert_equal 'bar', map['foo'].param_value
 34 |   end
 35 | 
 36 |   def test_format
 37 |     d = create_driver
 38 | 
 39 |     d.run(default_tag: 'test_tag') do
 40 |       d.feed({"a" => 1})
 41 |       d.feed({"a" => 2})
 42 |     end
 43 | 
 44 |     mapped = {'gen_host' => get_hostname, 'foo' => 'bar'}
 45 |     assert_equal [
 46 |       {"a" => 1}.merge(mapped),
 47 |       {"a" => 2}.merge(mapped),
 48 |     ], d.events.map { |e| e.last }
 49 |   end
 50 | 
 51 |   def test_dynamic_tag_with_tag
 52 |     d = create_driver %[
 53 |       tag foo.${tag}
 54 |     ]
 55 | 
 56 |     d.run(default_tag: 'test_tag') do
 57 |       d.feed({"k" => 'v'})
 58 |     end
 59 | 
 60 |     assert_equal 'foo.test_tag', d.events.first.first
 61 |   end
 62 | 
 63 |   def test_dynamic_tag_with_record_field
 64 |     d = create_driver %[
 65 |       tag foo.${record["k"]}
 66 |     ]
 67 | 
 68 |     d.run(default_tag: 'test_tag') do
 69 |       d.feed({"k" => 'v'})
 70 |     end
 71 | 
 72 |     assert_equal 'foo.v', d.events.first.first
 73 |   end
 74 | 
 75 |   def test_set_char_encoding
 76 |     d = create_driver %[
 77 |       tag foo.filtered
 78 |       char_encoding utf-8
 79 |     ]
 80 | 
 81 |     d.run(default_tag: 'test_tag') do
 82 |       d.feed({"k" => 'v'.force_encoding('BINARY')})
 83 |     end
 84 | 
 85 |     assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.events.map { |e| e.last }
 86 |   end
 87 | 
 88 |   def test_convert_char_encoding
 89 |     d = create_driver %[
 90 |       tag foo.filtered
 91 |       char_encoding utf-8:cp932
 92 |     ]
 93 | 
 94 |     d.run(default_tag: 'test_tag') do
 95 |       d.feed("k" => 'v'.force_encoding('utf-8'))
 96 |     end
 97 | 
 98 |     assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.events.map { |e| e.last }
 99 |   end
100 | 
101 |   def test_remove_one_key
102 |     d = create_driver %[
103 |       tag foo.filtered
104 |       remove_keys k1
105 |     ]
106 | 
107 |     d.run(default_tag: 'test_tag') do
108 |       d.feed({"k1" => 'v', "k2" => 'v'})
109 |     end
110 | 
111 |     assert_equal [{"k2" => 'v'}], d.events.map { |e| e.last }
112 |   end
113 | 
114 |   def test_remove_multiple_keys
115 |     d = create_driver %[
116 |       tag foo.filtered
117 |       remove_keys k1, k2, k3
118 |     ]
119 | 
120 |     d.run(default_tag: 'test_tag') do
121 |       d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v'})
122 |     end
123 | 
124 |     assert_equal [{"k4" => 'v'}], d.events.map { |e| e.last }
125 |   end
126 | 
127 |   def test_remove_non_whitelist_keys
128 |     d = create_driver %[
129 |       tag foo.filtered
130 |       <record>
131 |         foo bar
132 |       </record>
133 |       whitelist_keys k1, k2, k3
134 |     ]
135 | 
136 |     d.run(default_tag: 'test_tag') do
137 |       d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v'})
138 |     end
139 | 
140 |     assert_equal [{"k1" => 'v', "k2" => 'v', 'foo' => 'bar'}], d.events.map { |e| e.last }
141 |   end
142 | end
143 | 


--------------------------------------------------------------------------------