├── .gitignore ├── .rspec ├── .travis.yml ├── CHANGELOG.md ├── Gemfile ├── Gemfile.fluentd.0.10 ├── Gemfile.fluentd.0.12 ├── LICENSE ├── README.md ├── Rakefile ├── example ├── example.conf └── workers.conf ├── fluent-plugin-record-reformer.gemspec ├── lib └── fluent │ └── plugin │ ├── out_record_reformer.rb │ └── out_record_reformer │ ├── core.rb │ ├── v12.rb │ └── v14.rb └── test ├── bench_out_record_reformer.rb ├── helper.rb ├── output_test_driver.rb └── test_out_record_reformer.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /*.gem 2 | ~* 3 | #* 4 | *~ 5 | .bundle 6 | Gemfile.lock 7 | .rbenv-version 8 | vendor 9 | doc/* 10 | tmp/* 11 | coverage 12 | .yardoc 13 | pkg/ 14 | .ruby-version 15 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --colour 2 | --format documentation 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | rvm: 2 | - 2.1.* 3 | - 2.2.* 4 | - 2.3.0 5 | - 2.4.0 6 | gemfile: 7 | - Gemfile 8 | - Gemfile.fluentd.0.12 9 | - Gemfile.fluentd.0.10 10 | matrix: 11 | exclude: 12 | - rvm: 2.4.0 13 | gemfile: Gemfile.fluentd.0.10 14 | before_install: gem update bundler 15 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.9.1 (2017/07/26) 2 | 3 | Enhancements: 4 | 5 | * Support multi process workers of v0.14.12. 6 | 7 | ## 0.9.0 (2017/02/21) 8 | 9 | Enhancements: 10 | 11 | * Use v0.14 API for fluentd v0.14 12 | 13 | ## 0.8.3 (2017/01/26) 14 | 15 | Fixes 16 | 17 | * Apply `remove_keys` last, otherwise, `renew_time_key` could be removed before generating new time 18 | 19 | ## 0.8.2 (2016/08/21) 20 | 21 | Fixes 22 | 23 | * Prevent overwriting reserved placeholder keys such as tag, time, etc with `enable_ruby false` (thanks to @kimamula) 24 | 25 | ## 0.8.1 (2016/03/09) 26 | 27 | Fixes 28 | 29 | * Fix to be thread-safe 30 | 31 | Changes 32 | 33 | * Relax conditions which auto_typecast is applied for enable_ruby yes 34 | 35 | ## 0.8.0 (2016/01/28) 36 | 37 | Enhancements 38 | 39 | * Support `${record["key"]}` placeholder 40 | * Speed up `enable_ruby true` 41 | 42 | ## 0.7.2 (2015/12/29) 43 | 44 | Enhancements 45 | 46 | * Add desc to options (thanks to cosmo0920) 47 | 48 | ## 0.7.1 (2015/12/16) 49 | 50 | Enhancements 51 | 52 | * Add @id, @type, @label to BUILTIN_CONFIGURATIONS not to map into records (thanks to TrickyMonkey) 53 | 54 | ## 0.7.0 (2015/06/19) 55 | 56 | Enhancements 57 | 58 | * Add `auto_typecast` option (thanks to @piroor) 59 | 60 | ## 0.6.3 (2015/05/27) 61 | 62 | Fixes: 63 | 64 | * Fix not to include `renew_time_key` in records 65 | 66 | ## 0.6.2 (2015/05/27) 67 | 68 | Enhancements: 69 | 70 | * Add `renew_time_key` option (thanks to @tagomoris) 71 | 72 | ## 0.6.1 (2015/05/10) 73 | 74 | Enhancements: 75 | 76 | * Support label routing of Fluentd v0.12 77 | 78 | ## 0.6.0 (2015/04/11) 79 | 80 | Changes: 81 | 82 | * Accept field names starting with `@` (and any field names) in `enable_ruby false` 83 | 84 | ## 0.5.0 (2015/03/06) 85 | 86 | Enhancements: 87 | 88 | * Support JSON Array/Hash 89 | * Support placeholders for keys 90 | 91 | ## 0.4.0 (2014/10/31) 92 | 93 | Changes: 94 | 95 | * accept numbers as a record key 96 | * rescue if ruby code expansion failed, and log.warn 97 | * use newly test-unit gem instead of rspec 98 | 99 | ## 0.3.0 (2014/10/01) 100 | 101 | Fixes: 102 | 103 | * Fix to support camelCase record key name with `enable_ruby false` 104 | 105 | ## 0.2.10 (2014/09/22) 106 | 107 | Changes: 108 | 109 | * Remove fluentd version constraint 110 | 111 | ## 0.2.9 (2014/05/14) 112 | 113 | Enhancements: 114 | 115 | * Add `keep_keys` option 116 | 117 | ## 0.2.8 (2014/04/12) 118 | 119 | Changes: 120 | 121 | * Deprecate `output_tag` option. Use `tag` option instead. 122 | 123 | ## 0.2.7 (2014/03/26) 124 | 125 | Fixes: 126 | 127 | * Fix `log` method was not available in the inner class #5. 128 | 129 | ## 0.2.6 (2014/02/24) 130 | 131 | Enhancement: 132 | 133 | * Add debug log 134 | 135 | ## 0.2.5 (2014/02/04) 136 | 137 | Enhancement: 138 | 139 | * Support `log_level` option of Fleuntd v0.10.43 140 | 141 | ## 0.2.4 (2014/01/30) 142 | 143 | Fixes: 144 | 145 | * Fix `unitialized constant OpenStruct` error (thanks to emcgee) 146 | 147 | ## 0.2.3 (2014/01/25) 148 | 149 | Changes: 150 | 151 | * Change ${time} placeholder from integer to string when `enable_ruby false` 152 | 153 | ## 0.2.2 (2014/01/20) 154 | 155 | Enhancement: 156 | 157 | * Add `tag_prefix` and `tag_suffix` placeholders. Thanks to [xthexder](https://github.com/xthexder). 158 | 159 | ## 0.2.1 (2014/01/15) 160 | 161 | Enhancement: 162 | 163 | * Speed up 164 | 165 | ## 0.2.0 (2014/01/15) 166 | 167 | Enhancement: 168 | 169 | * Support a `record` directive 170 | * Add `remove_keys` option 171 | * Add `renew_record` option 172 | * Add `enable_ruby` option 173 | 174 | ## 0.1.1 (2013/11/21) 175 | 176 | Changes: 177 | 178 | * change the name of `tags` placeholder to `tag_parts`. `tags` is still available for old version compatibility, though 179 | 180 | ## 0.1.0 (2013/09/09) 181 | 182 | Enhancement: 183 | 184 | * require 'pathname', 'uri', 'cgi' to use these utilities in a placeholder 185 | 186 | ## 0.0.3 (2013/08/07) 187 | 188 | Enhancement: 189 | 190 | * Enable to reform tag 191 | 192 | ## 0.0.2 (2013/08/07) 193 | 194 | Enhancement: 195 | 196 | * Increase possible placeholders more such as `method`. 197 | 198 | ## 0.0.1 (2013/05/02) 199 | 200 | First release 201 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /Gemfile.fluentd.0.10: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gem 'fluentd', '~> 0.10.43' 4 | gemspec 5 | -------------------------------------------------------------------------------- /Gemfile.fluentd.0.12: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gem 'fluentd', '~> 0.12.0' 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 - 2015 Naotoshi Seo 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-plugin-record-reformer 2 | 3 | [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-record-reformer.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-record-reformer) 4 | 5 | Fluentd plugin to add or replace fields of a event record 6 | 7 | ## Requirements 8 | 9 | See [.travis.yml](.travis.yml) 10 | 11 | Note that `fluent-plugin-record-reformer` supports both v0.14 API and v0.12 API in one gem. 12 | 13 | ## Installation 14 | 15 | Use RubyGems: 16 | 17 | gem install fluent-plugin-record-reformer 18 | 19 | ## Configuration 20 | 21 | Example: 22 | 23 | 24 | type record_reformer 25 | remove_keys remove_me 26 | renew_record false 27 | enable_ruby false 28 | 29 | tag reformed.${tag_prefix[-2]} 30 | 31 | hostname ${hostname} 32 | input_tag ${tag} 33 | last_tag ${tag_parts[-1]} 34 | message ${record['message']}, yay! 35 | 36 | 37 | 38 | Assume following input is coming (indented): 39 | 40 | ```js 41 | foo.bar { 42 | "remove_me":"bar", 43 | "not_remove_me":"bar", 44 | "message":"Hello world!" 45 | } 46 | ``` 47 | 48 | then output becomes as below (indented): 49 | 50 | ```js 51 | reformed.foo { 52 | "not_remove_me":"bar", 53 | "hostname":"YOUR_HOSTNAME", 54 | "input_tag":"foo.bar", 55 | "last_tag":"bar", 56 | "message":"Hello world!, yay!", 57 | } 58 | ``` 59 | 60 | ## Configuration (Classic Style) 61 | 62 | Example: 63 | 64 | 65 | type record_reformer 66 | remove_keys remove_me 67 | renew_record false 68 | enable_ruby false 69 | tag reformed.${tag_prefix[-2]} 70 | 71 | hostname ${hostname} 72 | input_tag ${tag} 73 | last_tag ${tag_parts[-1]} 74 | message ${record['message']}, yay! 75 | 76 | 77 | This results in same, but please note that following option parameters are reserved, so can not be used as a record key. 78 | 79 | ## Option Parameters 80 | 81 | - output_tag (obsolete) 82 | 83 | The output tag name. This option is deprecated. Use `tag` option instead 84 | 85 | - tag 86 | 87 | The output tag name. 88 | 89 | - remove_keys 90 | 91 | Specify record keys to be removed by a string separated by , (comma) like 92 | 93 | remove_keys message,foo 94 | 95 | - renew_record *bool* 96 | 97 | `renew_record true` creates an output record newly without extending (merging) the input record fields. Default is `false`. 98 | 99 | - renew\_time\_key *string* 100 | 101 | `renew_time_key foo` overwrites the time of events with a value of the record field `foo` if exists. The value of `foo` must be a unix time. 102 | 103 | - keep_keys 104 | 105 | You may want to remain some record fields although you specify `renew_record true`. Then, specify record keys to be kept by a string separated by , (comma) like 106 | 107 | keep_keys message,foo 108 | 109 | - enable_ruby *bool* 110 | 111 | Enable to use ruby codes in placeholders. See `Placeholders` section. 112 | Default is `true` (just for lower version compatibility). 113 | 114 | - auto_typecast *bool* 115 | 116 | Automatically cast the field types. Default is false. 117 | NOTE: This option is effective only for field values comprised of a single placeholder. 118 | 119 | Effective Examples: 120 | 121 | foo ${foo} 122 | 123 | Non-Effective Examples: 124 | 125 | foo ${foo}${bar} 126 | foo ${foo}bar 127 | foo 1 128 | 129 | Internally, this **keeps** the type of value if the value text is comprised of a single placeholder, otherwise, values are treated as strings. 130 | 131 | When you need to cast field types manually, [out_typecast](https://github.com/tarom/fluent-plugin-typecast) and [filter_typecast](https://github.com/sonots/fluent-plugin-filter_typecast) are available. 132 | 133 | ## Placeholders 134 | 135 | Following placeholders are available: 136 | 137 | * ${record["key"]} Record value of `key` such as `${record["message"]}` in the above example (available from v0.8.0). 138 | * Originally, record placeholders were available as `${key}` such as `${message}`. This is still kept for the backward compatibility, but would be removed in the future. 139 | * ${hostname} Hostname of the running machine 140 | * ${tag} Input tag 141 | * ${time} Time of the event 142 | * ${tags[N]} (Obsolete. Use tag\_parts) Input tag splitted by '.' 143 | * ${tag\_parts[N]} Input tag splitted by '.' indexed with N such as `${tag_parts[0]}`, `${tag_parts[-1]}`. 144 | * ${tag\_prefix[N]} Tag parts before and on the index N. For example, 145 | 146 | Input tag: prefix.test.tag.suffix 147 | 148 | ${tag_prefix[0]} => prefix 149 | ${tag_prefix[1]} => prefix.test 150 | ${tag_prefix[-2]} => prefix.test.tag 151 | ${tag_prefix[-1]} => prefix.test.tag.suffix 152 | 153 | * ${tag\_suffix[N]} Tag parts after and on the index N. For example, 154 | 155 | Input tag: prefix.test.tag.suffix 156 | 157 | ${tag_suffix[0]} => prefix.test.tag.suffix 158 | ${tag_suffix[1]} => test.tag.suffix 159 | ${tag_suffix[-2]} => tag.suffix 160 | ${tag_suffix[-1]} => suffix 161 | 162 | It is also possible to write a ruby code in placeholders if you set `enable_ruby true` option, so you may write some codes as 163 | 164 | * ${time.strftime('%Y-%m-%dT%H:%M:%S%z')} 165 | * ${tag\_parts.last} 166 | 167 | but, please note that enabling ruby codes is not encouraged by security reasons and also in terms of the performance. 168 | 169 | ## Relatives 170 | 171 | Following plugins look similar: 172 | 173 | * [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier) 174 | * [fluent-plugin-format](https://github.com/mach/fluent-plugin-format) 175 | * [fluent-plugin-add](https://github.com/yu-yamada/fluent-plugin-add) 176 | * [filter_record_transformer](http://docs.fluentd.org/v0.12/articles/filter_record_transformer) is a Fluentd v0.12 built-in plugin which is based on record-reformer. 177 | 178 | ## ChangeLog 179 | 180 | See [CHANGELOG.md](CHANGELOG.md) for details. 181 | 182 | ## Contributing 183 | 184 | 1. Fork it 185 | 2. Create your feature branch (`git checkout -b my-new-feature`) 186 | 3. Commit your changes (`git commit -am 'Add some feature'`) 187 | 4. Push to the branch (`git push origin my-new-feature`) 188 | 5. Create new [Pull Request](../../pull/new/master) 189 | 190 | ## Copyright 191 | 192 | Copyright (c) 2013 - 2015 Naotoshi Seo. See [LICENSE](LICENSE) for details. 193 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "bundler/gem_tasks" 3 | 4 | require 'rake/testtask' 5 | desc 'Run test_unit based test' 6 | Rake::TestTask.new(:test) do |t| 7 | t.libs << "test" 8 | t.test_files = Dir["test/**/test_*.rb"].sort 9 | t.verbose = true 10 | #t.warning = true 11 | end 12 | task :default => :test 13 | 14 | desc 'Open an irb session preloaded with the gem library' 15 | task :console do 16 | sh 'irb -rubygems -I lib' 17 | end 18 | task :c => :console 19 | -------------------------------------------------------------------------------- /example/example.conf: -------------------------------------------------------------------------------- 1 | 2 | @type dummy 3 | tag dummy 4 | dummy {"message":"foo","time":1432732710,"members":["Alice"]} 5 | 6 | 7 | 8 | @type record_reformer 9 | renew_time_key time 10 | tag reformed.${tag} 11 | enable_ruby true 12 | auto_typecast true 13 | 14 | members ${members + ["Bob"]} 15 | 16 | 17 | 18 | 19 | @type stdout 20 | 21 | -------------------------------------------------------------------------------- /example/workers.conf: -------------------------------------------------------------------------------- 1 | 2 | workers 2 # v0.4.12 3 | 4 | 5 | @include example.conf 6 | -------------------------------------------------------------------------------- /fluent-plugin-record-reformer.gemspec: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | $:.push File.expand_path('../lib', __FILE__) 3 | 4 | Gem::Specification.new do |gem| 5 | gem.name = "fluent-plugin-record-reformer" 6 | gem.version = "0.9.1" 7 | gem.authors = ["Naotoshi Seo"] 8 | gem.email = "sonots@gmail.com" 9 | gem.homepage = "https://github.com/sonots/fluent-plugin-record-reformer" 10 | gem.description = "Fluentd plugin to add or replace fields of a event record" 11 | gem.summary = gem.description 12 | gem.licenses = ["MIT"] 13 | gem.has_rdoc = false 14 | 15 | gem.files = `git ls-files`.split("\n") 16 | gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 17 | gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 18 | gem.require_paths = ['lib'] 19 | 20 | gem.add_dependency "fluentd" 21 | gem.add_development_dependency "rake" 22 | gem.add_development_dependency "pry" 23 | gem.add_development_dependency "pry-nav" 24 | gem.add_development_dependency "test-unit" 25 | gem.add_development_dependency "test-unit-rr" 26 | gem.add_development_dependency "timecop" 27 | end 28 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_record_reformer.rb: -------------------------------------------------------------------------------- 1 | require 'fluent/version' 2 | major, minor, patch = Fluent::VERSION.split('.').map(&:to_i) 3 | if major > 0 || (major == 0 && minor >= 14) 4 | require_relative 'out_record_reformer/v14' 5 | else 6 | require_relative 'out_record_reformer/v12' 7 | end 8 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_record_reformer/core.rb: -------------------------------------------------------------------------------- 1 | require 'ostruct' 2 | require 'socket' 3 | 4 | module Fluent 5 | module RecordReformerOutputCore 6 | def initialize 7 | super 8 | end 9 | 10 | def self.included(klass) 11 | klass.config_param :output_tag, :string, :default => nil, # obsolete 12 | :desc => 'The output tag name. This option is deprecated. Use `tag` option instead.' 13 | klass.config_param :tag, :string, :default => nil, 14 | :desc => 'The output tag name.' 15 | klass.config_param :remove_keys, :string, :default => nil, 16 | :desc => 'Specify record keys to be removed by a string separated by , (comma).' 17 | klass.config_param :keep_keys, :string, :default => nil, 18 | :desc => 'Specify record keys to be kept by a string separated by , (comma).' 19 | klass.config_param :renew_record, :bool, :default => false, 20 | :desc => 'Creates an output record newly without extending (merging) the input record fields.' 21 | klass.config_param :renew_time_key, :string, :default => nil, 22 | :desc => 'Overwrites the time of events with a value of the record field.' 23 | klass.config_param :enable_ruby, :bool, :default => true, # true for lower version compatibility 24 | :desc => 'Enable to use ruby codes in placeholders.' 25 | klass.config_param :auto_typecast, :bool, :default => false, # false for lower version compatibility 26 | :desc => 'Automatically cast the field types.' 27 | end 28 | 29 | BUILTIN_CONFIGURATIONS = %W(@id @type @label type tag output_tag remove_keys renew_record keep_keys enable_ruby renew_time_key auto_typecast) 30 | 31 | def configure(conf) 32 | super 33 | 34 | map = {} 35 | conf.each_pair { |k, v| 36 | next if BUILTIN_CONFIGURATIONS.include?(k) 37 | conf.has_key?(k) # to suppress unread configuration warning 38 | map[k] = parse_value(v) 39 | } 40 | # directive 41 | conf.elements.select { |element| element.name == 'record' }.each { |element| 42 | element.each_pair { |k, v| 43 | element.has_key?(k) # to suppress unread configuration warning 44 | map[k] = parse_value(v) 45 | } 46 | } 47 | 48 | if @remove_keys 49 | @remove_keys = @remove_keys.split(',') 50 | end 51 | 52 | if @keep_keys 53 | raise Fluent::ConfigError, "out_record_reformer: `renew_record` must be true to use `keep_keys`" unless @renew_record 54 | @keep_keys = @keep_keys.split(',') 55 | end 56 | 57 | if @output_tag and @tag.nil? # for lower version compatibility 58 | log.warn "out_record_reformer: `output_tag` is deprecated. Use `tag` option instead." 59 | @tag = @output_tag 60 | end 61 | if @tag.nil? 62 | raise Fluent::ConfigError, "out_record_reformer: `tag` must be specified" 63 | end 64 | 65 | placeholder_expander_params = { 66 | :log => log, 67 | :auto_typecast => @auto_typecast, 68 | } 69 | @placeholder_expander = 70 | if @enable_ruby 71 | # require utilities which would be used in ruby placeholders 72 | require 'pathname' 73 | require 'uri' 74 | require 'cgi' 75 | RubyPlaceholderExpander.new(placeholder_expander_params) 76 | else 77 | PlaceholderExpander.new(placeholder_expander_params) 78 | end 79 | @map = @placeholder_expander.preprocess_map(map) 80 | @tag = @placeholder_expander.preprocess_map(@tag) 81 | 82 | @hostname = Socket.gethostname 83 | end 84 | 85 | def process(tag, es) 86 | tag_parts = tag.split('.') 87 | tag_prefix = tag_prefix(tag_parts) 88 | tag_suffix = tag_suffix(tag_parts) 89 | placeholder_values = { 90 | 'tag' => tag, 91 | 'tags' => tag_parts, # for old version compatibility 92 | 'tag_parts' => tag_parts, 93 | 'tag_prefix' => tag_prefix, 94 | 'tag_suffix' => tag_suffix, 95 | 'hostname' => @hostname, 96 | } 97 | last_record = nil 98 | es.each {|time, record| 99 | last_record = record # for debug log 100 | placeholder_values.merge!({ 101 | 'time' => @placeholder_expander.time_value(time), 102 | 'record' => record, 103 | }) 104 | new_tag, new_record = reform(@tag, record, placeholder_values) 105 | if new_tag 106 | if @renew_time_key && new_record.has_key?(@renew_time_key) 107 | time = new_record[@renew_time_key].to_i 108 | end 109 | @remove_keys.each {|k| new_record.delete(k) } if @remove_keys 110 | router.emit(new_tag, time, new_record) 111 | end 112 | } 113 | rescue => e 114 | log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.first}" 115 | log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}" 116 | end 117 | 118 | private 119 | 120 | def parse_value(value_str) 121 | if value_str.start_with?('{', '[') 122 | JSON.parse(value_str) 123 | else 124 | value_str 125 | end 126 | rescue => e 127 | log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", :error_class => e.class, :error => e.message 128 | value_str # emit as string 129 | end 130 | 131 | def reform(tag, record, placeholder_values) 132 | placeholders = @placeholder_expander.prepare_placeholders(placeholder_values) 133 | 134 | new_tag = expand_placeholders(tag, placeholders) 135 | 136 | new_record = @renew_record ? {} : record.dup 137 | @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record 138 | new_record.merge!(expand_placeholders(@map, placeholders)) 139 | 140 | [new_tag, new_record] 141 | end 142 | 143 | def expand_placeholders(value, placeholders) 144 | if value.is_a?(String) 145 | new_value = @placeholder_expander.expand(value, placeholders) 146 | elsif value.is_a?(Hash) 147 | new_value = {} 148 | value.each_pair do |k, v| 149 | new_key = @placeholder_expander.expand(k, placeholders, true) 150 | new_value[new_key] = expand_placeholders(v, placeholders) 151 | end 152 | elsif value.is_a?(Array) 153 | new_value = [] 154 | value.each_with_index do |v, i| 155 | new_value[i] = expand_placeholders(v, placeholders) 156 | end 157 | else 158 | new_value = value 159 | end 160 | new_value 161 | end 162 | 163 | def tag_prefix(tag_parts) 164 | return [] if tag_parts.empty? 165 | tag_prefix = [tag_parts.first] 166 | 1.upto(tag_parts.size-1).each do |i| 167 | tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}" 168 | end 169 | tag_prefix 170 | end 171 | 172 | def tag_suffix(tag_parts) 173 | return [] if tag_parts.empty? 174 | rev_tag_parts = tag_parts.reverse 175 | rev_tag_suffix = [rev_tag_parts.first] 176 | 1.upto(tag_parts.size-1).each do |i| 177 | rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}" 178 | end 179 | rev_tag_suffix.reverse! 180 | end 181 | 182 | # THIS CLASS MUST BE THREAD-SAFE 183 | class PlaceholderExpander 184 | attr_reader :placeholders, :log 185 | 186 | def initialize(params) 187 | @log = params[:log] 188 | @auto_typecast = params[:auto_typecast] 189 | end 190 | 191 | def time_value(time) 192 | Time.at(time).to_s 193 | end 194 | 195 | def preprocess_map(value, force_stringify = false) 196 | value 197 | end 198 | 199 | def prepare_placeholders(placeholder_values) 200 | placeholders = {} 201 | 202 | placeholder_values.each do |key, value| 203 | if value.kind_of?(Array) # tag_parts, etc 204 | size = value.size 205 | value.each_with_index do |v, idx| 206 | placeholders.store("${#{key}[#{idx}]}", v) 207 | placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1] 208 | end 209 | elsif value.kind_of?(Hash) # record, etc 210 | value.each do |k, v| 211 | unless placeholder_values.has_key?(k) # prevent overwriting the reserved keys such as tag 212 | placeholders.store("${#{k}}", v) 213 | end 214 | placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"] 215 | end 216 | else # string, interger, float, and others? 217 | placeholders.store("${#{key}}", value) 218 | end 219 | end 220 | 221 | placeholders 222 | end 223 | 224 | # Expand string with placeholders 225 | # 226 | # @param [String] str 227 | # @param [Boolean] force_stringify the value must be string, used for hash key 228 | def expand(str, placeholders, force_stringify = false) 229 | if @auto_typecast and !force_stringify 230 | single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/) 231 | if single_placeholder_matched 232 | log_if_unknown_placeholder($1, placeholders) 233 | return placeholders[single_placeholder_matched[1]] 234 | end 235 | end 236 | str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) { 237 | log_if_unknown_placeholder($1, placeholders) 238 | placeholders[$1] 239 | } 240 | end 241 | 242 | private 243 | 244 | def log_if_unknown_placeholder(placeholder, placeholders) 245 | unless placeholders.include?(placeholder) 246 | log.warn "record_reformer: unknown placeholder `#{placeholder}` found" 247 | end 248 | end 249 | end 250 | 251 | # THIS CLASS MUST BE THREAD-SAFE 252 | class RubyPlaceholderExpander 253 | attr_reader :log 254 | 255 | def initialize(params) 256 | @log = params[:log] 257 | @auto_typecast = params[:auto_typecast] 258 | @cleanroom_expander = CleanroomExpander.new 259 | end 260 | 261 | def time_value(time) 262 | Time.at(time) 263 | end 264 | 265 | # Preprocess record map to convert into ruby string expansion 266 | # 267 | # @param [Hash|String|Array] value record map config 268 | # @param [Boolean] force_stringify the value must be string, used for hash key 269 | def preprocess_map(value, force_stringify = false) 270 | new_value = nil 271 | if value.is_a?(String) 272 | if @auto_typecast and !force_stringify 273 | num_placeholders = value.scan('${').size 274 | if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}') 275 | new_value = value[2..-2] # ${..} => .. 276 | end 277 | end 278 | unless new_value 279 | new_value = "%Q[#{value.gsub('${', '#{')}]" # xx${..}xx => %Q[xx#{..}xx] 280 | end 281 | elsif value.is_a?(Hash) 282 | new_value = {} 283 | value.each_pair do |k, v| 284 | new_value[preprocess_map(k, true)] = preprocess_map(v) 285 | end 286 | elsif value.is_a?(Array) 287 | new_value = [] 288 | value.each_with_index do |v, i| 289 | new_value[i] = preprocess_map(v) 290 | end 291 | else 292 | new_value = value 293 | end 294 | new_value 295 | end 296 | 297 | def prepare_placeholders(placeholder_values) 298 | placeholder_values 299 | end 300 | 301 | # Expand string with placeholders 302 | # 303 | # @param [String] str 304 | def expand(str, placeholders, force_stringify = false) 305 | @cleanroom_expander.expand( 306 | str, 307 | placeholders['tag'], 308 | placeholders['time'], 309 | placeholders['record'], 310 | placeholders['tag_parts'], 311 | placeholders['tag_prefix'], 312 | placeholders['tag_suffix'], 313 | placeholders['hostname'], 314 | ) 315 | rescue => e 316 | log.warn "record_reformer: failed to expand `#{str}`", :error_class => e.class, :error => e.message 317 | log.warn_backtrace 318 | nil 319 | end 320 | 321 | class CleanroomExpander 322 | def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname) 323 | tags = tag_parts # for old version compatibility 324 | Thread.current[:record_reformer_record] = record # for old version compatibility 325 | instance_eval(__str_to_eval__) 326 | end 327 | 328 | # for old version compatibility 329 | def method_missing(name) 330 | key = name.to_s 331 | record = Thread.current[:record_reformer_record] 332 | if record.has_key?(key) 333 | record[key] 334 | else 335 | raise NameError, "undefined local variable or method `#{key}'" 336 | end 337 | end 338 | 339 | (Object.instance_methods).each do |m| 340 | undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/ 341 | end 342 | end 343 | end 344 | end 345 | end 346 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_record_reformer/v12.rb: -------------------------------------------------------------------------------- 1 | require_relative 'core' 2 | 3 | module Fluent 4 | class RecordReformerOutput < Output 5 | Fluent::Plugin.register_output('record_reformer', self) 6 | 7 | include ::Fluent::RecordReformerOutputCore 8 | 9 | def initialize 10 | super 11 | end 12 | 13 | # To support log_level option implemented by Fluentd v0.10.43 14 | unless method_defined?(:log) 15 | define_method("log") { $log } 16 | end 17 | 18 | # Define `router` method of v0.12 to support v0.10 or earlier 19 | unless method_defined?(:router) 20 | define_method("router") { Fluent::Engine } 21 | end 22 | 23 | def configure(conf) 24 | super 25 | end 26 | 27 | def emit(tag, es, chain) 28 | process(tag, es) 29 | chain.next 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_record_reformer/v14.rb: -------------------------------------------------------------------------------- 1 | require_relative 'core' 2 | 3 | module Fluent 4 | class Plugin::RecordReformerOutput < Plugin::Output 5 | Fluent::Plugin.register_output('record_reformer', self) 6 | 7 | helpers :event_emitter 8 | include ::Fluent::RecordReformerOutputCore 9 | 10 | def initialize 11 | super 12 | end 13 | 14 | def configure(conf) 15 | super 16 | end 17 | 18 | def multi_workers_ready? 19 | true 20 | end 21 | 22 | def process(tag, es) 23 | super 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/bench_out_record_reformer.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | require_relative 'helper' 3 | require 'fluent/plugin/out_record_reformer' 4 | require 'benchmark' 5 | Fluent::Test.setup 6 | 7 | def create_driver(config, tag = 'foo.bar') 8 | Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, tag).configure(config) 9 | end 10 | 11 | # setup 12 | message = {'message' => "2013/01/13T07:02:11.124202 INFO GET /ping"} 13 | time = Time.now.to_i 14 | 15 | enable_ruby_driver = create_driver(%[ 16 | enable_ruby true 17 | output_tag reformed.${tag} 18 | message ${tag_parts[0]} 19 | ]) 20 | disable_ruby_driver = create_driver(%[ 21 | enable_ruby false 22 | output_tag reformed.${tag} 23 | message ${tag_parts[0]} 24 | ]) 25 | 26 | # bench 27 | n = 1000 28 | Benchmark.bm(7) do |x| 29 | x.report("enable_ruby") { enable_ruby_driver.run { n.times { enable_ruby_driver.emit(message, time) } } } 30 | x.report("disable_ruby") { disable_ruby_driver.run { n.times { disable_ruby_driver.emit(message, time) } } } 31 | end 32 | 33 | #BEFORE REFACTORING 34 | # user system total real 35 | #enable_ruby 0.310000 0.000000 0.310000 ( 0.835560) 36 | #disable_ruby 0.150000 0.000000 0.150000 ( 0.679239) 37 | 38 | #AFTER REFACTORING (PlaceholderParser and RubyPlaceholderParser) 39 | # user system total real 40 | #enable_ruby 0.290000 0.010000 0.300000 ( 0.815281) 41 | #disable_ruby 0.060000 0.000000 0.060000 ( 0.588556) 42 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'test/unit/rr' 3 | require 'timecop' 4 | require 'fluent/log' 5 | require 'fluent/test' 6 | 7 | unless defined?(Test::Unit::AssertionFailedError) 8 | class Test::Unit::AssertionFailedError < StandardError 9 | end 10 | end 11 | 12 | # Reduce sleep period at 13 | # https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81 14 | module Fluent 15 | module Test 16 | class TestDriver 17 | def run(num_waits = 10, &block) 18 | @instance.start 19 | begin 20 | # wait until thread starts 21 | # num_waits.times { sleep 0.05 } 22 | sleep 0.05 23 | return yield 24 | ensure 25 | @instance.shutdown 26 | end 27 | end 28 | end 29 | end 30 | end 31 | 32 | require_relative 'output_test_driver' 33 | -------------------------------------------------------------------------------- /test/output_test_driver.rb: -------------------------------------------------------------------------------- 1 | # This test driver makes a compatible layer for v0.14 as of v0.12 2 | 3 | # d = create_driver(conf, use_v1, default_tag: @tag) 4 | # time = event_time("2010-05-04 03:02:01") 5 | # d.run do 6 | # d.emit(record, time) 7 | # end 8 | # d.emits 9 | 10 | require 'fluent/version' 11 | major, minor, patch = Fluent::VERSION.split('.').map(&:to_i) 12 | if major > 0 || (major == 0 && minor >= 14) 13 | require 'fluent/test/driver/output' 14 | require 'fluent/test/helpers' 15 | include Fluent::Test::Helpers 16 | 17 | class OutputTestDriver < Fluent::Test::Driver::Output 18 | def initialize(klass, tag) 19 | super(klass) 20 | @tag = tag 21 | end 22 | 23 | def configure(conf, use_v1) 24 | super(conf, syntax: use_v1 ? :v1 : :v0) 25 | end 26 | 27 | def run(&block) 28 | super(default_tag: @tag, &block) 29 | end 30 | 31 | def emit(record, time) 32 | feed(time, record) 33 | end 34 | 35 | def emits 36 | events 37 | end 38 | end 39 | 40 | def create_driver(conf, use_v1, default_tag: @tag) 41 | OutputTestDriver.new(Fluent::Plugin::RecordReformerOutput, default_tag).configure(conf, use_v1) 42 | end 43 | else 44 | def event_time(str) 45 | Time.parse(str) 46 | end 47 | 48 | def create_driver(conf, use_v1, default_tag: @tag) 49 | Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, default_tag).configure(conf, use_v1) 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/test_out_record_reformer.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helper' 2 | require 'time' 3 | require 'fluent/plugin/out_record_reformer' 4 | 5 | Fluent::Test.setup 6 | 7 | class RecordReformerOutputTest < Test::Unit::TestCase 8 | def emit(config, use_v1, msgs = ['']) 9 | d = create_driver(config, use_v1) 10 | d.run do 11 | records = msgs.map do |msg| 12 | next msg if msg.is_a?(Hash) 13 | { 'eventType0' => 'bar', 'message' => msg } 14 | end 15 | records.each do |record| 16 | d.emit(record, @time) 17 | end 18 | end 19 | 20 | @instance = d.instance 21 | d.emits 22 | end 23 | 24 | setup do 25 | @hostname = Socket.gethostname.chomp 26 | @tag = 'test.tag' 27 | @tag_parts = @tag.split('.') 28 | @time = event_time("2010-05-04 03:02:01") 29 | Timecop.freeze(@time) 30 | end 31 | 32 | teardown do 33 | Timecop.return 34 | end 35 | 36 | CONFIG = %[ 37 | tag reformed.${tag} 38 | 39 | hostname ${hostname} 40 | input_tag ${tag} 41 | time ${time.to_s} 42 | message ${hostname} ${tag_parts.last} ${URI.escape(message)} 43 | ] 44 | 45 | [true, false].each do |use_v1| 46 | sub_test_case 'configure' do 47 | test 'typical usage' do 48 | assert_nothing_raised do 49 | create_driver(CONFIG, use_v1) 50 | end 51 | end 52 | 53 | test "tag is not specified" do 54 | assert_raise(Fluent::ConfigError) do 55 | create_driver('', use_v1) 56 | end 57 | end 58 | 59 | test "keep_keys must be specified together with renew_record true" do 60 | assert_raise(Fluent::ConfigError) do 61 | create_driver(%[keep_keys a], use_v1) 62 | end 63 | end 64 | end 65 | 66 | sub_test_case "test options" do 67 | test 'typical usage' do 68 | msgs = ['1', '2'] 69 | emits = emit(CONFIG, use_v1, msgs) 70 | assert_equal 2, emits.size 71 | emits.each_with_index do |(tag, time, record), i| 72 | assert_equal("reformed.#{@tag}", tag) 73 | assert_equal('bar', record['eventType0']) 74 | assert_equal(@hostname, record['hostname']) 75 | assert_equal(@tag, record['input_tag']) 76 | assert_equal(Time.at(@time).localtime.to_s, record['time']) 77 | assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", record['message']) 78 | end 79 | end 80 | 81 | test '(obsolete) output_tag' do 82 | config = %[output_tag reformed.${tag}] 83 | msgs = ['1'] 84 | emits = emit(config, use_v1, msgs) 85 | emits.each_with_index do |(tag, time, record), i| 86 | assert_equal("reformed.#{@tag}", tag) 87 | end 88 | end 89 | 90 | test 'record directive' do 91 | config = %[ 92 | tag reformed.${tag} 93 | 94 | 95 | hostname ${hostname} 96 | tag ${tag} 97 | time ${time.to_s} 98 | message ${hostname} ${tag_parts.last} ${message} 99 | 100 | ] 101 | msgs = ['1', '2'] 102 | emits = emit(config, use_v1, msgs) 103 | emits.each_with_index do |(tag, time, record), i| 104 | assert_equal("reformed.#{@tag}", tag) 105 | assert_equal('bar', record['eventType0']) 106 | assert_equal(@hostname, record['hostname']) 107 | assert_equal(@tag, record['tag']) 108 | assert_equal(Time.at(@time).localtime.to_s, record['time']) 109 | assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", record['message']) 110 | end 111 | end 112 | 113 | test 'remove_keys' do 114 | config = CONFIG + %[remove_keys eventType0,message] 115 | emits = emit(config, use_v1) 116 | emits.each_with_index do |(tag, time, record), i| 117 | assert_equal("reformed.#{@tag}", tag) 118 | assert_not_include(record, 'eventType0') 119 | assert_equal(@hostname, record['hostname']) 120 | assert_equal(@tag, record['input_tag']) 121 | assert_equal(Time.at(@time).localtime.to_s, record['time']) 122 | assert_not_include(record, 'message') 123 | end 124 | end 125 | 126 | test 'renew_record' do 127 | config = CONFIG + %[renew_record true] 128 | msgs = ['1', '2'] 129 | emits = emit(config, use_v1, msgs) 130 | emits.each_with_index do |(tag, time, record), i| 131 | assert_equal("reformed.#{@tag}", tag) 132 | assert_not_include(record, 'eventType0') 133 | assert_equal(@hostname, record['hostname']) 134 | assert_equal(@tag, record['input_tag']) 135 | assert_equal(Time.at(@time).localtime.to_s, record['time']) 136 | assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", record['message']) 137 | end 138 | end 139 | 140 | test 'renew_time_key' do 141 | times = [ Time.at(event_time("2010-05-04 03:02:02")), Time.at(event_time("2010-05-04 03:02:03")) ] 142 | config = < 147 | event_time_key ${Time.parse(record["message"]).to_i} 148 | 149 | EOC 150 | msgs = times.map{|t| t.to_s } 151 | emits = emit(config, use_v1, msgs) 152 | emits.each_with_index do |(tag, time, record), i| 153 | assert_equal("reformed.#{@tag}", tag) 154 | assert_equal(times[i].to_i, time) 155 | assert_true(record.has_key?('event_time_key')) 156 | end 157 | end 158 | 159 | test 'renew_time_key and remove_keys' do 160 | config = < 166 | event_time_key ${Time.parse(record["message"]).to_i} 167 | 168 | EOC 169 | times = [ Time.at(event_time("2010-05-04 03:02:02")), Time.at(event_time("2010-05-04 03:02:03")) ] 170 | msgs = times.map{|t| t.to_s } 171 | emits = emit(config, use_v1, msgs) 172 | emits.each_with_index do |(tag, time, record), i| 173 | assert_equal("reformed.#{@tag}", tag) 174 | assert_equal(times[i].to_i, time) 175 | assert_false(record.has_key?('event_time_key')) 176 | end 177 | end 178 | 179 | test 'keep_keys' do 180 | config = %[tag reformed.${tag}\nrenew_record true\nkeep_keys eventType0,message] 181 | msgs = ['1', '2'] 182 | emits = emit(config, use_v1, msgs) 183 | emits.each_with_index do |(tag, time, record), i| 184 | assert_equal("reformed.#{@tag}", tag) 185 | assert_equal('bar', record['eventType0']) 186 | assert_equal(msgs[i], record['message']) 187 | end 188 | end 189 | 190 | test 'enable_ruby no' do 191 | config = %[ 192 | tag reformed.${tag} 193 | enable_ruby no 194 | 195 | message ${hostname} ${tag_parts.last} ${URI.encode(message)} 196 | 197 | ] 198 | msgs = ['1', '2'] 199 | emits = emit(config, use_v1, msgs) 200 | emits.each_with_index do |(tag, time, record), i| 201 | assert_equal("reformed.#{@tag}", tag) 202 | assert_equal("#{@hostname} ", record['message']) 203 | end 204 | end 205 | end 206 | 207 | sub_test_case 'test placeholders' do 208 | %w[yes no].each do |enable_ruby| 209 | 210 | test "hostname with enble_ruby #{enable_ruby}" do 211 | config = %[ 212 | tag tag 213 | enable_ruby #{enable_ruby} 214 | 215 | message ${hostname} 216 | 217 | ] 218 | emits = emit(config, use_v1) 219 | emits.each do |(tag, time, record)| 220 | assert_equal(@hostname, record['message']) 221 | end 222 | end 223 | 224 | test "tag with enable_ruby #{enable_ruby}" do 225 | config = %[ 226 | tag tag 227 | enable_ruby #{enable_ruby} 228 | 229 | message ${tag} 230 | 231 | ] 232 | emits = emit(config, use_v1) 233 | emits.each do |(tag, time, record)| 234 | assert_equal(@tag, record['message']) 235 | end 236 | end 237 | 238 | test "tag_parts with enable_ruby #{enable_ruby}" do 239 | config = %[ 240 | tag tag 241 | enable_ruby #{enable_ruby} 242 | 243 | message ${tag_parts[0]} ${tag_parts[-1]} 244 | 245 | ] 246 | expected = "#{@tag.split('.').first} #{@tag.split('.').last}" 247 | emits = emit(config, use_v1) 248 | emits.each do |(tag, time, record)| 249 | assert_equal(expected, record['message']) 250 | end 251 | end 252 | 253 | test "(obsolete) tags with enable_ruby #{enable_ruby}" do 254 | config = %[ 255 | tag tag 256 | enable_ruby #{enable_ruby} 257 | 258 | message ${tags[0]} ${tags[-1]} 259 | 260 | ] 261 | expected = "#{@tag.split('.').first} #{@tag.split('.').last}" 262 | emits = emit(config, use_v1) 263 | emits.each do |(tag, time, record)| 264 | assert_equal(expected, record['message']) 265 | end 266 | end 267 | 268 | test "${tag_prefix[N]} and ${tag_suffix[N]} with enable_ruby #{enable_ruby}" do 269 | config = %[ 270 | tag tag 271 | enable_ruby #{enable_ruby} 272 | 273 | message ${tag_prefix[1]} ${tag_prefix[-2]} ${tag_suffix[2]} ${tag_suffix[-3]} 274 | 275 | ] 276 | @tag = 'prefix.test.tag.suffix' 277 | expected = "prefix.test prefix.test.tag tag.suffix test.tag.suffix" 278 | emits = emit(config, use_v1) 279 | emits.each do |(tag, time, record)| 280 | assert_equal(expected, record['message']) 281 | end 282 | end 283 | 284 | test "time with enable_ruby #{enable_ruby}" do 285 | config = %[ 286 | tag tag 287 | enable_ruby #{enable_ruby} 288 | 289 | message ${time} 290 | 291 | ] 292 | emits = emit(config, use_v1) 293 | emits.each do |(tag, time, record)| 294 | assert_equal(Time.at(time).localtime.to_s, record['message']) 295 | end 296 | end 297 | 298 | test "record keys with enable_ruby #{enable_ruby}" do 299 | config = %[ 300 | tag tag 301 | enable_ruby #{enable_ruby} 302 | remove_keys eventType0 303 | 304 | message bar ${message} 305 | eventtype ${eventType0} 306 | 307 | ] 308 | msgs = ['1', '2'] 309 | emits = emit(config, use_v1, msgs) 310 | emits.each_with_index do |(tag, time, record), i| 311 | assert_not_include(record, 'eventType0') 312 | assert_equal("bar", record['eventtype']) 313 | assert_equal("bar #{msgs[i]}", record['message']) 314 | end 315 | end 316 | 317 | test "Prevent overriting reserved keys (such as tag, etc) #40 with enable_ruby #{enable_ruby}" do 318 | config = %[ 319 | tag tag 320 | enable_ruby #{enable_ruby} 321 | 322 | new_tag ${tag} 323 | new_time ${time} 324 | new_record_tag ${record["tag"]} 325 | new_record_time ${record["time"]} 326 | 327 | ] 328 | records = [{'tag' => 'tag', 'time' => 'time'}] 329 | emits = emit(config, use_v1, records) 330 | emits.each do |(tag, time, record)| 331 | assert_not_equal('tag', record['new_tag']) 332 | assert_equal(@tag, record['new_tag']) 333 | assert_not_equal('time', record['new_time']) 334 | assert_equal(Time.at(@time).localtime.to_s, record['new_time']) 335 | assert_equal('tag', record['new_record_tag']) 336 | assert_equal('time', record['new_record_time']) 337 | end 338 | end 339 | 340 | test "hash values with placeholders with enable_ruby #{enable_ruby}" do 341 | config = %[ 342 | tag tag 343 | enable_ruby #{enable_ruby} 344 | 345 | hash_field {"hostname":"${hostname}", "tag":"${tag}", "${tag}":100} 346 | 347 | ] 348 | msgs = ['1', '2'] 349 | es = emit(config, use_v1, msgs) 350 | es.each_with_index do |(tag, time, record), i| 351 | assert_equal({"hostname" => @hostname, "tag" => @tag, "#{@tag}" => 100}, record['hash_field']) 352 | end 353 | end 354 | 355 | test "array values with placeholders with enable_ruby #{enable_ruby}" do 356 | config = %[ 357 | tag tag 358 | enable_ruby #{enable_ruby} 359 | 360 | array_field ["${hostname}", "${tag}"] 361 | 362 | ] 363 | msgs = ['1', '2'] 364 | es = emit(config, use_v1, msgs) 365 | es.each_with_index do |(tag, time, record), i| 366 | assert_equal([@hostname, @tag], record['array_field']) 367 | end 368 | end 369 | 370 | test "array and hash values with placeholders with enable_ruby #{enable_ruby}" do 371 | config = %[ 372 | tag tag 373 | enable_ruby #{enable_ruby} 374 | 375 | mixed_field [{"tag":"${tag}"}] 376 | 377 | ] 378 | msgs = ['1', '2'] 379 | es = emit(config, use_v1, msgs) 380 | es.each_with_index do |(tag, time, record), i| 381 | assert_equal([{"tag" => @tag}], record['mixed_field']) 382 | end 383 | end 384 | 385 | if use_v1 == true 386 | # works with only v1 config 387 | test "keys with placeholders with enable_ruby #{enable_ruby}" do 388 | config = %[ 389 | tag tag 390 | enable_ruby #{enable_ruby} 391 | renew_record true 392 | 393 | ${hostname} hostname 394 | foo.${tag} tag 395 | 396 | ] 397 | msgs = ['1', '2'] 398 | es = emit(config, use_v1, msgs) 399 | es.each_with_index do |(tag, time, record), i| 400 | assert_equal({@hostname=>'hostname',"foo.#{@tag}"=>'tag'}, record) 401 | end 402 | end 403 | end 404 | 405 | test "disabled autodetectction of value type with enable_ruby #{enable_ruby}" do 406 | config = %[ 407 | tag tag 408 | enable_ruby #{enable_ruby} 409 | auto_typecast false 410 | 411 | single ${source} 412 | multiple ${source}${source} 413 | with_prefix prefix-${source} 414 | with_suffix ${source}-suffix 415 | 416 | ] 417 | msgs = [ 418 | { "source" => "string" }, 419 | { "source" => 123 }, 420 | { "source" => [1, 2] }, 421 | { "source" => {a:1, b:2} }, 422 | { "source" => nil }, 423 | ] 424 | expected_results = [ 425 | { :single => "string", 426 | :multiple => "stringstring", 427 | :with_prefix => "prefix-string", 428 | :with_suffix => "string-suffix" }, 429 | { :single => 123.to_s, 430 | :multiple => "#{123.to_s}#{123.to_s}", 431 | :with_prefix => "prefix-#{123.to_s}", 432 | :with_suffix => "#{123.to_s}-suffix" }, 433 | { :single => [1, 2].to_s, 434 | :multiple => "#{[1, 2].to_s}#{[1, 2].to_s}", 435 | :with_prefix => "prefix-#{[1, 2].to_s}", 436 | :with_suffix => "#{[1, 2].to_s}-suffix" }, 437 | { :single => {a:1, b:2}.to_s, 438 | :multiple => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}", 439 | :with_prefix => "prefix-#{{a:1, b:2}.to_s}", 440 | :with_suffix => "#{{a:1, b:2}.to_s}-suffix" }, 441 | { :single => nil.to_s, 442 | :multiple => "#{nil.to_s}#{nil.to_s}", 443 | :with_prefix => "prefix-#{nil.to_s}", 444 | :with_suffix => "#{nil.to_s}-suffix" }, 445 | ] 446 | actual_results = [] 447 | es = emit(config, use_v1, msgs) 448 | es.each_with_index do |(tag, time, record), i| 449 | actual_results << { 450 | :single => record["single"], 451 | :multiple => record["multiple"], 452 | :with_prefix => record["with_prefix"], 453 | :with_suffix => record["with_suffix"], 454 | } 455 | end 456 | assert_equal(expected_results, actual_results) 457 | end 458 | 459 | test "enabled autodetectction of value type with enable_ruby #{enable_ruby}" do 460 | config = %[ 461 | tag tag 462 | enable_ruby #{enable_ruby} 463 | auto_typecast true 464 | 465 | single ${source} 466 | multiple ${source}${source} 467 | with_prefix prefix-${source} 468 | with_suffix ${source}-suffix 469 | 470 | ] 471 | msgs = [ 472 | { "source" => "string" }, 473 | { "source" => 123 }, 474 | { "source" => [1, 2] }, 475 | { "source" => {a:1, b:2} }, 476 | { "source" => nil }, 477 | ] 478 | expected_results = [ 479 | { :single => "string", 480 | :multiple => "stringstring", 481 | :with_prefix => "prefix-string", 482 | :with_suffix => "string-suffix" }, 483 | { :single => 123, 484 | :multiple => "#{123.to_s}#{123.to_s}", 485 | :with_prefix => "prefix-#{123.to_s}", 486 | :with_suffix => "#{123.to_s}-suffix" }, 487 | { :single => [1, 2], 488 | :multiple => "#{[1, 2].to_s}#{[1, 2].to_s}", 489 | :with_prefix => "prefix-#{[1, 2].to_s}", 490 | :with_suffix => "#{[1, 2].to_s}-suffix" }, 491 | { :single => {a:1, b:2}, 492 | :multiple => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}", 493 | :with_prefix => "prefix-#{{a:1, b:2}.to_s}", 494 | :with_suffix => "#{{a:1, b:2}.to_s}-suffix" }, 495 | { :single => nil, 496 | :multiple => "#{nil.to_s}#{nil.to_s}", 497 | :with_prefix => "prefix-#{nil.to_s}", 498 | :with_suffix => "#{nil.to_s}-suffix" }, 499 | ] 500 | actual_results = [] 501 | es = emit(config, use_v1, msgs) 502 | es.each_with_index do |(tag, time, record), i| 503 | actual_results << { 504 | :single => record["single"], 505 | :multiple => record["multiple"], 506 | :with_prefix => record["with_prefix"], 507 | :with_suffix => record["with_suffix"], 508 | } 509 | end 510 | assert_equal(expected_results, actual_results) 511 | end 512 | 513 | test %Q[record["key"] with enable_ruby #{enable_ruby}] do 514 | config = %[ 515 | tag tag 516 | enable_ruby #{enable_ruby} 517 | auto_typecast true 518 | 519 | _timestamp ${record["@timestamp"]} 520 | _foo_bar ${record["foo.bar"]} 521 | 522 | ] 523 | d = create_driver(config, use_v1) 524 | record = { 525 | "foo.bar" => "foo.bar", 526 | "@timestamp" => 10, 527 | } 528 | es = emit(config, use_v1, [record]) 529 | es.each_with_index do |(tag, time, r), i| 530 | assert { r['_timestamp'] == record['@timestamp'] } 531 | assert { r['_foo_bar'] == record['foo.bar'] } 532 | end 533 | end 534 | end 535 | 536 | test 'unknown placeholder (enable_ruby no)' do 537 | config = %[ 538 | tag tag 539 | enable_ruby no 540 | 541 | message ${unknown} 542 | 543 | ] 544 | d = create_driver(config, use_v1) 545 | mock(d.instance.log).warn("record_reformer: unknown placeholder `${unknown}` found") 546 | d.run { d.emit({}, @time) } 547 | assert_equal 1, d.emits.size 548 | end 549 | 550 | test 'failed to expand record field (enable_ruby yes)' do 551 | config = %[ 552 | tag tag 553 | enable_ruby yes 554 | 555 | message ${unknown['bar']} 556 | 557 | ] 558 | d = create_driver(config, use_v1) 559 | mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything) 560 | d.run { d.emit({}, @time) } 561 | # emit, but nil value 562 | assert_equal 1, d.emits.size 563 | d.emits.each do |(tag, time, record)| 564 | assert_nil(record['message']) 565 | end 566 | end 567 | 568 | test 'failed to expand tag (enable_ruby yes)' do 569 | config = %[ 570 | tag ${unknown['bar']} 571 | enable_ruby yes 572 | ] 573 | d = create_driver(config, use_v1) 574 | mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything) 575 | d.run { d.emit({}, @time) } 576 | # nil tag message should not be emitted 577 | assert_equal 0, d.emits.size 578 | end 579 | 580 | test 'expand fields starting with @ (enable_ruby no)' do 581 | config = %[ 582 | tag tag 583 | enable_ruby no 584 | 585 | foo ${@timestamp} 586 | 587 | ] 588 | d = create_driver(config, use_v1) 589 | message = {"@timestamp" => "foo"} 590 | d.run { d.emit(message, @time) } 591 | d.emits.each do |(tag, time, record)| 592 | assert_equal message["@timestamp"], record["foo"] 593 | end 594 | end 595 | 596 | # https://github.com/sonots/fluent-plugin-record-reformer/issues/35 597 | test 'auto_typecast placeholder containing {} (enable_ruby yes)' do 598 | config = %[ 599 | tag tag 600 | enable_ruby yes 601 | auto_typecast yes 602 | 603 | foo ${record.map{|k,v|v}} 604 | 605 | ] 606 | d = create_driver(config, use_v1) 607 | message = {"@timestamp" => "foo"} 608 | d.run { d.emit(message, @time) } 609 | d.emits.each do |(tag, time, record)| 610 | assert_equal [message["@timestamp"]], record["foo"] 611 | end 612 | end 613 | 614 | test 'expand fields starting with @ (enable_ruby yes)' do 615 | config = %[ 616 | tag tag 617 | enable_ruby yes 618 | 619 | foo ${__send__("@timestamp")} 620 | 621 | ] 622 | d = create_driver(config, use_v1) 623 | message = {"@timestamp" => "foo"} 624 | d.run { d.emit(message, @time) } 625 | d.emits.each do |(tag, time, record)| 626 | assert_equal message["@timestamp"], record["foo"] 627 | end 628 | end 629 | end 630 | 631 | test "compatibility test (enable_ruby yes) (use_v1 #{use_v1})" do 632 | config = %[ 633 | tag tag 634 | enable_ruby yes 635 | auto_typecast yes 636 | 637 | _message prefix-${message}-suffix 638 | _time ${Time.at(time)} 639 | _number ${number == '-' ? 0 : number} 640 | _match ${/0x[0-9a-f]+/.match(hex)[0]} 641 | _timestamp ${__send__("@timestamp")} 642 | _foo_bar ${__send__('foo.bar')} 643 | 644 | ] 645 | d = create_driver(config, use_v1) 646 | record = { 647 | "number" => "-", 648 | "hex" => "0x10", 649 | "foo.bar" => "foo.bar", 650 | "@timestamp" => 10, 651 | "message" => "10", 652 | } 653 | es = emit(config, use_v1, [record]) 654 | es.each_with_index do |(tag, time, r), i| 655 | assert { r['_message'] == "prefix-#{record['message']}-suffix" } 656 | assert { r['_time'] == Time.at(@time) } 657 | assert { r['_number'] == 0 } 658 | assert { r['_match'] == record['hex'] } 659 | assert { r['_timestamp'] == record['@timestamp'] } 660 | assert { r['_foo_bar'] == record['foo.bar'] } 661 | end 662 | end 663 | end 664 | end 665 | --------------------------------------------------------------------------------