├── .gitignore ├── .travis.yml ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── example ├── blank.conf ├── blank.rb ├── blank2.conf ├── example1.conf ├── test1.conf ├── test1.rb ├── test2.conf └── test_in_out.rb ├── fluent-plugin-norikra.gemspec ├── lib └── fluent │ └── plugin │ ├── in_norikra.rb │ ├── norikra │ ├── config_section.rb │ ├── fetch_request.rb │ ├── input.rb │ ├── output.rb │ ├── query.rb │ ├── query_generator.rb │ ├── record_filter.rb │ └── target.rb │ ├── norikra_target.rb │ ├── out_norikra.rb │ └── out_norikra_filter.rb └── test ├── helper.rb ├── plugin ├── test_in_norikra.rb ├── test_out_norikra.rb └── test_out_norikra_filter.rb ├── test_config_section.rb ├── test_query.rb ├── test_query_generator.rb ├── test_record_filter.rb └── test_target.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: ruby 3 | rvm: 4 | - 2.1 5 | - 2.2 6 | - 2.3.0 7 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in fluent-plugin-norikra.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013- TAGOMORI Satoshi 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-plugin-norikra 2 | 3 | [Fluentd](http://fluentd.org/) plugins to send/receive events to/from Norikra server. 4 | 5 | Norikra is an open source server software provides "Stream Processing" with SQL, written in JRuby, runs on JVM, licensed under GPLv2. 6 | For more details, see: http://norikra.github.io/ . 7 | 8 | fluent-plugin-norikra has 3 plugins: in\_norikra, out\_norikra and out\_norikra\_filter. 9 | * in\_norikra 10 | * fetch events of query results from Norikra server 11 | * out\_norikra 12 | * send events to Norikra server 13 | * out\_norikra\_filter 14 | * launch Norikra server as child process dynamically, as needed 15 | * use Norikra server as event filter (like out\_exec\_filter) 16 | * register/execute queries for targets newly incoming 17 | 18 | # Setup 19 | 20 | `fluent-plugin-norikra` works with Norikra server, on same server with Fluentd, or anywhere reachable over network from Fluentd. 21 | For Norikra server setup, see: http://norikra.github.io/ . 22 | 23 | NOTES: 24 | * Fluentd and fluent-plugin-norikra requires CRuby (MatzRuby). 25 | * Norikra requires JRuby. 26 | 27 | To use out\_norikra\_filter with dynamic Norikra server launching, check actual path of command `norikra` under installed JRuby tree. (ex: `$HOME/.rbenv/versions/jruby-1.7.8/bin/norikra`) 28 | 29 | To use this plugin: 30 | 1. run `gem install fluent-plugin-norikra` or `fluent-gem install fluent-plugin-norikra` to install plugin 31 | 1. edit configuration files 32 | 1. execute fluentd 33 | 34 | # Configuration 35 | 36 | For variations, see `example` directory. 37 | 38 | ## NorikraOutput 39 | 40 | Sends events to remote Norikra server. Minimal configurations are: 41 | ```apache 42 | 43 | @type norikra 44 | norikra norikra.server.local:26571 45 | 46 | remove_tag_prefix data 47 | target_map_tag true # fluentd's tag 'data.event' -> norikra's target 'event' 48 | 49 | ``` 50 | 51 | NorikraOutput plugin opens Norikra's target for newly incoming tags. You can specify fields to include/exclude, and specify types of each fields, for each targets (and all targets by `default`). Definitions in `` overwrites `` specifications. 52 | 53 | ```apache 54 | 55 | @type norikra 56 | norikra norikra.server.local:26571 57 | 58 | target_map_tag true # fluentd's tag -> norikra's target 59 | remove_tag_prefix data 60 | 61 | # other options: 62 | # target_map_key KEY_NAME # use specified key's value as target in fluentd event 63 | # target_string STRING # use fixed target name specified 64 | # drop_error_record true # drop records chunk which includes records to occur ClientError on norikra server 65 | # # default: true 66 | # # (ex: specified (non-optional) fields missing or invalid value for specified type) 67 | # drop_server_error_record true # drop records chunk when any ServerError occurs 68 | # # default: false (to retry) 69 | 70 | 71 | include * # send all fields values to norikra 72 | exclude time # exclude 'time' field from sending event values 73 | # AND/OR 'include_regexp' and 'exclude_regexp' available 74 | field_integer seq # field 'seq' defined as integer for all targets 75 | escape_fieldname yes # Escape field name special chars (non alphabetical or numerical names) with underscore('_') 76 | # This is friendly for query access (ex: field.key1.cpu_total) 77 | # Default: no 78 | 79 | 80 | 81 | field_string name,address 82 | field_integer age 83 | field_float height,weight 84 | field_boolean superuser 85 | 86 | 87 | ``` 88 | 89 | With default setting, all fields are defined as 'string', so you must use `field_xxxx` parameters for numerical processing in query (For more details, see Norikra and Esper's documents). 90 | 91 | If fluentd's events has so many variations of sets of fields, you can specify not to include fields automatically, with `auto_field` option: 92 | 93 | ```apache 94 | 95 | @type norikra 96 | norikra norikra.server.local:26571 97 | 98 | target_map_tag true # fluentd's tag 'data.event' -> norikra's target 'event' 99 | remove_tag_prefix data 100 | 101 | 102 | auto_field false # norikra includes fields only used in queries. 103 | 104 | 105 | ``` 106 | 107 | Fields which are referred in queries are automatically registered on norikra server in spite of `auto_field false`. 108 | 109 | Use `time_key FIELDNAME` to include time of Fluentd's event into data field of Norikra (by milliseconds with Norikra/Esper's rule). This is useful for queries with `.win:ext_timed_batch(FIELD, PERIOD)` views. 110 | 111 | ** NOTE: and sections in NorikraOutput ignores sections. see NorikraFilterOutput ** 112 | 113 | ## NorikraInput 114 | 115 | Fetch events from Norikra server, and emits these into Fluentd itself. NorikraInput uses Norikra's API `event` (for queries), and `sweep` (for query groups). 116 | 117 | Minimal configurations: 118 | ```apache 119 | 120 | @type norikra 121 | norikra norikra.server.local:26571 122 | 123 | method sweep 124 | # target QUERY_GROUP_NAME # not specified => default query group 125 | tag query_name 126 | tag_prefix norikra.query 127 | # other options: 128 | # tag field FIELDNAME : tag by value with specified field name in output event 129 | # tag string STRING : fixed string specified 130 | interval 3s # interval to call api 131 | 132 | 133 | ``` 134 | 135 | Available `` methods are `event` and `sweep`. `target` parameter is handled as query name for `event`, and as query group name for `sweep`. 136 | ```apache 137 | 138 | @type norikra 139 | norikra norikra.server.local:26571 140 | 141 | method event 142 | target data_count_1hour 143 | tag string data.count.1hour 144 | interval 60m 145 | 146 | 147 | method event 148 | target data_count_5min 149 | tag string data.count.5min 150 | interval 5m 151 | 152 | 153 | method sweep 154 | target count_queries 155 | tag field target_name 156 | tag_prefix data.count.all 157 | interval 15s 158 | 159 | 160 | ``` 161 | 162 | ## NorikraFilterOutput 163 | 164 | NorikraFilterOutput has all features of both of NorikraInput and NorikraOutput, and also has additional features: 165 | * execute Norikra server 166 | * runs queries for newly incoming targets. 167 | 168 | If you runs Norikra as standalone process, better configurations are to use NorikraInput and NorikraOutput separately. NorikraFilterOutput is for simple aggregations and filterings. 169 | 170 | Configuration example to receive tags like `event.foo` and send norikra's target `foo`, and get count of its records per minute, and per hour with built-in Norikra server: 171 | ```apache 172 | 173 | @type norikra_filter 174 | 175 | path /home/username/.rbenv/versions/jruby-1.7.4/bin/norikra 176 | # opts -Xmx2g # options of 'norikra start' 177 | 178 | 179 | remove_tag_prefix event 180 | target_map_tag yes 181 | 182 | 183 | 184 | name count_min_${target} 185 | group count_query_group # or default when omitted 186 | expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minute) 187 | tag count.min.${target} 188 | fetch_interval 10s 189 | 190 | 191 | name count_hour_${target} 192 | group count_query_group 193 | expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 hour) 194 | tag count.hour.${target} 195 | 196 | 197 | 198 | ``` 199 | 200 | Results of queries automatically registered by NorikraFilterOutput with `tag` parameter, will be fetched automatically by this plugin, and re-emitted into Fluentd itself. 201 | 202 | Other all options are available as same as NorikraInput and NorikraOutput. ``, `` and `` sections, `auto_field`, `include|exclude` and `field_xxxx` specifiers for targets and parameters for `` sections. 203 | 204 | ### Input event data filtering 205 | 206 | If you want send known fields only, specify `exclude *` and `include` or `include_regexp` like this: 207 | 208 | 209 | exclude * 210 | include path,status,method,bytes,rhost,referer,agent,duration 211 | include_pattern ^(query_|header_).* 212 | 213 | # ... 214 | 215 | 216 | Or you can specify to include as default, and exclude known some fields: 217 | 218 | 219 | include * 220 | exclude user_secret 221 | include_pattern ^(header_).* 222 | 223 | # ... 224 | 225 | 226 | NOTE: These configurations of `` section overwrites of configurations in `` section. 227 | 228 | ### Target mapping 229 | 230 | Norikra's target (like table name) can be generated from: 231 | 232 | * tag 233 | * one target per one tag 234 | * `target_map_tag yes` 235 | * value of specified field 236 | * targets from values in specified field of record, dynamically 237 | * `target_map_key foo` 238 | * fixed string (in configuration file) 239 | * all records are sent in single target 240 | * `target_string from_fluentd` 241 | 242 | # TODO 243 | 244 | * write about these topics 245 | * error logs for new target, success logs of retry 246 | 247 | # Copyright 248 | 249 | * Copyright (c) 2013- TAGOMORI Satoshi (tagomoris) 250 | * License 251 | * Apache License, version 2.0 252 | 253 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | require 'rake/testtask' 4 | Rake::TestTask.new(:test) do |test| 5 | test.libs << 'lib' << 'test' 6 | test.pattern = 'test/**/test_*.rb' 7 | test.verbose = true 8 | end 9 | 10 | task :default => :test 11 | -------------------------------------------------------------------------------- /example/blank.conf: -------------------------------------------------------------------------------- 1 | 2 | type forward 3 | 4 | 5 | 6 | type norikra_filter 7 | norikra localhost:26571 # this is default 8 | 9 | path /Users/tagomoris/.rbenv/versions/jruby-1.7.8/bin/norikra # $HOME 10 | 11 | 12 | remove_tag_prefix event 13 | target_map_tag yes 14 | 15 | -------------------------------------------------------------------------------- /example/blank.rb: -------------------------------------------------------------------------------- 1 | source { 2 | type "forward" 3 | } 4 | 5 | home_dir = ::Object::ENV['HOME'] 6 | 7 | match('event.*') { 8 | type "norikra_filter" 9 | norikra "localhost:26571" 10 | server { 11 | path "#{home_dir}/.rbenv/versions/jruby-1.7.8/bin/norikra" 12 | } 13 | remove_tag_prefix "event" 14 | target_map_tag true 15 | } 16 | -------------------------------------------------------------------------------- /example/blank2.conf: -------------------------------------------------------------------------------- 1 | 2 | type forward 3 | 4 | 5 | 6 | type norikra_filter 7 | norikra localhost:26571 # this is default 8 | 9 | remove_tag_prefix event 10 | target_map_tag yes 11 | 12 | 13 | auto_field false 14 | 15 | 16 | -------------------------------------------------------------------------------- /example/example1.conf: -------------------------------------------------------------------------------- 1 | 2 | type norikra_filter 3 | norikra localhost:26571 4 | 5 | 6 | path /home/user/.rbenv/versions/jruby-1.7.4/bin/norikra 7 | 8 | 9 | remove_tag_prefix event 10 | 11 | target_map_tag yes 12 | # or 13 | # target_map_key KEYNAME 14 | # or 15 | # target_string TARGET_STRING 16 | 17 | 18 | include * 19 | exclude yyyymmdd,hhmmss 20 | exclude_regexp f_.* 21 | # OR 22 | # exclude * 23 | # include foo,bar,baz 24 | # include_regexp status.* 25 | field_boolean flag 26 | field_integer status,duration,bytes 27 | 28 | 29 | name pv_${target} 30 | expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE not flag 31 | tag pv.${target} 32 | # group pv_query_group # default: nil (default group) 33 | fetch_interval 15s # default -> time_batch / 4 ? -> (none) -> 60s 34 | # fetch_interval is ignored when section specified 35 | 36 | 37 | name errors_${target} 38 | expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE status >= 500 39 | tag errors.${target} 40 | fetch_interval 15s 41 | 42 | 43 | 44 | 45 | field_int display 46 | 47 | 48 | name search_words 49 | expression SELECT count(distinct query_search) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE query_search.length() > 0 50 | tag search.words 51 | 52 | 53 | name search_rate 54 | expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minutes) WHERE query_search.length() > 0 55 | tag search.rate 56 | 57 | 58 | 59 | 60 | method sweep # listen(not implemented) 61 | tag query_name 62 | # tag field FIELDNAME 63 | # tag string TAG_STRING 64 | tag_prefix cep 65 | interval 5s 66 | 67 | -------------------------------------------------------------------------------- /example/test1.conf: -------------------------------------------------------------------------------- 1 | 2 | type forward 3 | 4 | 5 | 6 | type norikra_filter 7 | norikra localhost:26571 8 | 9 | path /Users/tagomoris/.rbenv/versions/jruby-1.7.8/bin/norikra # $HOME 10 | 11 | 12 | remove_tag_prefix test 13 | target_map_tag yes 14 | 15 | 16 | 17 | name count_${target} 18 | expression SELECT '${target}' as target,count(*) AS cnt FROM ${target}.win:time_batch(30 sec) 19 | 20 | 21 | 22 | method sweep 23 | tag field target 24 | tag_prefix count 25 | interval 5s 26 | 27 | 28 | 29 | 30 | type null 31 | 32 | 33 | 34 | type stdout 35 | 36 | -------------------------------------------------------------------------------- /example/test1.rb: -------------------------------------------------------------------------------- 1 | source { 2 | type :forward 3 | } 4 | 5 | home_dir = ::Object::ENV['HOME'] 6 | 7 | match('test.*') { 8 | type :norikra_filter 9 | norikra 'localhost:26571' 10 | server { 11 | path "#{home_dir}/.rbenv/versions/jruby-1.7.8/bin/norikra" 12 | } 13 | 14 | remove_tag_prefix 'test' 15 | target_map_tag true 16 | 17 | default { 18 | query { 19 | name "count_${target}" 20 | expression "SELECT '${target}' as target,count(*) AS cnt FROM ${target}.win:time_batch(30 sec)" 21 | group "testing" 22 | tag "count.x.${target}" 23 | } 24 | } 25 | 26 | fetch { 27 | method :sweep 28 | tag 'field target' 29 | tag_prefix 'count' 30 | interval 5 31 | } 32 | } 33 | 34 | match('fluent.*') { 35 | type :null 36 | } 37 | 38 | match('**') { 39 | type :stdout 40 | } 41 | -------------------------------------------------------------------------------- /example/test2.conf: -------------------------------------------------------------------------------- 1 | 2 | type forward 3 | 4 | 5 | 6 | type norikra_filter 7 | norikra localhost:26571 8 | target_map_tag yes 9 | drop_server_error_record yes 10 | drop_error_record yes 11 | 12 | 13 | include * 14 | exclude hhmmss 15 | 16 | 17 | 18 | field_string vhost,path,method,referer,rhost,userlabel,agent,ua_name,ua_category,ua_os,ua_version,ua_vendor 19 | field_integer status,bytes,duration 20 | field_boolean FLAG,status_redirection,status_errors,rhost_internal,suffix_miscfile,suffix_imagefile,agent_bot 21 | 22 | 23 | 24 | method sweep 25 | tag query_name 26 | tag_prefix norikra.event 27 | interval 5s 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /example/test_in_out.rb: -------------------------------------------------------------------------------- 1 | source { 2 | type :forward 3 | } 4 | 5 | match('test.*') { 6 | type :norikra 7 | norikra 'localhost:26571' 8 | 9 | remove_tag_prefix 'test' 10 | target_map_tag true 11 | 12 | default { 13 | include '*' 14 | exclude 'hhmmss' 15 | } 16 | 17 | target('data') { 18 | field_string 'name' 19 | field_integer 'age' 20 | } 21 | } 22 | 23 | source { 24 | type :norikra 25 | 26 | fetch { 27 | method :sweep 28 | # target => nil (group: default) 29 | tag 'field target' 30 | tag_prefix 'norikra.query' 31 | interval 3 32 | } 33 | 34 | fetch { 35 | method :event 36 | target 'data_count' 37 | tag 'string norikra.count.data' 38 | interval 5 39 | } 40 | } 41 | 42 | match('fluent.**') { 43 | type :null 44 | } 45 | 46 | match('**') { 47 | type :stdout 48 | } 49 | -------------------------------------------------------------------------------- /fluent-plugin-norikra.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "fluent-plugin-norikra" 5 | spec.version = "0.4.4" 6 | spec.authors = ["TAGOMORI Satoshi"] 7 | spec.email = ["tagomoris@gmail.com"] 8 | spec.description = %q{process events on fluentd with SQL like query, with built-in Norikra server if needed.} 9 | spec.summary = %q{Fluentd plugin to do CEP with norikra} 10 | spec.homepage = "https://github.com/norikra/fluent-plugin-norikra" 11 | spec.license = "Apache-2.0" 12 | 13 | spec.files = `git ls-files`.split($/) 14 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 15 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 16 | spec.require_paths = ["lib"] 17 | 18 | spec.add_runtime_dependency "rack", "~> 1.6" # to prevent to install rack 2.0 (it requires ruby 2.2.2 or later) 19 | spec.add_runtime_dependency "norikra-client", ">= 1.4.0" 20 | spec.add_runtime_dependency "fluentd", "< 0.14.0" 21 | 22 | spec.add_development_dependency "bundler", "~> 1.3" 23 | spec.add_development_dependency "rake" 24 | spec.add_development_dependency "test-unit" 25 | end 26 | -------------------------------------------------------------------------------- /lib/fluent/plugin/in_norikra.rb: -------------------------------------------------------------------------------- 1 | require 'fluent/input' 2 | require_relative 'norikra/input' 3 | 4 | require 'norikra-client' 5 | 6 | module Fluent 7 | class NorikraInput < Fluent::Input 8 | include Fluent::NorikraPlugin::InputMixin 9 | 10 | Fluent::Plugin.register_input('norikra', self) 11 | 12 | config_param :norikra, :string, :default => 'localhost:26571' 13 | 14 | config_param :connect_timeout, :integer, :default => nil 15 | config_param :send_timeout, :integer, :default => nil 16 | config_param :receive_timeout, :integer, :default => nil 17 | 18 | # tags 19 | # 20 | # method event 21 | # target QUERY_NAME 22 | # interval 5s 23 | # tag query_name 24 | # # tag field FIELDNAME 25 | # # tag string FIXED_STRING 26 | # tag_prefix norikra.event # actual tag: norikra.event.QUERYNAME 27 | # 28 | # 29 | # method sweep 30 | # target QUERY_GROUP # or unspecified => default 31 | # interval 60s 32 | # tag field group_by_key 33 | # tag_prefix norikra.query 34 | # 35 | 36 | # Define `log` method for v0.10.42 or earlier 37 | unless method_defined?(:log) 38 | define_method("log") { $log } 39 | end 40 | 41 | def configure(conf) 42 | super 43 | 44 | @host,@port = @norikra.split(':', 2) 45 | @port = @port.to_i 46 | 47 | conf.elements.each do |element| 48 | case element.name 49 | when 'fetch' 50 | # ignore: processed in InputMixin, and set @fetch_queue 51 | else 52 | raise Fluent::ConfigError, "unknown configuration section name for this plugin: #{element.name}" 53 | end 54 | end 55 | 56 | setup_input(conf) 57 | end 58 | 59 | def client(opts={}) 60 | Norikra::Client.new(@host, @port, { 61 | :connect_timeout => opts[:connect_timeout] || @connect_timeout, 62 | :send_timeout => opts[:send_timeout] || @send_timeout, 63 | :receive_timeout => opts[:receive_timeout] || @receive_timeout, 64 | }) 65 | end 66 | 67 | def start 68 | super 69 | start_input 70 | end 71 | 72 | def shutdown 73 | stop_input 74 | shutdown_input 75 | end 76 | 77 | def fetchable? 78 | true 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/config_section.rb: -------------------------------------------------------------------------------- 1 | module Fluent::NorikraPlugin 2 | class ConfigSection 3 | attr_accessor :target, :target_matcher 4 | attr_accessor :auto_field, :time_key, :escape_fieldname 5 | attr_accessor :filter_params, :field_definitions, :query_generators 6 | 7 | def initialize(section, enable_auto_query=true) 8 | @target = nil 9 | @target_matcher = nil 10 | if section.name == 'default' 11 | # nil 12 | elsif section.name == 'target' 13 | # unescaped target name (tag style with dots) 14 | @target = section.arg 15 | @target_matcher = Fluent::GlobMatchPattern.new(section.arg) 16 | else 17 | raise ArgumentError, "invalid section for this class, #{section.name}: ConfigSection" 18 | end 19 | 20 | @auto_field = Fluent::Config.bool_value(section['auto_field']) 21 | @time_key = section['time_key'] 22 | @escape_fieldname = Fluent::Config.bool_value(section['escape_fieldname']) 23 | 24 | @filter_params = { 25 | :include => section['include'], 26 | :include_regexp => section['include_regexp'], 27 | :exclude => section['exclude'], 28 | :exclude_regexp => section['exclude_regexp'] 29 | } 30 | @field_definitions = { 31 | :string => (section['field_string'] || '').split(','), 32 | :boolean => (section['field_boolean'] || '').split(','), 33 | :integer => (section['field_integer'] || '').split(','), 34 | :float => (section['field_float'] || '').split(','), 35 | } 36 | 37 | @query_generators = [] 38 | section.elements.each do |element| 39 | if element.name == 'query' && enable_auto_query 40 | opt = {} 41 | if element.has_key?('fetch_interval') 42 | opt['fetch_interval'] = Fluent::Config.time_value(element['fetch_interval']) 43 | end 44 | @query_generators.push(QueryGenerator.new(element['name'], element['group'], element['expression'], element['tag'], opt)) 45 | end 46 | end 47 | end 48 | 49 | def +(other) 50 | if other.nil? 51 | other = self.class.new(Fluent::Config::Element.new('target', 'dummy', {}, [])) 52 | end 53 | r = self.class.new(Fluent::Config::Element.new('target', (other.target ? other.target : self.target), {}, [])) 54 | r.auto_field = (other.auto_field.nil? ? self.auto_field : other.auto_field) 55 | r.time_key = other.time_key || self.time_key 56 | 57 | others_filter = {} 58 | other.filter_params.keys.each do |k| 59 | others_filter[k] = other.filter_params[k] if other.filter_params[k] 60 | end 61 | r.filter_params = self.filter_params.merge(others_filter) 62 | r.field_definitions = { 63 | :string => self.field_definitions[:string] + other.field_definitions[:string], 64 | :boolean => self.field_definitions[:boolean] + other.field_definitions[:boolean], 65 | :integer => self.field_definitions[:integer] + other.field_definitions[:integer], 66 | :float => self.field_definitions[:float] + other.field_definitions[:float], 67 | } 68 | r.query_generators = self.query_generators + other.query_generators 69 | r 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/fetch_request.rb: -------------------------------------------------------------------------------- 1 | module Fluent::NorikraPlugin 2 | class FetchRequest 3 | METHODS = [:event, :sweep] 4 | TAG_TYPES = ['query_name', 'field', 'string'] 5 | 6 | attr_accessor :method, :target, :interval, :tag_generator, :tag_prefix 7 | attr_accessor :time 8 | 9 | def initialize(method, target, interval, tag_type, tag_arg, tag_prefix) 10 | raise ArgumentError, "unknown method '#{method}'" unless METHODS.include?(method.to_sym) 11 | 12 | @method = method.to_sym 13 | @target = target 14 | @interval = interval.to_i 15 | 16 | raise ArgumentError, "unknown tag type specifier '#{tag_type}'" unless TAG_TYPES.include?(tag_type.to_s) 17 | raw_tag_prefix = tag_prefix.to_s 18 | if (! raw_tag_prefix.empty?) && (! raw_tag_prefix.end_with?('.')) # tag_prefix specified, and ends without dot 19 | raw_tag_prefix += '.' 20 | end 21 | 22 | @tag_generator = case tag_type.to_s 23 | when 'query_name' then lambda{|query_name,record| raw_tag_prefix + query_name} 24 | when 'field' then lambda{|query_name,record| raw_tag_prefix + (record[tag_arg] || 'NULL')} 25 | when 'string' then lambda{|query_name,record| raw_tag_prefix + tag_arg} 26 | else 27 | raise "bug" 28 | end 29 | @time = Time.now + 1 # should be fetched soon ( 1sec later ) 30 | end 31 | 32 | def <=>(other) 33 | self.time <=> other.time 34 | end 35 | 36 | def next! 37 | @time = Time.now + @interval 38 | end 39 | 40 | # returns hash: { tag => [[time, record], ...], ... } 41 | def fetch(client) 42 | # events { query_name => [[time, record], ...], ... } 43 | events = case @method 44 | when :event then event(client) 45 | when :sweep then sweep(client) 46 | else 47 | raise "BUG: unknown method: #{@method}" 48 | end 49 | 50 | output = {} 51 | 52 | events.keys.each do |query_name| 53 | events[query_name].each do |time, record| 54 | tag = @tag_generator.call(query_name, record) 55 | output[tag] ||= [] 56 | output[tag] << [time, record] 57 | end 58 | end 59 | 60 | output 61 | end 62 | 63 | def event(client) 64 | events = client.event(@target) # [[time(int from epoch), event], ...] 65 | {@target => events} 66 | end 67 | 68 | def sweep(client) 69 | client.sweep(@target) # {query_name => event_array, ...} 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/input.rb: -------------------------------------------------------------------------------- 1 | require_relative 'fetch_request' 2 | 3 | module Fluent::NorikraPlugin 4 | module InputMixin 5 | # 6 | # method event 7 | # target QUERY_NAME 8 | # interval 5s 9 | # tag query_name 10 | # # tag field FIELDNAME 11 | # # tag string FIXED_STRING 12 | # tag_prefix norikra.event # actual tag: norikra.event.QUERYNAME 13 | # 14 | # 15 | # method sweep 16 | # target QUERY_GROUP # or unspecified => default 17 | # interval 60s 18 | # tag field group_by_key 19 | # tag_prefix norikra.query 20 | # 21 | 22 | def setup_input(conf) 23 | @fetch_queue = [] 24 | 25 | conf.elements.each do |e| 26 | next unless e.name == 'fetch' 27 | method = e['method'] 28 | target = e['target'] 29 | interval_str = e['interval'] 30 | tag = e['tag'] 31 | unless method && interval_str && tag 32 | raise Fluent::ConfigError, " must be specified with method/interval/tag" 33 | end 34 | if method == 'event' and target.nil? 35 | raise Fluent::ConfigError, " method 'event' requires 'target' for fetch target query name" 36 | end 37 | 38 | interval = Fluent::Config.time_value(interval_str) 39 | tag_type, tag_arg = tag.split(/ /, 2) 40 | req = FetchRequest.new(method, target, interval, tag_type, tag_arg, e['tag_prefix']) 41 | 42 | @fetch_queue << req 43 | end 44 | 45 | @fetch_queue_mutex = Mutex.new 46 | end 47 | 48 | def start_input 49 | @fetch_worker_running = true 50 | @fetch_thread = Thread.new(&method(:fetch_worker)) 51 | end 52 | 53 | def stop_input 54 | @fetch_worker_running = false 55 | end 56 | 57 | def shutdown_input 58 | # @fetch_thread.kill 59 | @fetch_thread.join 60 | end 61 | 62 | def insert_fetch_queue(request) 63 | @fetch_queue_mutex.synchronize do 64 | request.next! if request.time < Time.now 65 | # if @fetch_queue.size > 0 66 | # next_pos = @fetch_queue.bsearch{|req| req.time > request.time} 67 | # @fetch_queue.insert(next_pos, request) 68 | # else 69 | # @fetch_queue.push(request) 70 | # end 71 | @fetch_queue.push(request) 72 | @fetch_queue.sort! 73 | end 74 | rescue => e 75 | log.error "unknown log encountered", :error_class => e.class, :message => e.message 76 | end 77 | 78 | def fetch_worker 79 | while sleep(1) 80 | break unless @fetch_worker_running 81 | next unless fetchable? 82 | next if @fetch_queue.first.nil? || @fetch_queue.first.time > Time.now 83 | 84 | now = Time.now 85 | while @fetch_queue.first.time <= now 86 | req = @fetch_queue.shift 87 | 88 | begin 89 | data = req.fetch(client()) 90 | rescue => e 91 | log.error "failed to fetch", :norikra => "#{@host}:#{@port}", :method => req.method, :target => req.target, :error => e.class, :message => e.message 92 | end 93 | 94 | if data 95 | data.each do |tag, event_array| 96 | next unless event_array 97 | event_array.each do |time,event| 98 | begin 99 | router.emit(tag, time, event) 100 | rescue => e 101 | log.error "failed to emit event from norikra query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message, :tag => tag, :record => event 102 | end 103 | end 104 | end 105 | end 106 | 107 | insert_fetch_queue(req) 108 | end 109 | end 110 | end 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/output.rb: -------------------------------------------------------------------------------- 1 | require_relative 'config_section' 2 | require_relative 'query' 3 | require_relative 'query_generator' 4 | require_relative 'record_filter' 5 | require_relative 'target' 6 | 7 | require_relative 'fetch_request' 8 | 9 | module Fluent::NorikraPlugin 10 | module OutputMixin 11 | def setup_output(conf, enable_auto_query) 12 | @enable_auto_query = enable_auto_query 13 | 14 | @target_generator = case 15 | when @target_string 16 | lambda {|tag,record| @target_string} 17 | when @target_map_key 18 | lambda {|tag,record| record[@target_map_key]} 19 | when @target_map_tag 20 | lambda {|tag,record| tag.gsub(/^#{@remove_tag_prefix}(\.)?/, '')} 21 | else 22 | raise Fluent::ConfigError, "no one way specified to decide target" 23 | end 24 | 25 | # target map already prepared (opened, and related queries registered) 26 | @target_map = {} # 'target' => instance of Fluent::NorikraPlugin::Target 27 | 28 | # for conversion from query_name to tag 29 | @query_map = {} # 'query_name' => instance of Fluent::NorikraPlugin::Query 30 | 31 | @default_target = ConfigSection.new(Fluent::Config::Element.new('default', nil, {}, []), @enable_auto_query) 32 | @config_targets = {} 33 | 34 | conf.elements.each do |element| 35 | case element.name 36 | when 'default' 37 | @default_target = ConfigSection.new(element, @enable_auto_query) 38 | when 'target' 39 | c = ConfigSection.new(element, @enable_auto_query) 40 | @config_targets[c.target] = c 41 | end 42 | end 43 | 44 | @target_mutex = Mutex.new 45 | end 46 | 47 | def start_output 48 | @register_worker_running = true 49 | @register_queue = [] 50 | @registered_targets = {} 51 | @register_thread = Thread.new(&method(:register_worker)) 52 | end 53 | 54 | def stop_output 55 | @register_worker_running = false 56 | end 57 | 58 | def shutdown_output 59 | # @register_thread.kill 60 | @register_thread.join 61 | end 62 | 63 | def prepared?(target_names) 64 | fetchable? && target_names.reduce(true){|r,t| r && @target_map.values.any?{|target| target.escaped_name == t}} 65 | end 66 | 67 | def fetch_event_registration(query) 68 | return if query.tag.nil? || query.tag.empty? 69 | req = FetchRequest.new(:event, query.name, query.interval, 'string', query.tag, nil) 70 | insert_fetch_queue(req) 71 | end 72 | 73 | def register_worker 74 | while sleep(0.25) 75 | break unless @register_worker_running 76 | next unless fetchable? 77 | 78 | c = client() 79 | 80 | targets = @register_queue.shift(10) 81 | targets.each do |t| 82 | next if @target_map[t.name] 83 | 84 | log.debug "Preparing norikra target #{t.name} on #{@host}:#{@port}" 85 | if prepare_target(c, t) 86 | log.debug "success to prepare target #{t.name} on #{@host}:#{@port}" 87 | 88 | if @enable_auto_query 89 | raise "bug" unless self.respond_to?(:insert_fetch_queue) 90 | 91 | t.queries.each do |query| 92 | @query_map[query.name] = query 93 | fetch_event_registration(query) 94 | end 95 | end 96 | @target_map[t.name] = t 97 | @registered_targets.delete(t.name) 98 | else 99 | log.error "Failed to prepare norikra data for target:#{t.name}" 100 | @register_queue.push(t) 101 | end 102 | end 103 | end 104 | end 105 | 106 | def prepare_target(client, target) 107 | # target open and reserve fields 108 | log.debug "Going to prepare about target" 109 | begin 110 | unless client.targets.include?(target.escaped_name) 111 | log.debug "opening target #{target.escaped_name}" 112 | client.open(target.escaped_name, target.reserve_fields, target.auto_field) 113 | log.debug "opening target #{target.escaped_name}, done." 114 | end 115 | 116 | reserving = target.reserve_fields 117 | reserved = [] 118 | client.fields(target.escaped_name).each do |field| 119 | if reserving[field['name']] 120 | reserved.push(field['name']) 121 | if reserving[field['name']] != field['type'] 122 | log.warn "field type mismatch, reserving:#{reserving[field['name']]} but reserved:#{field['type']}" 123 | end 124 | end 125 | end 126 | 127 | reserving.each do |fieldname,type| 128 | client.reserve(target.escaped_name, fieldname, type) unless reserved.include?(fieldname) 129 | end 130 | rescue => e 131 | log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message 132 | return false 133 | end 134 | 135 | # query registration 136 | begin 137 | registered = Hash[client.queries.map{|q| [q['name'], q['expression']]}] 138 | target.queries.each do |query| 139 | if registered.has_key?(query.name) # query already registered 140 | if registered[query.name] != query.expression 141 | log.warn "query name and expression mismatch, check norikra server status. target query name:#{query.name}" 142 | end 143 | next 144 | end 145 | client.register(query.name, query.group, query.expression) 146 | 147 | @query_map[query.name] = query 148 | fetch_event_registration(query) 149 | end 150 | rescue => e 151 | log.warn "failed to register query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message 152 | end 153 | end 154 | 155 | def format_stream(tag, es) 156 | tobe_registered_target_names = [] 157 | 158 | out = '' 159 | 160 | es.each do |time,record| 161 | target = @target_generator.call(tag, record) 162 | 163 | tgt = @target_mutex.synchronize do 164 | t = @target_map[target] 165 | unless t 166 | unless tobe_registered_target_names.include?(target) 167 | conf = @config_targets[target] 168 | unless conf 169 | @config_targets.values.each do |c| 170 | if c.target_matcher.match(target) 171 | conf = c 172 | break 173 | end 174 | end 175 | end 176 | t = Target.new(target, @default_target + conf) 177 | @registered_targets[target] = t 178 | @register_queue.push(t) 179 | tobe_registered_target_names.push(target) 180 | end 181 | t = @registered_targets[target] 182 | end 183 | t 184 | end 185 | 186 | event = tgt.filter(time, record) 187 | 188 | out << [tgt.escaped_name,event].to_msgpack 189 | end 190 | 191 | out 192 | end 193 | 194 | def write(chunk) 195 | events_map = {} # target => [event] 196 | chunk.msgpack_each do |target, event| 197 | events_map[target] ||= [] 198 | events_map[target].push(event) 199 | end 200 | 201 | unless prepared?(events_map.keys) 202 | raise RuntimeError, "norikra server is not ready for this targets: #{events_map.keys.join(',')}" 203 | end 204 | 205 | c = client() 206 | 207 | events_map.each do |target, events| 208 | begin 209 | c.send(target, events) 210 | rescue Norikra::RPC::ClientError => e 211 | raise unless @drop_error_record 212 | log.warn "Norikra server reports ClientError, and dropped", target: target, message: e.message 213 | rescue Norikra::RPC::ServerError => e 214 | raise unless @drop_server_error_record 215 | log.warn "Norikra server reports ServerError, and dropped", target: target, message: e.message 216 | rescue Norikra::RPC::ServiceUnavailableError => e 217 | raise unless @drop_when_shutoff 218 | log.warn "Norikra server is now in Shutoff mode, and dropped", target: target, message: e.message 219 | end 220 | end 221 | end 222 | 223 | end 224 | end 225 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/query.rb: -------------------------------------------------------------------------------- 1 | module Fluent::NorikraPlugin 2 | class Query 3 | attr_accessor :name, :group, :expression, :tag, :interval 4 | 5 | def initialize(name, group, expression, tag, interval) 6 | @name = name 7 | @group = group 8 | @expression = expression 9 | @tag = tag 10 | @interval = interval 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/query_generator.rb: -------------------------------------------------------------------------------- 1 | module Fluent::NorikraPlugin 2 | class QueryGenerator 3 | attr_reader :fetch_interval 4 | 5 | def initialize(name_template, group, expression_template, tag_template, opts={}) 6 | @name_template = name_template || '' 7 | @group = group 8 | @expression_template = expression_template || '' 9 | @tag_template = tag_template || '' 10 | if @name_template.empty? || @expression_template.empty? 11 | raise Fluent::ConfigError, "query's name/expression must be specified" 12 | end 13 | @fetch_interval = case 14 | when opts['fetch_interval'] 15 | Fluent::Config.time_value(opts['fetch_interval']) 16 | when @expression_template =~ /\.win:time_batch\(([^\)]+)\)/ 17 | y,mon,w,d,h,m,s,msec = self.class.parse_time_period($1) 18 | (h * 3600 + m * 60 + s) / 5 19 | else 20 | 60 21 | end 22 | end 23 | 24 | def generate(name, escaped) 25 | Fluent::NorikraPlugin::Query.new( 26 | self.class.replace_target(name, @name_template), 27 | @group, 28 | self.class.replace_target(escaped, @expression_template), 29 | self.class.replace_target(name, @tag_template), 30 | @fetch_interval 31 | ) 32 | end 33 | 34 | def self.replace_target(t, str) 35 | str.gsub('${target}', t) 36 | end 37 | 38 | def self.parse_time_period(string) 39 | #### http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html/epl_clauses.html#epl-syntax-time-periods 40 | # time-period : [year-part] [month-part] [week-part] [day-part] [hour-part] [minute-part] [seconds-part] [milliseconds-part] 41 | # year-part : (number|variable_name) ("years" | "year") 42 | # month-part : (number|variable_name) ("months" | "month") 43 | # week-part : (number|variable_name) ("weeks" | "week") 44 | # day-part : (number|variable_name) ("days" | "day") 45 | # hour-part : (number|variable_name) ("hours" | "hour") 46 | # minute-part : (number|variable_name) ("minutes" | "minute" | "min") 47 | # seconds-part : (number|variable_name) ("seconds" | "second" | "sec") 48 | # milliseconds-part : (number|variable_name) ("milliseconds" | "millisecond" | "msec") 49 | m = /^\s*(\d+ years?)? ?(\d+ months?)? ?(\d+ weeks?)? ?(\d+ days?)? ?(\d+ hours?)? ?(\d+ (?:min|minute|minutes))? ?(\d+ (?:sec|second|seconds))? ?(\d+ (?:msec|millisecond|milliseconds))?/.match(string) 50 | years = (m[1] || '').split(' ',2).first.to_i 51 | months = (m[2] || '').split(' ',2).first.to_i 52 | weeks = (m[3] || '').split(' ',2).first.to_i 53 | days = (m[4] || '').split(' ',2).first.to_i 54 | hours = (m[5] || '').split(' ',2).first.to_i 55 | minutes = (m[6] || '').split(' ',2).first.to_i 56 | seconds = (m[7] || '').split(' ',2).first.to_i 57 | msecs = (m[8] || '').split(' ',2).first.to_i 58 | return [years, months, weeks, days, hours, minutes, seconds, msecs] 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/record_filter.rb: -------------------------------------------------------------------------------- 1 | module Fluent::NorikraPlugin 2 | class RecordFilter 3 | attr_reader :default_policy, :include_fields, :include_regexp, :exclude_fields, :exclude_regexp 4 | 5 | def initialize(include='', include_regexp='', exclude='', exclude_regexp='') 6 | include ||= '' 7 | include_regexp ||= '' 8 | exclude ||= '' 9 | exclude_regexp ||= '' 10 | 11 | @default_policy = nil 12 | if include == '*' && exclude == '*' 13 | raise Fluent::ConfigError, "invalid configuration, both of 'include' and 'exclude' are '*'" 14 | end 15 | if include.empty? && include_regexp.empty? && exclude.empty? && exclude_regexp.empty? # assuming "include *" 16 | @default_policy = :include 17 | elsif exclude.empty? && exclude_regexp.empty? || exclude == '*' # assuming "exclude *" 18 | @default_policy = :exclude 19 | elsif include.empty? && include_regexp.empty? || include == '*' # assuming "include *" 20 | @default_policy = :include 21 | else 22 | raise Fluent::ConfigError, "unknown default policy. specify 'include *' or 'exclude *'" 23 | end 24 | 25 | @include_fields = nil 26 | @include_regexp = nil 27 | @exclude_fields = nil 28 | @exclude_regexp = nil 29 | 30 | if @default_policy == :exclude 31 | @include_fields = include.split(',') 32 | @include_regexp = Regexp.new(include_regexp) unless include_regexp.empty? 33 | if @include_fields.empty? && @include_regexp.nil? 34 | raise Fluent::ConfigError, "no one fields specified. specify 'include' or 'include_regexp'" 35 | end 36 | else 37 | @exclude_fields = exclude.split(',') 38 | @exclude_regexp = Regexp.new(exclude_regexp) unless exclude_regexp.empty? 39 | end 40 | end 41 | 42 | def filter(record) 43 | if @default_policy == :include 44 | if @exclude_fields.empty? && @exclude_regexp.nil? 45 | record 46 | else 47 | record = record.dup 48 | record.keys.each do |f| 49 | record.delete(f) if @exclude_fields.include?(f) || @exclude_regexp && @exclude_regexp.match(f) 50 | end 51 | record 52 | end 53 | else # default policy exclude 54 | data = {} 55 | record.keys.each do |f| 56 | data[f] = record[f] if @include_fields.include?(f) || @include_regexp && @include_regexp.match(f) 57 | end 58 | data 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra/target.rb: -------------------------------------------------------------------------------- 1 | module Fluent::NorikraPlugin 2 | class Target 3 | attr_accessor :name, :auto_field, :time_key, :fields, :queries 4 | attr_reader :escaped_name 5 | 6 | def self.escape(src) 7 | if src.nil? || src.empty? 8 | return 'FluentdGenerated' 9 | end 10 | 11 | dst = src.gsub(/[^_a-zA-Z0-9]/, '_') 12 | unless dst =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/ 13 | unless dst =~ /^[a-zA-Z]/ 14 | dst = 'Fluentd' + dst 15 | end 16 | unless dst =~ /[a-zA-Z0-9]$/ 17 | dst = dst + 'Generated' 18 | end 19 | end 20 | dst 21 | end 22 | 23 | def initialize(target, config) 24 | @name = target 25 | @escaped_name = self.class.escape(@name) 26 | @auto_field = config.auto_field.nil? ? true : config.auto_field 27 | @time_key = config.time_key 28 | @escape_fieldname = config.escape_fieldname 29 | 30 | @filter = RecordFilter.new(*([:include, :include_regexp, :exclude, :exclude_regexp].map{|s| config.filter_params[s]})) 31 | @fields = config.field_definitions 32 | if @time_key 33 | @fields[:integer].push @time_key 34 | end 35 | @queries = config.query_generators.map{|g| g.generate(@name, @escaped_name)} 36 | end 37 | 38 | def filter(time, record) 39 | r = @filter.filter(record) 40 | if @time_key 41 | # Fluentd time (sec) -> Norikra timestamp (milliseconds) 42 | r = r.merge({ @time_key => time * 1000 }) 43 | end 44 | if @escape_fieldname 45 | escape_recursive(r) 46 | else 47 | r 48 | end 49 | end 50 | 51 | def escape_recursive(record) 52 | return record unless record.is_a?(Hash) || record.is_a?(Array) 53 | return record.map{|v| escape_recursive(v) } if record.is_a?(Array) 54 | 55 | # Hash 56 | r = {} 57 | record.keys.each do |key| 58 | k = if key =~ /[^$_a-zA-Z0-9]/ 59 | key.gsub(/[^$_a-zA-Z0-9]/, '_') 60 | else 61 | key 62 | end 63 | v = escape_recursive(record[key]) 64 | r[k] = v 65 | end 66 | r 67 | end 68 | 69 | def reserve_fields 70 | f = {} 71 | @fields.keys.each do |type_sym| 72 | @fields[type_sym].each do |fieldname| 73 | f[fieldname] = type_sym.to_s 74 | end 75 | end 76 | f 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/fluent/plugin/norikra_target.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/norikra/fluent-plugin-norikra/a3e8ff39f7631a932ad5d2367eec2ea628a07d1d/lib/fluent/plugin/norikra_target.rb -------------------------------------------------------------------------------- /lib/fluent/plugin/out_norikra.rb: -------------------------------------------------------------------------------- 1 | require 'fluent/output' 2 | require_relative 'norikra/output' 3 | 4 | require 'norikra-client' 5 | 6 | module Fluent 7 | class NorikraOutput < Fluent::BufferedOutput 8 | include Fluent::NorikraPlugin::OutputMixin 9 | 10 | Fluent::Plugin.register_output('norikra', self) 11 | 12 | config_set_default :flush_interval, 1 # 1sec 13 | 14 | config_param :norikra, :string, :default => 'localhost:26571' 15 | 16 | config_param :connect_timeout, :integer, :default => nil 17 | config_param :send_timeout, :integer, :default => nil 18 | config_param :receive_timeout, :integer, :default => nil 19 | 20 | #for OutputMixin 21 | config_param :remove_tag_prefix, :string, :default => nil 22 | config_param :target_map_tag, :bool, :default => false 23 | config_param :target_map_key, :string, :default => nil 24 | config_param :target_string, :string, :default => nil 25 | config_param :drop_error_record, :bool, :default => true 26 | config_param :drop_server_error_record, :bool, :default => false 27 | config_param :drop_when_shutoff, :bool, :default => false 28 | 29 | # 30 | # 31 | 32 | # Define `log` method for v0.10.42 or earlier 33 | unless method_defined?(:log) 34 | define_method("log") { $log } 35 | end 36 | 37 | def configure(conf) 38 | super 39 | 40 | @host,@port = @norikra.split(':', 2) 41 | @port = @port.to_i 42 | 43 | if !@target_map_tag && @target_map_key.nil? && @target_string.nil? 44 | raise Fluent::ConfigError, 'target naming not specified (target_map_tag/target_map_key/target_string)' 45 | end 46 | 47 | setup_output(conf, false) # disabled in and 48 | end 49 | 50 | def client(opts={}) 51 | Norikra::Client.new(@host, @port, { 52 | :connect_timeout => opts[:connect_timeout] || @connect_timeout, 53 | :send_timeout => opts[:send_timeout] || @send_timeout, 54 | :receive_timeout => opts[:receive_timeout] || @receive_timeout, 55 | }) 56 | end 57 | 58 | def start 59 | super 60 | start_output 61 | end 62 | 63 | def shutdown 64 | stop_output 65 | shutdown_output 66 | end 67 | 68 | def fetchable? 69 | true 70 | end 71 | 72 | # For Fluentd 0.14 compatibility. 73 | # Fluent::Compat::BufferedOutput expects the plugin class itself 74 | # (but not its included module) to define `format_stream` when overriding. 75 | def format_stream(*) 76 | super 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_norikra_filter.rb: -------------------------------------------------------------------------------- 1 | require 'fluent/output' 2 | require_relative 'norikra/input' 3 | require_relative 'norikra/output' 4 | 5 | require 'norikra-client' 6 | 7 | module Fluent 8 | class NorikraFilterOutput < Fluent::BufferedOutput 9 | include Fluent::NorikraPlugin::InputMixin 10 | include Fluent::NorikraPlugin::OutputMixin 11 | 12 | Fluent::Plugin.register_output('norikra_filter', self) 13 | 14 | config_set_default :flush_interval, 1 # 1sec 15 | 16 | config_param :norikra, :string, :default => 'localhost:26571' 17 | 18 | config_param :connect_timeout, :integer, :default => nil 19 | config_param :send_timeout, :integer, :default => nil 20 | config_param :receive_timeout, :integer, :default => nil 21 | 22 | # 23 | attr_reader :execute_server, :execute_server_path 24 | 25 | #for OutputMixin 26 | config_param :remove_tag_prefix, :string, :default => nil 27 | config_param :target_map_tag, :bool, :default => false 28 | config_param :target_map_key, :string, :default => nil 29 | config_param :target_string, :string, :default => nil 30 | config_param :drop_error_record, :bool, :default => true 31 | config_param :drop_server_error_record, :bool, :default => false 32 | config_param :drop_when_shutoff, :bool, :default => false 33 | 34 | # 35 | # 36 | 37 | # tags 38 | # 39 | # method event 40 | # target QUERY_NAME 41 | # interval 5s 42 | # tag query_name 43 | # # tag field FIELDNAME 44 | # # tag string FIXED_STRING 45 | # tag_prefix norikra.event # actual tag: norikra.event.QUERYNAME 46 | # 47 | # 48 | # method sweep 49 | # target QUERY_GROUP # or unspecified => default 50 | # interval 60s 51 | # tag field group_by_key 52 | # tag_prefix norikra.query 53 | # 54 | 55 | # Define `log` method for v0.10.42 or earlier 56 | unless method_defined?(:log) 57 | define_method("log") { $log } 58 | end 59 | 60 | def configure(conf) 61 | super 62 | 63 | @host,@port = @norikra.split(':', 2) 64 | @port = @port.to_i 65 | 66 | if !@target_map_tag && @target_map_key.nil? && @target_string.nil? 67 | raise Fluent::ConfigError, 'target naming not specified (target_map_tag/target_map_key/target_string)' 68 | end 69 | 70 | @execute_server = false 71 | 72 | conf.elements.each do |element| 73 | case element.name 74 | when 'server' 75 | @execute_server = true 76 | @execute_jruby_path = element['jruby'] 77 | @execute_server_path = element['path'] 78 | @execute_server_opts = element['opts'] 79 | end 80 | end 81 | 82 | setup_output(conf, true) # enabled in and 83 | setup_input(conf) 84 | end 85 | 86 | def client(opts={}) 87 | Norikra::Client.new(@host, @port, { 88 | :connect_timeout => opts[:connect_timeout] || @connect_timeout, 89 | :send_timeout => opts[:send_timeout] || @send_timeout, 90 | :receive_timeout => opts[:receive_timeout] || @receive_timeout, 91 | }) 92 | end 93 | 94 | def start 95 | super 96 | 97 | @norikra_started = false 98 | 99 | if @execute_server 100 | @norikra_pid = nil 101 | @norikra_thread = Thread.new(&method(:server_starter)) 102 | # @norikra_started will be set in server_starter 103 | else 104 | @norikra_pid = nil 105 | @norikra_thread = nil 106 | @norikra_started = true 107 | end 108 | 109 | start_output 110 | start_input 111 | end 112 | 113 | def shutdown 114 | stop_output 115 | stop_input 116 | Process.kill(:TERM, @norikra_pid) if @execute_server 117 | 118 | shutdown_output 119 | shutdown_input 120 | 121 | if @execute_server 122 | begin 123 | counter = 0 124 | while !Process.waitpid(@norikra_pid, Process::WNOHANG) 125 | sleep 1 126 | break if counter > 3 127 | end 128 | rescue Errno::ECHILD 129 | # norikra server process exited. 130 | end 131 | end 132 | end 133 | 134 | def server_starter 135 | log.info "starting Norikra server process #{@host}:#{@port}" 136 | base_options = [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s] 137 | cmd,options = if @execute_jruby_path 138 | [@execute_jruby_path, [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]] 139 | else 140 | [@execute_server_path, ['start', '-H', @host, '-P', @port.to_s]] 141 | end 142 | if @execute_server_opts 143 | options += @execute_server_opts.split(/ +/) 144 | end 145 | @norikra_pid = fork do 146 | ENV.keys.select{|k| k =~ /^(RUBY|GEM|BUNDLE|RBENV|RVM|rvm)/}.each {|k| ENV.delete(k)} 147 | exec([cmd, 'norikra(fluentd)'], *options) 148 | end 149 | connecting = true 150 | log.info "trying to confirm norikra server status..." 151 | while connecting 152 | begin 153 | log.debug "start to connect norikra server #{@host}:#{@port}" 154 | client(:connect_timeout => 1, :send_timeout => 1, :receive_timeout => 1).targets 155 | # discard result: no exceptions is success 156 | connecting = false 157 | next 158 | rescue HTTPClient::TimeoutError 159 | log.debug "Norikra server test connection timeout. retrying..." 160 | rescue Errno::ECONNREFUSED 161 | log.debug "Norikra server test connection refused. retrying..." 162 | rescue => e 163 | log.error "unknown error in confirming norikra server, #{e.class}:#{e.message}" 164 | end 165 | sleep 3 166 | end 167 | log.info "confirmed that norikra server #{@host}:#{@port} started." 168 | @norikra_started = true 169 | end 170 | 171 | def fetchable? 172 | @norikra_started 173 | end 174 | end 175 | end 176 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | begin 4 | Bundler.setup(:default, :development) 5 | rescue Bundler::BundlerError => e 6 | $stderr.puts e.message 7 | $stderr.puts "Run `bundle install` to install missing gems" 8 | exit e.status_code 9 | end 10 | require 'test/unit' 11 | 12 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 13 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 14 | 15 | require 'fluent/test' 16 | unless ENV.has_key?('VERBOSE') 17 | nulllogger = Object.new 18 | nulllogger.instance_eval {|obj| 19 | def method_missing(method, *args) 20 | # pass 21 | end 22 | } 23 | $log = nulllogger 24 | end 25 | 26 | require 'fluent/plugin/in_norikra' 27 | require 'fluent/plugin/out_norikra' 28 | require 'fluent/plugin/out_norikra_filter' 29 | 30 | class Test::Unit::TestCase 31 | end 32 | -------------------------------------------------------------------------------- /test/plugin/test_in_norikra.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class NorikraInputTest < Test::Unit::TestCase 4 | CONFIG = %[ 5 | ] 6 | 7 | def create_driver(conf=CONFIG, tag='test') 8 | Fluent::Test::InputTestDriver.new(Fluent::NorikraInput).configure(conf) 9 | end 10 | 11 | def test_init 12 | create_driver 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /test/plugin/test_out_norikra.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class NorikraOutputTest < Test::Unit::TestCase 4 | CONFIG = %[ 5 | target_map_tag yes 6 | ] 7 | 8 | def create_driver(conf=CONFIG, tag='test') 9 | Fluent::Test::OutputTestDriver.new(Fluent::NorikraOutput, tag).configure(conf) 10 | end 11 | 12 | def test_init 13 | create_driver 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /test/plugin/test_out_norikra_filter.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | 3 | class NorikraFilterOutputTest < Test::Unit::TestCase 4 | CONFIG = %[ 5 | target_map_tag yes 6 | ] 7 | 8 | def create_driver(conf=CONFIG, tag='test') 9 | Fluent::Test::OutputTestDriver.new(Fluent::NorikraFilterOutput, tag).configure(conf) 10 | end 11 | 12 | def test_init 13 | create_driver 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /test/test_config_section.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | require 'fluent/plugin/norikra/config_section' 3 | 4 | class ConfigSectionTest < Test::Unit::TestCase 5 | def setup 6 | Fluent::Test.setup 7 | @this = Fluent::NorikraPlugin::ConfigSection 8 | end 9 | 10 | def test_init_default 11 | q1 = Fluent::Config::Element.new('query', nil, { 12 | 'name' => 'q1_${target}', 13 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1', 14 | 'tag' => 'q1.${target}' 15 | }, []) 16 | q2 = Fluent::Config::Element.new('query', nil, { 17 | 'name' => 'q2_${target}', 18 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0', 19 | 'tag' => 'q2.${target}' 20 | }, []) 21 | c1 = Fluent::Config::Element.new('default', nil, { 22 | 'include' => '*', 23 | 'exclude' => 'flag', 24 | 'exclude_regexp' => 'f_.*', 25 | 'field_string' => 's1,s2,s3', 26 | 'field_boolean' => 'bool1,bool2', 27 | 'field_integer' => 'i1,i2,i3,i4,num1,num2', 28 | 'field_float' => 'f1,f2,d', 29 | }, [q1,q2]) 30 | s1 = @this.new(c1) 31 | 32 | assert_nil s1.target 33 | assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => 'f_.*'}, s1.filter_params) 34 | assert_equal({ 35 | :string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2), 36 | :float => %w(f1 f2 d), 37 | }, s1.field_definitions) 38 | assert_equal 2, s1.query_generators.size 39 | assert_equal (10 * 60 / 5), s1.query_generators.map(&:fetch_interval).sort.first 40 | end 41 | 42 | def test_init_default_without_query 43 | q1 = Fluent::Config::Element.new('query', nil, { 44 | 'name' => 'q1_${target}', 45 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1', 46 | 'tag' => 'q1.${target}' 47 | }, []) 48 | q2 = Fluent::Config::Element.new('query', nil, { 49 | 'name' => 'q2_${target}', 50 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0', 51 | 'tag' => 'q2.${target}' 52 | }, []) 53 | c1 = Fluent::Config::Element.new('default', nil, { 54 | 'include' => '*', 55 | 'exclude' => 'flag', 56 | 'exclude_regexp' => 'f_.*', 57 | 'field_string' => 's1,s2,s3', 58 | 'field_boolean' => 'bool1,bool2', 59 | 'field_integer' => 'i1,i2,i3,i4,num1,num2', 60 | 'field_float' => 'f1,f2,d', 61 | }, [q1,q2]) 62 | s1 = @this.new(c1, false) 63 | assert_equal 0, s1.query_generators.size 64 | end 65 | 66 | def test_init_default_with_time_key 67 | q1 = Fluent::Config::Element.new('query', nil, { 68 | 'name' => 'q1_${target}', 69 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1', 70 | 'tag' => 'q1.${target}' 71 | }, []) 72 | q2 = Fluent::Config::Element.new('query', nil, { 73 | 'name' => 'q2_${target}', 74 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0', 75 | 'tag' => 'q2.${target}' 76 | }, []) 77 | c1 = Fluent::Config::Element.new('default', nil, { 78 | 'time_key' => 'timestamp', 79 | 'include' => '*', 80 | 'exclude' => 'flag', 81 | 'exclude_regexp' => 'f_.*', 82 | 'field_string' => 's1,s2,s3', 83 | 'field_boolean' => 'bool1,bool2', 84 | 'field_integer' => 'i1,i2,i3,i4,num1,num2', 85 | 'field_float' => 'f1,f2,d', 86 | }, [q1,q2]) 87 | s1 = @this.new(c1, false) 88 | 89 | assert_nil s1.target 90 | assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => 'f_.*'}, s1.filter_params) 91 | assert_equal({ 92 | :string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2), 93 | :float => %w(f1 f2 d), 94 | }, s1.field_definitions) 95 | assert_equal 'timestamp', s1.time_key 96 | end 97 | 98 | def test_init_target 99 | q3 = Fluent::Config::Element.new('query', nil, { 100 | 'name' => 'q3_test2', 101 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"', 102 | 'tag' => 'q3.test2' 103 | }, []) 104 | c2 = Fluent::Config::Element.new('target', 'test2', { 105 | 'exclude_regexp' => '(f|g)_.*', 106 | 'field_float' => 'd1,d2,d3,d4' 107 | }, [q3]) 108 | s2 = @this.new(c2) 109 | 110 | assert_equal 'test2', s2.target 111 | assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => '(f|g)_.*'}, s2.filter_params) 112 | assert_equal({:string => [], :boolean => [], :integer => [], :float => %w(d1 d2 d3 d4)}, s2.field_definitions) 113 | assert_equal 1, s2.query_generators.size 114 | assert_equal (30 * 60 / 5), s2.query_generators.map(&:fetch_interval).sort.first 115 | end 116 | 117 | def test_init_target_query_only 118 | q4 = Fluent::Config::Element.new('query', nil, { 119 | 'name' => 'q4_test3', 120 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q4 > 10', 121 | 'tag' => 'q4.test3', 122 | 'fetch_interval' => '1s' 123 | }, []) 124 | c3 = Fluent::Config::Element.new('target', 'test3', {}, [q4]) 125 | s3 = @this.new(c3) 126 | 127 | assert_equal 'test3', s3.target 128 | assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s3.filter_params) 129 | assert_equal({:string => [], :boolean => [], :integer => [], :float => []}, s3.field_definitions) 130 | assert_equal 1, s3.query_generators.size 131 | end 132 | 133 | def test_init_target_without_query 134 | c4 = Fluent::Config::Element.new('target', 'test4', { 135 | 'field_integer' => 'status' 136 | }, []) 137 | s4 = @this.new(c4) 138 | 139 | assert_equal 'test4', s4.target 140 | assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s4.filter_params) 141 | assert_equal({:string => [], :boolean => [], :integer => ['status'], :float => []}, s4.field_definitions) 142 | assert_equal 0, s4.query_generators.size 143 | end 144 | 145 | def test_init_target_blank 146 | c5 = Fluent::Config::Element.new('target', 'test5', {}, []) 147 | s5 = @this.new(c5) 148 | 149 | assert_equal 'test5', s5.target 150 | assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s5.filter_params) 151 | assert_equal({:string => [], :boolean => [], :integer => [], :float => []}, s5.field_definitions) 152 | assert_equal 0, s5.query_generators.size 153 | end 154 | 155 | def test_join 156 | q1 = Fluent::Config::Element.new('query', nil, { 157 | 'name' => 'q1_${target}', 158 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1', 159 | 'tag' => 'q1.${target}' 160 | }, []) 161 | q2 = Fluent::Config::Element.new('query', nil, { 162 | 'name' => 'q2_${target}', 163 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0', 164 | 'tag' => 'q2.${target}' 165 | }, []) 166 | c1 = Fluent::Config::Element.new('default', nil, { 167 | 'time_key' => 'ts', 168 | 'include' => '*', 169 | 'exclude' => 'flag', 170 | 'exclude_regexp' => 'f_.*', 171 | 'field_string' => 's1,s2,s3', 172 | 'field_boolean' => 'bool1,bool2', 173 | 'field_integer' => 'i1,i2,i3,i4,num1,num2', 174 | 'field_float' => 'f1,f2,d', 175 | }, [q1,q2]) 176 | s1 = @this.new(c1) 177 | 178 | q3 = Fluent::Config::Element.new('query', nil, { 179 | 'name' => 'q3_test', 180 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"', 181 | 'tag' => 'q3.test' 182 | }, []) 183 | c2 = Fluent::Config::Element.new('target', 'test', { 184 | 'time_key' => 'timestamp', 185 | 'exclude_regexp' => '(f|g)_.*', 186 | 'field_float' => 'd1,d2,d3,d4' 187 | }, [q3]) 188 | s2 = @this.new(c2) 189 | 190 | s = s1 + s2 191 | 192 | assert_equal 'test', s.target 193 | assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => '(f|g)_.*'}, s.filter_params) 194 | assert_equal({ 195 | :string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2), 196 | :float => %w(f1 f2 d d1 d2 d3 d4) 197 | }, s.field_definitions) 198 | assert_equal 3, s.query_generators.size 199 | assert_equal (10 * 60 / 5), s.query_generators.map(&:fetch_interval).sort.first 200 | assert_equal 'timestamp', s.time_key 201 | end 202 | 203 | def test_join_with_nil 204 | q1 = Fluent::Config::Element.new('query', nil, { 205 | 'name' => 'q1_${target}', 206 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1', 207 | 'tag' => 'q1.${target}' 208 | }, []) 209 | q2 = Fluent::Config::Element.new('query', nil, { 210 | 'name' => 'q2_${target}', 211 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0', 212 | 'tag' => 'q2.${target}' 213 | }, []) 214 | c1 = Fluent::Config::Element.new('default', nil, { 215 | 'include' => '*', 216 | 'exclude' => 'flag', 217 | 'exclude_regexp' => 'f_.*', 218 | 'field_string' => 's1,s2,s3', 219 | 'field_boolean' => 'bool1,bool2', 220 | 'field_integer' => 'i1,i2,i3,i4,num1,num2', 221 | 'field_float' => 'f1,f2,d', 222 | }, [q1,q2]) 223 | s1 = @this.new(c1) 224 | 225 | s = s1 + nil 226 | 227 | assert_equal 'dummy', s.target 228 | assert_equal s1.filter_params, s.filter_params 229 | assert_equal s1.field_definitions, s.field_definitions 230 | assert_equal s1.query_generators.size, s.query_generators.size 231 | assert_equal s1.query_generators.map(&:fetch_interval).sort.first, s.query_generators.map(&:fetch_interval).sort.first 232 | end 233 | 234 | def test_join_without_query 235 | q1 = Fluent::Config::Element.new('query', nil, { 236 | 'name' => 'q1_${target}', 237 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1', 238 | 'tag' => 'q1.${target}' 239 | }, []) 240 | q2 = Fluent::Config::Element.new('query', nil, { 241 | 'name' => 'q2_${target}', 242 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0', 243 | 'tag' => 'q2.${target}' 244 | }, []) 245 | c1 = Fluent::Config::Element.new('default', nil, { 246 | 'time_key' => 'timestamp', 247 | 'include' => '*', 248 | 'exclude' => 'flag', 249 | 'exclude_regexp' => 'f_.*', 250 | 'field_string' => 's1,s2,s3', 251 | 'field_boolean' => 'bool1,bool2', 252 | 'field_integer' => 'i1,i2,i3,i4,num1,num2', 253 | 'field_float' => 'f1,f2,d', 254 | }, [q1,q2]) 255 | s1 = @this.new(c1, false) 256 | 257 | q3 = Fluent::Config::Element.new('query', nil, { 258 | 'name' => 'q3_test', 259 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"', 260 | 'tag' => 'q3.test' 261 | }, []) 262 | c2 = Fluent::Config::Element.new('target', 'test', { 263 | 'exclude_regexp' => '(f|g)_.*', 264 | 'field_float' => 'd1,d2,d3,d4' 265 | }, [q3]) 266 | s2 = @this.new(c2, false) 267 | 268 | s = s1 + s2 269 | assert_equal 0, s.query_generators.size 270 | assert_equal 'timestamp', s.time_key 271 | end 272 | end 273 | -------------------------------------------------------------------------------- /test/test_query.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | require 'fluent/plugin/norikra/query' 3 | 4 | class QueryTest < Test::Unit::TestCase 5 | def test_init 6 | q = Fluent::NorikraPlugin::Query.new('name', nil, 'expression', 'tag', 10) 7 | assert_equal 'name', q.name 8 | assert_nil q.group 9 | assert_equal 'expression', q.expression 10 | assert_equal 'tag', q.tag 11 | assert_equal 10, q.interval 12 | 13 | q = Fluent::NorikraPlugin::Query.new('name', 'group', 'expression', 'tag', 10) 14 | assert_equal 'group', q.group 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /test/test_query_generator.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | require 'fluent/plugin/norikra/query_generator' 3 | 4 | class QueryGeneratorTest < Test::Unit::TestCase 5 | def setup 6 | Fluent::Test.setup 7 | @this = Fluent::NorikraPlugin::QueryGenerator 8 | end 9 | 10 | def test_replace_target 11 | expected = 'SELECT * FROM replaced.win:time_batch(10 hours) WHERE x=1' 12 | assert_equal expected, @this.replace_target('replaced', 'SELECT * FROM ${target}.win:time_batch(10 hours) WHERE x=1') 13 | end 14 | 15 | def test_parse_time_period 16 | assert_equal [0,0,0,0,0,0,0,0], @this.parse_time_period('') 17 | 18 | assert_equal [10,0,0,0,0,0,0,0], @this.parse_time_period(' 10 year') 19 | assert_equal [0,11,0,0,0,0,0,0], @this.parse_time_period('11 month') 20 | assert_equal [0,233,0,0,0,0,0,0], @this.parse_time_period('233 months') 21 | assert_equal [0,0,10,0,0,0,0,0], @this.parse_time_period('10 weeks') 22 | assert_equal [0,0,0,1,0,0,0,0], @this.parse_time_period('1 day') 23 | assert_equal [0,0,0,201,0,0,0,0], @this.parse_time_period('201 days') 24 | assert_equal [0,0,0,0,1,0,0,0], @this.parse_time_period('1 hour') 25 | assert_equal [0,0,0,0,11,0,0,0], @this.parse_time_period('11 hours') 26 | assert_equal [0,0,0,0,0,2,0,0], @this.parse_time_period('2 minutes') 27 | assert_equal [0,0,0,0,0,1,0,0], @this.parse_time_period('1 min') 28 | assert_equal [0,0,0,0,0,133,0,0], @this.parse_time_period('133 minute') 29 | assert_equal [0,0,0,0,0,0,12,0], @this.parse_time_period('12 sec') 30 | assert_equal [0,0,0,0,0,0,1,0], @this.parse_time_period('1 second') 31 | assert_equal [0,0,0,0,0,0,256,0], @this.parse_time_period(' 256 seconds') 32 | assert_equal [0,0,0,0,0,0,0,1], @this.parse_time_period('1 msec') 33 | assert_equal [0,0,0,0,0,0,0,111], @this.parse_time_period('111 milliseconds') 34 | 35 | assert_equal [1,12,4,365,23,59,60,0], @this.parse_time_period('1 year 12 months 4 weeks 365 days 23 hours 59 min 60 seconds') 36 | end 37 | 38 | def test_generate 39 | g = @this.new('query_${target}', 'test_group', 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}') 40 | q = g.generate('test', 'test') 41 | assert_equal 'query_test', q.name 42 | assert_equal 'test_group', q.group 43 | assert_equal 'SELECT * FROM test.win:time_batch( 10 min ) WHERE x=1', q.expression 44 | assert_equal 'tag.test', q.tag 45 | 46 | g = @this.new('query_${target}', nil, 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}') 47 | q = g.generate('test', 'test') 48 | assert_nil q.group 49 | end 50 | 51 | def test_fetch_interval 52 | g = @this.new('query_${target}', nil, 'SELECT * FROM ${target}.win:time_batch( 12 min ) WHERE x=1', 'tag.${target}') 53 | assert_equal (12*60/5), g.fetch_interval 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /test/test_record_filter.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | require 'fluent/plugin/norikra/record_filter' 3 | 4 | class RecordFilterTest < Test::Unit::TestCase 5 | def setup 6 | Fluent::Test.setup 7 | @this = Fluent::NorikraPlugin::RecordFilter 8 | end 9 | 10 | def test_errors 11 | assert_raise(Fluent::ConfigError){ @this.new('*', '', '*', '') } 12 | assert_raise(Fluent::ConfigError){ @this.new('x', 'y', 'z', 'p') } 13 | assert_raise(Fluent::ConfigError){ @this.new('','', '*','') } 14 | end 15 | 16 | def test_init 17 | f = @this.new() 18 | assert_equal :include, f.default_policy 19 | assert_equal [], f.exclude_fields 20 | assert_nil f.exclude_regexp 21 | end 22 | 23 | def test_filter_default # return record itself 24 | f = @this.new(nil,nil,nil,nil) 25 | r = {'x'=>1,'y'=>2,'z'=>'3'} 26 | assert_equal r, f.filter(r) 27 | assert_equal r.object_id, f.filter(r).object_id 28 | 29 | f = @this.new('','','','') 30 | r = {'x'=>1,'y'=>2,'z'=>'3'} 31 | assert_equal r, f.filter(r) 32 | assert_equal r.object_id, f.filter(r).object_id 33 | end 34 | 35 | def test_filter_exclude_keys 36 | f = @this.new('*',nil,'x,y,z') 37 | r = {'a'=>'1','b'=>'2','c'=>'3','x'=>1,'y'=>2,'z'=>3} 38 | assert_equal 3, f.filter(r).size 39 | assert_equal({'a'=>'1','b'=>'2','c'=>'3'}, f.filter(r)) 40 | assert_equal 6, r.size # check original record not to be broken 41 | end 42 | 43 | def test_filter_exclude_regexp 44 | f = @this.new('*',nil,nil,'f_.*') 45 | r = {'a'=>'1','b'=>'2','c'=>'3','f_x'=>1,'f_y'=>2,'f_z'=>3} 46 | assert_equal 3, f.filter(r).size 47 | assert_equal({'a'=>'1','b'=>'2','c'=>'3'}, f.filter(r)) 48 | assert_equal 6, r.size # check original record not to be broken 49 | end 50 | 51 | def test_filter_excludes 52 | f = @this.new('*',nil,'b,c','f_.*') 53 | r = {'a'=>'1','b'=>'2','c'=>'3','f_x'=>1,'f_y'=>2,'f_z'=>3} 54 | assert_equal 1, f.filter(r).size 55 | assert_equal({'a'=>'1'}, f.filter(r)) 56 | assert_equal 6, r.size # check original record not to be broken 57 | end 58 | 59 | def test_filter_include_keys 60 | f = @this.new('a,b,c',nil,'*','') 61 | r = {'a'=>'1','b'=>'2','c'=>'3','x'=>1,'y'=>2,'z'=>3} 62 | assert_equal 3, f.filter(r).size 63 | assert_equal({'a'=>'1','b'=>'2','c'=>'3'}, f.filter(r)) 64 | assert_equal 6, r.size # check original record not to be broken 65 | end 66 | 67 | def test_filter_include_regexp 68 | f = @this.new('','f_','*','') 69 | r = {'f_a'=>'1','f_b'=>'2','f_c'=>'3','x'=>1,'y'=>2,'z'=>3} 70 | assert_equal 3, f.filter(r).size 71 | assert_equal({'f_a'=>'1','f_b'=>'2','f_c'=>'3'}, f.filter(r)) 72 | assert_equal 6, r.size # check original record not to be broken 73 | end 74 | 75 | def test_filter_includes 76 | f = @this.new('y,z','f_','*','') 77 | r = {'f_a'=>'1','f_b'=>'2','f_c'=>'3','x'=>1,'y'=>2,'z'=>3} 78 | assert_equal 5, f.filter(r).size 79 | assert_equal({'f_a'=>'1','f_b'=>'2','f_c'=>'3','y'=>2,'z'=>3}, f.filter(r)) 80 | assert_equal 6, r.size # check original record not to be broken 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /test/test_target.rb: -------------------------------------------------------------------------------- 1 | require 'helper' 2 | require 'fluent/plugin/norikra/target' 3 | 4 | class TargetTest < Test::Unit::TestCase 5 | def setup 6 | Fluent::Test.setup 7 | @this = Fluent::NorikraPlugin::Target 8 | end 9 | 10 | def test_target_name_escape 11 | assert_equal 'target1', @this.escape('target1') 12 | assert_equal 'target1_subtarget1', @this.escape('target1.subtarget1') 13 | assert_equal 'test_tag_foo', @this.escape('test.tag.foo') 14 | 15 | assert_equal 'FluentdGenerated', @this.escape('') 16 | assert_equal 'Fluentd_Generated', @this.escape(':') 17 | assert_equal 'a', @this.escape('a') 18 | assert_equal 'Fluentd_a', @this.escape('_a') 19 | assert_equal 'a_Generated', @this.escape('a_') 20 | end 21 | 22 | Q1 = Fluent::Config::Element.new('query', nil, { 23 | 'name' => 'q1_${target}', 24 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1', 25 | 'tag' => 'q1.${target}' 26 | }, []) 27 | Q2 = Fluent::Config::Element.new('query', nil, { 28 | 'name' => 'q2_${target}', 29 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0', 30 | 'tag' => 'q2.${target}' 31 | }, []) 32 | C1 = Fluent::Config::Element.new('default', nil, { 33 | 'include' => '*', 34 | 'exclude' => 'flag', 35 | 'exclude_regexp' => 'f_.*', 36 | 'field_string' => 's1,s2,s3', 37 | 'field_boolean' => 'bool1,bool2', 38 | 'field_integer' => 'i1,i2,i3,i4,num1,num2', 39 | 'field_float' => 'f1,f2,d', 40 | }, [Q1,Q2]) 41 | S1 = Fluent::NorikraPlugin::ConfigSection.new(C1) 42 | 43 | Q3 = Fluent::Config::Element.new('query', nil, { 44 | 'name' => 'q3_test', 45 | 'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"', 46 | 'tag' => 'q3.test' 47 | }, []) 48 | C2 = Fluent::Config::Element.new('target', 'test', { 49 | 'time_key' => 'timestamp', 50 | 'exclude_regexp' => '(f|g)_.*', 51 | 'field_float' => 'd1,d2,d3,d4' 52 | }, [Q3]) 53 | S2 = Fluent::NorikraPlugin::ConfigSection.new(C2) 54 | 55 | def test_instanciate 56 | t = @this.new('test', S1 + S2) 57 | 58 | assert_equal 'test', t.name 59 | assert_equal({ 60 | :string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2 timestamp), 61 | :float => %w(f1 f2 d d1 d2 d3 d4) 62 | }, t.fields) 63 | assert_equal 3, t.queries.size 64 | 65 | now = Time.now.to_i 66 | 67 | r = t.filter(now, {'x'=>1,'y'=>'y','z'=>'zett','flag'=>true,'f_x'=>'true','g_1'=>'g'}) 68 | assert_equal 4, r.size 69 | assert_equal({'x'=>1,'y'=>'y','z'=>'zett','timestamp'=>(now*1000)}, r) 70 | 71 | # reserve_fields 72 | assert_equal({ 73 | 's1' => 'string', 's2' => 'string', 's3' => 'string', 74 | 'bool1' => 'boolean', 'bool2' => 'boolean', 75 | 'i1' => 'integer', 'i2' => 'integer', 'i3' => 'integer', 'i4' => 'integer', 'num1' => 'integer', 'num2' => 'integer', 76 | 'f1' => 'float', 'f2' => 'float', 77 | 'd' => 'float', 'd1' => 'float', 'd2' => 'float', 'd3' => 'float', 'd4' => 'float', 78 | 'timestamp' => 'integer', # time_key 79 | }, t.reserve_fields) 80 | end 81 | 82 | def test_queries 83 | t = @this.new('test.service', S1 + S2) 84 | 85 | assert_equal 3, t.queries.size 86 | 87 | assert_equal 'q1_test.service', t.queries[0].name 88 | assert_equal 'SELECT * FROM test_service.win:time_batch(10 min) WHERE q1', t.queries[0].expression 89 | assert_equal 'q1.test.service', t.queries[0].tag 90 | 91 | assert_equal 'q2_test.service', t.queries[1].name 92 | assert_equal 'SELECT * FROM test_service.win:time_batch(50 min) WHERE q2.length() > 0', t.queries[1].expression 93 | assert_equal 'q2.test.service', t.queries[1].tag 94 | 95 | assert_equal 'q3_test', t.queries[2].name 96 | assert_equal 'SELECT * FROM test_service.win:time_batch(30 min) WHERE q3="/"', t.queries[2].expression 97 | assert_equal 'q3.test', t.queries[2].tag 98 | end 99 | 100 | C3 = Fluent::Config::Element.new('target', 'test', { 101 | 'escape_fieldname' => 'no', 102 | }, []) 103 | S3 = Fluent::NorikraPlugin::ConfigSection.new(C3) 104 | C4 = Fluent::Config::Element.new('target', 'test', { 105 | 'escape_fieldname' => 'yes', 106 | }, []) 107 | S4 = Fluent::NorikraPlugin::ConfigSection.new(C4) 108 | 109 | def test_escape_fieldname 110 | now = Time.now.to_i 111 | 112 | t = @this.new('test.service', S3) 113 | r = t.filter(now, {'a 1' => '1', 'b 2' => 2, 'c-1' => { 'd/1' => '1', 'd 2' => '2' }, 'f' => [1, 2, {'g+1' => 3}] }) 114 | assert_equal '1', r['a 1'] 115 | assert_equal 2, r['b 2'] 116 | assert_equal '1', r['c-1']['d/1'] 117 | assert_equal '2', r['c-1']['d 2'] 118 | assert_equal 1, r['f'][0] 119 | assert_equal 2, r['f'][1] 120 | assert_equal 3, r['f'][2]['g+1'] 121 | 122 | assert_nil r['a_1'] 123 | assert_nil r['b_2'] 124 | assert_nil r['c_1'] 125 | assert_nil r['f'][2]['g_1'] 126 | 127 | t = @this.new('test.service', S4) 128 | r = t.filter(now, {'a 1' => '1', 'b 2' => 2, 'c-1' => { 'd/1' => '1', 'd 2' => '2' }, 'f' => [1, 2, {'g+1' => 3}] }) 129 | assert_nil r['a 1'] 130 | assert_nil r['b 2'] 131 | assert_nil r['c-1'] 132 | assert_equal 1, r['f'][0] 133 | assert_equal 2, r['f'][1] 134 | assert_nil r['f'][2]['g+1'] 135 | 136 | assert_equal '1', r['a_1'] 137 | assert_equal 2, r['b_2'] 138 | assert_equal '1', r['c_1']['d_1'] 139 | assert_equal '2', r['c_1']['d_2'] 140 | assert_equal 3, r['f'][2]['g_1'] 141 | end 142 | end 143 | --------------------------------------------------------------------------------