├── VERSION ├── .gitignore ├── AUTHORS ├── Gemfile ├── lib └── fluent │ └── plugin │ ├── thrift │ ├── fb303_constants.rb │ ├── scribe_constants.rb │ ├── fb303_types.rb │ ├── scribe_types.rb │ ├── scribe.thrift │ ├── scribe.rb │ ├── fb303.thrift │ └── facebook_service.rb │ ├── out_scribe.rb │ └── in_scribe.rb ├── example.conf ├── .travis.yml ├── bin └── fluent-scribe-remote ├── Rakefile ├── fluent-plugin-scribe.gemspec ├── ChangeLog ├── README.rdoc └── test └── plugin ├── test_in_scribe.rb └── test_out_scribe.rb /VERSION: -------------------------------------------------------------------------------- 1 | 0.10.10 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pkg/* 2 | Gemfile.lock 3 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Kazuki Ohta 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in fluent-plugin-webhdfs.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/fb303_constants.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | 7 | require 'fb303_types' 8 | 9 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/scribe_constants.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | 7 | require 'scribe_types' 8 | 9 | -------------------------------------------------------------------------------- /example.conf: -------------------------------------------------------------------------------- 1 | # Scribe input 2 | 3 | type scribe 4 | port 1463 5 | 6 | 7 | # match tag=debug.** and dump to console 8 | 9 | type stdout 10 | 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | 3 | rvm: 4 | - 2.1.10 5 | - 2.2.9 6 | - 2.3.6 7 | - 2.4.3 8 | - 2.5.0 9 | 10 | branches: 11 | only: 12 | - master 13 | 14 | script: bundle exec rake test 15 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/fb303_types.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | 7 | 8 | module Fb_status 9 | DEAD = 0 10 | STARTING = 1 11 | ALIVE = 2 12 | STOPPING = 3 13 | STOPPED = 4 14 | WARNING = 5 15 | VALUE_MAP = {0 => "DEAD", 1 => "STARTING", 2 => "ALIVE", 3 => "STOPPING", 4 => "STOPPED", 5 => "WARNING"} 16 | VALID_VALUES = Set.new([DEAD, STARTING, ALIVE, STOPPING, STOPPED, WARNING]).freeze 17 | end 18 | 19 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/scribe_types.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | 7 | require 'fb303_types' 8 | 9 | 10 | module ResultCode 11 | OK = 0 12 | TRY_LATER = 1 13 | VALUE_MAP = {0 => "OK", 1 => "TRY_LATER"} 14 | VALID_VALUES = Set.new([OK, TRY_LATER]).freeze 15 | end 16 | 17 | class LogEntry 18 | include ::Thrift::Struct, ::Thrift::Struct_Union 19 | CATEGORY = 1 20 | MESSAGE = 2 21 | 22 | FIELDS = { 23 | CATEGORY => {:type => ::Thrift::Types::STRING, :name => 'category'}, 24 | MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'} 25 | } 26 | 27 | def struct_fields; FIELDS; end 28 | 29 | def validate 30 | end 31 | 32 | ::Thrift::Struct.generate_accessors self 33 | end 34 | 35 | -------------------------------------------------------------------------------- /bin/fluent-scribe-remote: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'thrift' 3 | require 'json' 4 | $:.unshift File.join(File.dirname(__FILE__), '../lib/fluent/plugin/thrift') 5 | require 'fb303_types' 6 | require 'fb303_constants' 7 | require 'facebook_service' 8 | require 'scribe_types' 9 | require 'scribe_constants' 10 | require 'scribe' 11 | 12 | host = 'localhost' 13 | port = 1463 14 | 15 | socket = Thrift::Socket.new host, port.to_i 16 | transport = Thrift::FramedTransport.new socket 17 | protocol = Thrift::BinaryProtocol.new transport, false, false 18 | client = Scribe::Client.new protocol 19 | transport.open 20 | 21 | # 2011/09/02 Kazuki Ohta 22 | # explicitly specify TCP_NODELAY for low-latency communication. 23 | raw_sock = socket.to_io 24 | raw_sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1 25 | 26 | entry = LogEntry.new 27 | entry.category = 'debug.hoge' 28 | entry.message = {'a' => 'b'}.to_json 29 | p client.Log([entry]) 30 | 31 | transport.close 32 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | require "bundler/gem_tasks" 3 | 4 | require 'rake' 5 | require 'rake/clean' 6 | 7 | task "thrift_gen" do 8 | system "rm -f common.thrift jobtracker.thrift" 9 | system "wget https://raw.github.com/facebook/scribe/master/if/scribe.thrift" 10 | system "wget https://raw.github.com/apache/thrift/trunk/contrib/fb303/if/fb303.thrift" 11 | system "mv scribe.thrift lib/fluent/plugin/thrift/" 12 | system "mv fb303.thrift lib/fluent/plugin/thrift/" 13 | system "mkdir -p tmp" 14 | system "sed -i '' 's/fb303\\/if\\///g' lib/fluent/plugin/thrift/scribe.thrift" 15 | system "thrift --gen rb -o tmp lib/fluent/plugin/thrift/fb303.thrift" 16 | system "thrift --gen rb -o tmp lib/fluent/plugin/thrift/scribe.thrift" 17 | system "mv tmp/gen-rb/* lib/fluent/plugin/thrift/" 18 | system "rm -fR tmp" 19 | end 20 | 21 | require 'rake/testtask' 22 | Rake::TestTask.new(:test) do |test| 23 | test.libs << 'lib' << 'test' 24 | test.pattern = 'test/**/test_*.rb' 25 | test.verbose = true 26 | end 27 | 28 | task :default => :test 29 | -------------------------------------------------------------------------------- /fluent-plugin-scribe.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | Gem::Specification.new do |gem| 4 | gem.name = "fluent-plugin-scribe" 5 | gem.version = "1.0.0" 6 | 7 | gem.authors = ["Kazuki Ohta", "TAGOMORI Satoshi"] 8 | gem.email = ["kazuki.ohta@gmail.com", "tagomoris@gmail.com"] 9 | gem.summary = %q{Scribe Input/Output plugin for Fluentd event collector} 10 | gem.description = %q{Fluentd input/output plugin to handle Facebook scribed thrift protocol} 11 | gem.homepage = "https://github.com/fluent/fluent-plugin-scribe" 12 | gem.license = "APLv2" 13 | 14 | gem.files = `git ls-files`.split($\) 15 | gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } 16 | gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) 17 | gem.require_paths = ["lib"] 18 | 19 | gem.add_development_dependency "rake" 20 | gem.add_development_dependency "test-unit", '~> 3.0.2' 21 | gem.add_runtime_dependency "fluentd" 22 | gem.add_runtime_dependency "thrift", "~> 0.8.0" 23 | end 24 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/scribe.thrift: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/thrift --gen cpp:pure_enums --gen php 2 | 3 | ## Copyright (c) 2007-2008 Facebook 4 | ## 5 | ## Licensed under the Apache License, Version 2.0 (the "License"); 6 | ## you may not use this file except in compliance with the License. 7 | ## You may obtain a copy of the License at 8 | ## 9 | ## http://www.apache.org/licenses/LICENSE-2.0 10 | ## 11 | ## Unless required by applicable law or agreed to in writing, software 12 | ## distributed under the License is distributed on an "AS IS" BASIS, 13 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | ## See the License for the specific language governing permissions and 15 | ## limitations under the License. 16 | ## 17 | ## See accompanying file LICENSE or visit the Scribe site at: 18 | ## http://developers.facebook.com/scribe/ 19 | 20 | include "fb303.thrift" 21 | 22 | namespace cpp scribe.thrift 23 | namespace java scribe.thrift 24 | namespace perl Scribe.Thrift 25 | 26 | enum ResultCode 27 | { 28 | OK, 29 | TRY_LATER 30 | } 31 | 32 | struct LogEntry 33 | { 34 | 1: string category, 35 | 2: string message 36 | } 37 | 38 | service scribe extends fb303.FacebookService 39 | { 40 | ResultCode Log(1: list messages); 41 | } 42 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | Release 1.0.0 - 2018/12/11 2 | * Use fluentd v1 API 3 | 4 | Release 0.10.14 - 2014/12/19 5 | * add ignore_invalid_record to out_scribe 6 | * improve input performance by reducing the times of emit 7 | 8 | Release 0.10.13 - 2014/10/29 9 | * add validation for msg_format 10 | * use Yajl instead of JSON for parsing record 11 | * add ignore_invalid_record option to ignore invalid record and return OK to scribed 12 | 13 | Release 0.10.12 - 2014/09/04 14 | * support 'log_level' option 15 | * fix to do shutdown plugin instance not to duplicate listen 16 | 17 | Release 0.10.11 - 2013/09/04 18 | * add format_to_json bool option with default value false 19 | 20 | Release 0.10.10 - 2013/02/06 21 | * change 'written entries' log level from 'info' to 'debug' 22 | 23 | Release 0.10.9 - 2011/05/30 24 | * overwrite default value of buffer_chunk_limit to 1MB 25 | * improve memory usage efficiency 26 | 27 | Release 0.10.8 - 2011/04/27 28 | * support msg_format option at in_scribe 29 | 30 | Release 0.10.7 - 2011/02/04 31 | * support add_newline option at out_scribe 32 | 33 | Release 0.10.6 - 2011/01/23 34 | * support add_prefix / remove_prefix option 35 | * update thrift to 0.8.0 36 | * support remove_newline option 37 | 38 | Release 0.10.5 - 2011/11/16 39 | * fixed broken gemspec 40 | 41 | Release 0.10.4 - 2011/11/14 42 | * fixed encoding problem (work-around for thrift-rb) 43 | - contributed by @tagomoris and @frsyuki 44 | 45 | Release 0.10.3 - 2011/10/16 46 | * modify summary of gemspec 47 | 48 | Release 0.10.2 - 2011/10/16 49 | * implement out_scribe plugin 50 | 51 | Release 0.10.1 - 2011/10/16 52 | * added tests for in_scribe 53 | * use Configurable, introduced from fluentd v0.10 54 | 55 | Release 0.10.0 - 2011/10/16 56 | * compatibility changes for fluentd v0.10 57 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/scribe.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | 7 | require 'thrift' 8 | require 'facebook_service' 9 | require 'scribe_types' 10 | 11 | module Scribe 12 | class Client < FacebookService::Client 13 | include ::Thrift::Client 14 | 15 | def Log(messages) 16 | send_Log(messages) 17 | return recv_Log() 18 | end 19 | 20 | def send_Log(messages) 21 | send_message('Log', Log_args, :messages => messages) 22 | end 23 | 24 | def recv_Log() 25 | result = receive_message(Log_result) 26 | return result.success unless result.success.nil? 27 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'Log failed: unknown result') 28 | end 29 | 30 | end 31 | 32 | class Processor < FacebookService::Processor 33 | include ::Thrift::Processor 34 | 35 | def process_Log(seqid, iprot, oprot) 36 | args = read_args(iprot, Log_args) 37 | result = Log_result.new() 38 | result.success = @handler.Log(args.messages) 39 | write_result(result, oprot, 'Log', seqid) 40 | end 41 | 42 | end 43 | 44 | # HELPER FUNCTIONS AND STRUCTURES 45 | 46 | class Log_args 47 | include ::Thrift::Struct, ::Thrift::Struct_Union 48 | MESSAGES = 1 49 | 50 | FIELDS = { 51 | MESSAGES => {:type => ::Thrift::Types::LIST, :name => 'messages', :element => {:type => ::Thrift::Types::STRUCT, :class => LogEntry}} 52 | } 53 | 54 | def struct_fields; FIELDS; end 55 | 56 | def validate 57 | end 58 | 59 | ::Thrift::Struct.generate_accessors self 60 | end 61 | 62 | class Log_result 63 | include ::Thrift::Struct, ::Thrift::Struct_Union 64 | SUCCESS = 0 65 | 66 | FIELDS = { 67 | SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success', :enum_class => ResultCode} 68 | } 69 | 70 | def struct_fields; FIELDS; end 71 | 72 | def validate 73 | unless @success.nil? || ResultCode::VALID_VALUES.include?(@success) 74 | raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field success!') 75 | end 76 | end 77 | 78 | ::Thrift::Struct.generate_accessors self 79 | end 80 | 81 | end 82 | 83 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/fb303.thrift: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | * fb303.thrift 22 | */ 23 | 24 | namespace java com.facebook.fb303 25 | namespace cpp facebook.fb303 26 | namespace perl Facebook.FB303 27 | 28 | /** 29 | * Common status reporting mechanism across all services 30 | */ 31 | enum fb_status { 32 | DEAD = 0, 33 | STARTING = 1, 34 | ALIVE = 2, 35 | STOPPING = 3, 36 | STOPPED = 4, 37 | WARNING = 5, 38 | } 39 | 40 | /** 41 | * Standard base service 42 | */ 43 | service FacebookService { 44 | 45 | /** 46 | * Returns a descriptive name of the service 47 | */ 48 | string getName(), 49 | 50 | /** 51 | * Returns the version of the service 52 | */ 53 | string getVersion(), 54 | 55 | /** 56 | * Gets the status of this service 57 | */ 58 | fb_status getStatus(), 59 | 60 | /** 61 | * User friendly description of status, such as why the service is in 62 | * the dead or warning state, or what is being started or stopped. 63 | */ 64 | string getStatusDetails(), 65 | 66 | /** 67 | * Gets the counters for this service 68 | */ 69 | map getCounters(), 70 | 71 | /** 72 | * Gets the value of a single counter 73 | */ 74 | i64 getCounter(1: string key), 75 | 76 | /** 77 | * Sets an option 78 | */ 79 | void setOption(1: string key, 2: string value), 80 | 81 | /** 82 | * Gets an option 83 | */ 84 | string getOption(1: string key), 85 | 86 | /** 87 | * Gets all options 88 | */ 89 | map getOptions(), 90 | 91 | /** 92 | * Returns a CPU profile over the given time interval (client and server 93 | * must agree on the profile format). 94 | */ 95 | string getCpuProfile(1: i32 profileDurationInSec), 96 | 97 | /** 98 | * Returns the unix time that the server has been running since 99 | */ 100 | i64 aliveSince(), 101 | 102 | /** 103 | * Tell the server to reload its configuration, reopen log files, etc 104 | */ 105 | oneway void reinitialize(), 106 | 107 | /** 108 | * Suggest a shutdown to the server 109 | */ 110 | oneway void shutdown(), 111 | 112 | } 113 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = Scribe input plugin for Fluentd[http://fluentd.org/] 2 | 3 | == Overview 4 | 5 | This is a plugin for fluentd[http://fluentd.org] data collector. This plugin adds the Scribe[https://github.com/facebook/scribe] compatible interface to fluentd. 6 | 7 | == What's Scribe? 8 | 9 | Scribe[https://github.com/facebook/scribe] is a server for aggregating log data streamed in real time from a large number of servers, developed at Facebook. 10 | 11 | It uses Thrift[http://thrift.apache.org/], a cross-language RPC framework, to communicate between clients and servers. 12 | 13 | == What's Scribe plugin for fluent? 14 | 15 | The Scribe plugin for fluentd, which enables fluentd to talk the Scribe protocol. Scribe protocol is defined as follows, in Thrift-IDL format: 16 | 17 | enum ResultCode 18 | { 19 | OK, 20 | TRY_LATER 21 | } 22 | 23 | struct LogEntry 24 | { 25 | 1: string category, 26 | 2: string message 27 | } 28 | 29 | service scribe extends fb303.FacebookService 30 | { 31 | ResultCode Log(1: list messages); 32 | } 33 | 34 | The category field is used as fluentd 'tag'. 35 | 36 | == How to use? 37 | 38 | fluent-plugin-scribe contains both input and output. 39 | 40 | === Scribe Input 41 | 42 | Please add the following configurations to fluent.conf. This allows your Scribe client to import logs through port 1463. 43 | 44 | # Scribe input 45 | 46 | type scribe 47 | port 1463 48 | 49 | 50 | These options are supported. 51 | 52 | * port: port number (default: 1463) 53 | * bind: bind address (default: 0.0.0.0) 54 | * server_type: server architecture either in 'simple', 'threaded', 'thread_pool', 'nonblocking' (default: nonblocking) 55 | * is_framed: use framed protocol or not (default: true) 56 | * add_prefix: prefix string, added to the tag (default: nil) 57 | * msg_format: format of the messages either in 'text', 'json', 'url_param' (default: text) 58 | 59 | === Scribe Output 60 | 61 | Please add the following configurations to fluent.conf. This allows fluentd to output its logs into another Scribe server. Note that fluentd conveys semi-structured data while Scribe conveys unstructured data, thus, 'field_ref' parameter is prepared to specify which field will be transferred. 62 | 63 | # Scribe output 64 | 65 | type scribe 66 | host scribed-host.local 67 | port 1463 68 | field_ref message 69 | 70 | 71 | These options are supported. 72 | 73 | * host: host name or address (default: localhost) 74 | * port: port number (default: 1463) 75 | * field_ref: field name which sent as scribe log message (default: message) 76 | * timeout: thrift protocol timeout (default: 30) 77 | * format_to_json: if true/yes, format entire record as json, and send as message (default: false) 78 | * remove_prefix: prefix string, removed from the tag (default: nil) 79 | 80 | == For Developers 81 | 82 | To run fluentd with this plugin on chaging, 83 | 84 | $ bundle # (or 'bundle update') 85 | $ bundle exec fluentd -v -v -v -c example.conf 86 | 87 | Then please execute the sample client. 88 | 89 | $ bundle exec bin/fluent-scribe-remote 90 | 91 | == Contributors 92 | 93 | * {Satoshi Tagomori}[https://github.com/tagomoris] 94 | * {Sadayuki Furuhashi}[https://github.com/frsyuki] 95 | * {Ken Robertson}[https://github.com/krobertson] 96 | 97 | == Copyright 98 | 99 | Copyright:: Copyright (c) 2011 Treasure Data, Inc. 100 | License:: Apache License, Version 2.0 101 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_scribe.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluent 3 | # 4 | # Copyright (C) 2011 Kazuki Ohta 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | module Fluent 19 | class ScribeOutput < BufferedOutput 20 | Fluent::Plugin.register_output('scribe', self) 21 | 22 | config_param :host, :string, :default => 'localhost' 23 | config_param :port, :integer, :default => 1463 24 | config_param :field_ref, :string, :default => 'message' 25 | config_param :timeout, :integer, :default => 30 26 | 27 | config_param :remove_prefix, :string, :default => nil 28 | config_param :add_newline, :bool, :default => false 29 | config_param :ignore_invalid_record, :bool, :default => false 30 | config_param :default_category, :string, :default => 'unknown' 31 | config_param :format_to_json, :bool, :default => false 32 | 33 | unless method_defined?(:log) 34 | define_method(:log) { $log } 35 | end 36 | 37 | def initialize 38 | require 'thrift' 39 | $:.unshift File.join(File.dirname(__FILE__), 'thrift') 40 | require 'fb303_types' 41 | require 'fb303_constants' 42 | require 'facebook_service' 43 | require 'scribe_types' 44 | require 'scribe_constants' 45 | require 'scribe' 46 | super 47 | end 48 | 49 | def configure(conf) 50 | # override default buffer_chunk_limit 51 | conf['buffer_chunk_limit'] ||= '1m' 52 | 53 | super 54 | end 55 | 56 | def start 57 | super 58 | 59 | if @remove_prefix 60 | @removed_prefix_string = @remove_prefix + '.' 61 | @removed_length = @removed_prefix_string.length 62 | end 63 | end 64 | 65 | def shutdown 66 | super 67 | end 68 | 69 | def format(tag, time, record) 70 | if @remove_prefix and 71 | ( (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or 72 | tag == @remove_prefix) 73 | [(tag[@removed_length..-1] || @default_category), record].to_msgpack 74 | else 75 | [tag, record].to_msgpack 76 | end 77 | end 78 | 79 | def write(chunk) 80 | socket = Thrift::Socket.new @host, @port, @timeout 81 | transport = Thrift::FramedTransport.new socket 82 | protocol = Thrift::BinaryProtocol.new transport, false, false 83 | client = Scribe::Client.new protocol 84 | 85 | transport.open 86 | begin 87 | entries = [] 88 | 89 | chunk.msgpack_each do |arr| 90 | tag, record = arr 91 | next unless @format_to_json || record.has_key?(@field_ref) 92 | 93 | message = @format_to_json ? record : record[@field_ref] 94 | 95 | if message.kind_of?(Array) or message.kind_of?(Hash) 96 | begin 97 | message = message.to_json 98 | rescue => e 99 | if @ignore_invalid_record 100 | # This warning can be disabled by 'log_level error' 101 | log.warn "got invalid message", message: message, error: e, error_class: e.class 102 | next 103 | else 104 | # Keep existence behaviour 105 | raise 106 | end 107 | end 108 | end 109 | 110 | if @add_newline 111 | message = message + "\n" 112 | end 113 | 114 | entry = LogEntry.new 115 | entry.category = tag 116 | entry.message = message.force_encoding('ASCII-8BIT') 117 | 118 | entries << entry 119 | end 120 | 121 | log.debug "Writing #{entries.count} entries to scribe" 122 | client.Log(entries) 123 | ensure 124 | transport.close 125 | end 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /test/plugin/test_in_scribe.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'fluent/test' 3 | require 'fluent/plugin/in_scribe' 4 | 5 | require 'thrift' 6 | 7 | class ScribeInputTest < Test::Unit::TestCase 8 | def setup 9 | Fluent::Test.setup 10 | end 11 | 12 | CONFIG = %[ 13 | port 14630 14 | bind 127.0.0.1 15 | ] 16 | 17 | def create_driver(conf=CONFIG) 18 | Fluent::Test::InputTestDriver.new(Fluent::ScribeInput).configure(conf) 19 | end 20 | 21 | def shutdown_driver(driver) 22 | return unless driver.instance.instance_eval{ @thread } 23 | driver.instance.shutdown 24 | driver.instance.instance_eval{ @thread && @thread.join } 25 | end 26 | 27 | def test_configure 28 | d = create_driver 29 | assert_equal 14630, d.instance.port 30 | assert_equal '127.0.0.1', d.instance.bind 31 | assert_equal false, d.instance.remove_newline 32 | end 33 | 34 | def test_time 35 | d = create_driver 36 | 37 | time = Time.parse("2011-01-02 13:14:15 UTC").to_i 38 | Fluent::Engine.now = time 39 | 40 | d.expect_emit "tag1", time, {"message"=>'aiueo'} 41 | d.expect_emit "tag2", time, {"message"=>'aiueo'} 42 | 43 | emits = [ 44 | ['tag1', time, {"message"=>'aiueo'}], 45 | ['tag2', time, {"message"=>'aiueo'}], 46 | ] 47 | d.run do 48 | emits.each { |tag, time, record| 49 | res = message_send(tag, record['message']) 50 | assert_equal ResultCode::OK, res 51 | } 52 | end 53 | 54 | shutdown_driver(d) 55 | end 56 | 57 | def test_add_prefix 58 | d = create_driver(CONFIG + %[ 59 | add_prefix scribe 60 | ]) 61 | 62 | time = Time.parse("2011-01-02 13:14:15 UTC").to_i 63 | Fluent::Engine.now = time 64 | 65 | d.expect_emit "scribe.tag1", time, {"message"=>'aiueo'} 66 | d.expect_emit "scribe.tag2", time, {"message"=>'aiueo'} 67 | 68 | emits = [ 69 | ['tag1', time, {"message"=>'aiueo'}], 70 | ['tag2', time, {"message"=>'aiueo'}], 71 | ] 72 | d.run do 73 | emits.each { |tag, time, record| 74 | res = message_send(tag, record['message']) 75 | assert_equal ResultCode::OK, res 76 | } 77 | end 78 | 79 | shutdown_driver(d) 80 | 81 | d2 = create_driver(CONFIG + %[ 82 | add_prefix scribe.input 83 | ]) 84 | 85 | time = Time.parse("2011-01-02 13:14:15 UTC").to_i 86 | Fluent::Engine.now = time 87 | 88 | d2.expect_emit "scribe.input.tag3", time, {"message"=>'aiueo'} 89 | d2.expect_emit "scribe.input.tag4", time, {"message"=>'aiueo'} 90 | 91 | emits = [ 92 | ['tag3', time, {"message"=>'aiueo'}], 93 | ['tag4', time, {"message"=>'aiueo'}], 94 | ] 95 | d2.run do 96 | emits.each { |tag, time, record| 97 | res = message_send(tag, record['message']) 98 | assert_equal ResultCode::OK, res 99 | } 100 | end 101 | 102 | shutdown_driver(d2) 103 | end 104 | 105 | def test_remove_newline 106 | d = create_driver(CONFIG + %[ 107 | remove_newline true 108 | ]) 109 | assert_equal true, d.instance.remove_newline 110 | 111 | time = Time.parse("2011-01-02 13:14:15 UTC").to_i 112 | Fluent::Engine.now = time 113 | 114 | d.expect_emit "tag1", time, {"message"=>'aiueo'} 115 | d.expect_emit "tag2", time, {"message"=>'kakikukeko'} 116 | d.expect_emit "tag3", time, {"message"=>'sasisuseso'} 117 | 118 | emits = [ 119 | ['tag1', time, {"message"=>"aiueo\n"}], 120 | ['tag2', time, {"message"=>"kakikukeko\n"}], 121 | ['tag3', time, {"message"=>"sasisuseso"}], 122 | ] 123 | d.run do 124 | emits.each { |tag, time, record| 125 | res = message_send(tag, record['message']) 126 | assert_equal ResultCode::OK, res 127 | } 128 | end 129 | 130 | shutdown_driver(d) 131 | end 132 | 133 | def test_msg_format_json 134 | d = create_driver(CONFIG + %[ 135 | msg_format json 136 | ]) 137 | assert_equal :json, d.instance.msg_format 138 | 139 | time = Time.parse("2011-01-02 13:14:15 UTC").to_i 140 | Fluent::Engine.now = time 141 | 142 | d.expect_emit "tag1", time, {"a"=>1} 143 | d.expect_emit "tag2", time, {"a"=>1, "b"=>2} 144 | d.expect_emit "tag3", time, {"a"=>1, "b"=>2, "c"=>3} 145 | 146 | emits = [ 147 | ['tag1', time, {"a"=>1}.to_json], 148 | ['tag2', time, {"a"=>1, "b"=>2}.to_json], 149 | ['tag3', time, {"a"=>1, "b"=>2, "c"=>3}.to_json], 150 | ] 151 | d.run do 152 | emits.each { |tag, time, message| 153 | res = message_send(tag, message) 154 | assert_equal ResultCode::OK, res 155 | } 156 | end 157 | 158 | shutdown_driver(d) 159 | end 160 | 161 | data do 162 | Fluent::ScribeInput.new 163 | { 164 | 'true' => [ResultCode::OK, true], 165 | 'false' => [ResultCode::TRY_LATER, false] 166 | } 167 | end 168 | def test_msg_format_json_with_ignore_invalid_record(data) 169 | result, opt = data 170 | d = create_driver(CONFIG + %[ 171 | msg_format json 172 | ignore_invalid_record #{opt} 173 | ]) 174 | assert_equal :json, d.instance.msg_format 175 | assert_equal opt, d.instance.ignore_invalid_record 176 | 177 | time = Time.parse("2011-01-02 13:14:15 UTC").to_i 178 | Fluent::Engine.now = time 179 | 180 | emits = [['tag1', time, '{"a":']] 181 | d.run do 182 | emits.each { |tag, time, message| 183 | res = message_send(tag, message) 184 | assert_equal result, res 185 | } 186 | end 187 | 188 | shutdown_driver(d) 189 | end 190 | 191 | def test_msg_format_url_param 192 | d = create_driver(CONFIG + %[ 193 | msg_format url_param 194 | ]) 195 | assert_equal :url_param, d.instance.msg_format 196 | 197 | time = Time.parse("2011-01-02 13:14:15 UTC").to_i 198 | Fluent::Engine.now = time 199 | 200 | d.expect_emit "tag0", time, {} 201 | d.expect_emit "tag1", time, {"a"=>'1'} 202 | d.expect_emit "tag2", time, {"a"=>'1', "b"=>'2'} 203 | d.expect_emit "tag3", time, {"a"=>'1', "b"=>'2', "c"=>'3'} 204 | d.expect_emit "tag4", time, {"a"=>'1', "b"=>'2', "c"=>'3=4'} 205 | 206 | emits = [ 207 | ['tag0', time, ""], 208 | ['tag1', time, "a=1"], 209 | ['tag2', time, "a=1&b=2"], 210 | ['tag3', time, "a=1&b=2&c=3"], 211 | ['tag4', time, "a=1&b=2&c=3=4"], 212 | ] 213 | d.run do 214 | emits.each { |tag, time, message| 215 | res = message_send(tag, message) 216 | assert_equal ResultCode::OK, res 217 | } 218 | end 219 | 220 | shutdown_driver(d) 221 | end 222 | 223 | def message_send(tag, msg) 224 | socket = Thrift::Socket.new '127.0.0.1', 14630 225 | transport = Thrift::FramedTransport.new socket 226 | protocol = Thrift::BinaryProtocol.new transport, false, false 227 | client = Scribe::Client.new protocol 228 | transport.open 229 | raw_sock = socket.to_io 230 | raw_sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1 231 | entry = LogEntry.new 232 | entry.category = tag 233 | entry.message = msg.to_s 234 | res = client.Log([entry]) 235 | transport.close 236 | res 237 | end 238 | end 239 | -------------------------------------------------------------------------------- /lib/fluent/plugin/in_scribe.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Fluent 3 | # 4 | # Copyright (C) 2011 Kazuki Ohta 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | require 'fluent/plugin/input' 20 | 21 | module Fluent 22 | class ScribeInput < ::Fluent::Plugin::Input 23 | Plugin.register_input('scribe', self) 24 | 25 | SUPPORTED_FORMAT = { 26 | 'text' => :text, 27 | 'json' => :json, 28 | 'url_param' => :url_param, 29 | } 30 | 31 | config_param :port, :integer, :default => 1463 32 | config_param :bind, :string, :default => '0.0.0.0' 33 | config_param :server_type, :string, :default => 'nonblocking' 34 | config_param :is_framed, :bool, :default => true 35 | config_param :body_size_limit, :size, :default => 32*1024*1024 # TODO default 36 | config_param :add_prefix, :string, :default => nil 37 | config_param :remove_newline, :bool, :default => false 38 | config_param :ignore_invalid_record, :bool, :default => false 39 | config_param :msg_format, :default => :text do |val| 40 | f = SUPPORTED_FORMAT[val] 41 | raise ConfigError, "unsupported msg_format: #{val}" unless f 42 | f 43 | end 44 | 45 | unless method_defined?(:log) 46 | define_method(:log) { $log } 47 | end 48 | 49 | def initialize 50 | require 'cgi' 51 | require 'yajl' 52 | require 'thrift' 53 | $:.unshift File.join(File.dirname(__FILE__), 'thrift') 54 | require 'fb303_types' 55 | require 'fb303_constants' 56 | require 'facebook_service' 57 | require 'scribe_types' 58 | require 'scribe_constants' 59 | require 'scribe' 60 | super 61 | end 62 | 63 | def configure(conf) 64 | super 65 | end 66 | 67 | def start 68 | log.debug "listening scribe on #{@bind}:#{@port}" 69 | 70 | handler = FluentScribeHandler.new 71 | handler.add_prefix = @add_prefix 72 | handler.remove_newline = @remove_newline 73 | handler.msg_format = @msg_format 74 | handler.ignore_invalid_record = @ignore_invalid_record 75 | handler.logger = log 76 | handler.router = router 77 | processor = Scribe::Processor.new handler 78 | 79 | @transport = Thrift::ServerSocket.new @bind, @port 80 | if @is_framed 81 | transport_factory = Thrift::FramedTransportFactory.new 82 | else 83 | transport_factory = Thrift::BufferedTransportFactory.new 84 | end 85 | 86 | # 2011/09/29 Kazuki Ohta 87 | # This section is a workaround to set strict_read and strict_write option. 88 | # Ruby-Thrift 0.7 set them both 'true' in default, but Scribe protocol set 89 | # them both 'false'. 90 | protocol_factory = Thrift::BinaryProtocolFactory.new 91 | protocol_factory.instance_eval {|obj| 92 | def get_protocol(trans) # override 93 | return Thrift::BinaryProtocol.new(trans, 94 | strict_read=false, 95 | strict_write=false) 96 | end 97 | } 98 | 99 | case @server_type 100 | when 'simple' 101 | @server = Thrift::SimpleServer.new processor, @transport, transport_factory, protocol_factory 102 | when 'threaded' 103 | @server = Thrift::ThreadedServer.new processor, @transport, transport_factory, protocol_factory 104 | when 'thread_pool' 105 | @server = Thrift::ThreadPoolServer.new processor, @transport, transport_factory, protocol_factory 106 | when 'nonblocking' 107 | @server = Thrift::NonblockingServer.new processor, @transport, transport_factory, protocol_factory 108 | else 109 | raise ConfigError, "in_scribe: unsupported server_type '#{@server_type}'" 110 | end 111 | @thread = Thread.new(&method(:run)) 112 | end 113 | 114 | def shutdown 115 | @transport.close unless @transport.closed? 116 | #@thread.join # TODO 117 | end 118 | 119 | def run 120 | @server.serve 121 | rescue => e 122 | log.error "unexpected error", :error => e.inspect 123 | log.error_backtrace 124 | end 125 | 126 | class FluentScribeHandler 127 | attr_accessor :add_prefix 128 | attr_accessor :remove_newline 129 | attr_accessor :msg_format 130 | attr_accessor :ignore_invalid_record 131 | attr_accessor :logger # Use logger instead of log to avoid confusion with Log method 132 | attr_accessor :router 133 | 134 | def Log(msgs) 135 | bucket = {} # tag -> events(array of [time,record]) 136 | time_now = Engine.now 137 | begin 138 | msgs.each do |msg| 139 | begin 140 | record = create_record(msg) 141 | rescue => e 142 | if @ignore_invalid_record 143 | # This warning can be disabled by 'log_level error' 144 | logger.warn "got invalid record", message: msg, error_class: e.class, error: e 145 | next 146 | end 147 | 148 | raise 149 | end 150 | tag = @add_prefix ? @add_prefix + '.' + msg.category : msg.category 151 | bucket[tag] ||= [] 152 | bucket[tag].push([time_now,record]) 153 | end 154 | rescue => e 155 | logger.error "unexpected error", error_class: e.class, error: e 156 | logger.error_backtrace 157 | return ResultCode::TRY_LATER 158 | end 159 | 160 | begin 161 | bucket.each do |tag,events| 162 | router.emit_array(tag, events) 163 | end 164 | return ResultCode::OK 165 | rescue => e 166 | logger.error "unexpected error", error_class: e.class, error: e 167 | logger.error_backtrace 168 | return ResultCode::TRY_LATER 169 | end 170 | end 171 | 172 | private 173 | def create_record(msg) 174 | case @msg_format 175 | when :text 176 | if @remove_newline 177 | return { 'message' => msg.message.force_encoding('UTF-8').chomp } 178 | else 179 | return { 'message' => msg.message.force_encoding('UTF-8') } 180 | end 181 | when :json 182 | js = Yajl.load(msg.message.force_encoding('UTF-8')) 183 | raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash) 184 | return js 185 | when :url_param 186 | s = msg.message.force_encoding('UTF-8') 187 | return Hash[ s.split('&').map { |kv| 188 | k,v = kv.split('=', 2); 189 | [CGI.unescape(k), CGI.unescape(v)] 190 | } 191 | ] 192 | else 193 | raise 'Invalid format: #{@msg_format}' 194 | end 195 | end 196 | end 197 | end 198 | end 199 | -------------------------------------------------------------------------------- /test/plugin/test_out_scribe.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'fluent/test' 3 | require 'fluent/plugin/out_scribe' 4 | 5 | require 'thrift' 6 | 7 | class ScribeOutputTest < Test::Unit::TestCase 8 | CONFIG = %[ 9 | host 127.0.0.1 10 | port 14630 11 | ] 12 | CONFIG_TO_JSON = %[ 13 | host 127.0.0.1 14 | port 14630 15 | format_to_json yes 16 | ] 17 | 18 | def create_driver(conf=CONFIG, tag='test') 19 | Fluent::Test::BufferedOutputTestDriver.new(Fluent::ScribeOutput, tag).configure(conf) 20 | end 21 | 22 | def test_configure 23 | d = create_driver('') 24 | 25 | assert_equal 'localhost', d.instance.host 26 | assert_equal 1463, d.instance.port 27 | assert_equal 'message', d.instance.field_ref 28 | assert_equal 30, d.instance.timeout 29 | assert_equal 'unknown', d.instance.default_category 30 | assert_nil d.instance.remove_prefix 31 | 32 | d = create_driver 33 | 34 | assert_equal '127.0.0.1', d.instance.host 35 | assert_equal 14630, d.instance.port 36 | end 37 | 38 | def test_format 39 | time = Time.parse("2011-12-21 13:14:15 UTC").to_i 40 | 41 | d = create_driver 42 | d.emit({"message" => "testing first", "message2" => "testing first another data"}, time) 43 | d.emit({"message" => "testing second", "message2" => "testing second another data"}, time) 44 | d.emit({"message" => "testing third", "message2" => "testing third another data"}, time) 45 | d.expect_format [d.tag, {"message" => "testing first", "message2" => "testing first another data"}].to_msgpack 46 | d.expect_format [d.tag, {"message" => "testing second", "message2" => "testing second another data"}].to_msgpack 47 | d.expect_format [d.tag, {"message" => "testing third", "message2" => "testing third another data"}].to_msgpack 48 | d.run 49 | 50 | d = create_driver(CONFIG + %[ 51 | field_ref message2 52 | remove_prefix test 53 | ], 'test.scribeplugin') 54 | assert_equal 'test.scribeplugin', d.tag 55 | 56 | d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time) 57 | d.emit({"message" => "xxx testing second", "message2" => "xxx testing second another data"}, time) 58 | d.expect_format ['scribeplugin', {"message" => "xxx testing first", "message2" => "xxx testing first another data"}].to_msgpack 59 | d.expect_format ['scribeplugin', {"message" => "xxx testing second", "message2" => "xxx testing second another data"}].to_msgpack 60 | d.run 61 | 62 | d = create_driver(CONFIG + %[ 63 | field_ref message2 64 | remove_prefix test 65 | ], 'xxx.test.scribeplugin') 66 | assert_equal 'xxx.test.scribeplugin', d.tag 67 | d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time) 68 | d.expect_format ['xxx.test.scribeplugin', {"message" => "xxx testing first", "message2" => "xxx testing first another data"}].to_msgpack 69 | d.run 70 | 71 | d = create_driver(CONFIG + %[ 72 | field_ref message2 73 | remove_prefix test 74 | ], 'test') 75 | assert_equal 'test', d.tag 76 | d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time) 77 | d.expect_format ['unknown', {"message" => "xxx testing first", "message2" => "xxx testing first another data"}].to_msgpack 78 | d.run 79 | 80 | d = create_driver(CONFIG + %[ 81 | field_ref message2 82 | remove_prefix test 83 | add_newline true 84 | ], 'test.scribeplugin') 85 | assert_equal 'test.scribeplugin', d.tag 86 | 87 | d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time) 88 | d.emit({"message" => "xxx testing second", "message2" => "xxx testing second another data"}, time) 89 | d.expect_format ['scribeplugin', {"message" => "xxx testing first", "message2" => "xxx testing first another data"}].to_msgpack 90 | d.expect_format ['scribeplugin', {"message" => "xxx testing second", "message2" => "xxx testing second another data"}].to_msgpack 91 | d.run 92 | end 93 | 94 | def test_write 95 | time = Time.parse("2011-12-21 13:14:15 UTC").to_i 96 | 97 | d = create_driver 98 | d.emit({"message" => "testing first", "message2" => "testing first another data"}, time) 99 | d.emit({"message" => "testing second", "message2" => "testing second another data"}, time) 100 | d.emit({"message" => "testing third", "message2" => "testing third another data"}, time) 101 | result = d.run 102 | assert_equal ResultCode::OK, result 103 | assert_equal [[d.tag, 'testing first'], [d.tag, 'testing second'], [d.tag,'testing third']], $handler.last 104 | 105 | d = create_driver(CONFIG + %[ 106 | field_ref message2 107 | remove_prefix test 108 | ], 'test.scribeplugin') 109 | assert_equal 'test.scribeplugin', d.tag 110 | d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time) 111 | d.emit({"message" => "xxx testing second", "message2" => "xxx testing second another data"}, time) 112 | result = d.run 113 | assert_equal ResultCode::OK, result 114 | assert_equal [['scribeplugin', 'xxx testing first another data'], ['scribeplugin', 'xxx testing second another data']], $handler.last 115 | 116 | d = create_driver(CONFIG + %[ 117 | field_ref message2 118 | remove_prefix test 119 | add_newline true 120 | ], 'test.scribeplugin') 121 | assert_equal 'test.scribeplugin', d.tag 122 | d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time) 123 | d.emit({"message" => "xxx testing second", "message2" => "xxx testing second another data"}, time) 124 | result = d.run 125 | assert_equal ResultCode::OK, result 126 | assert_equal [['scribeplugin', "xxx testing first another data\n"], ['scribeplugin', "xxx testing second another data\n"]], $handler.last 127 | 128 | d = create_driver(CONFIG + %[ 129 | field_ref message2 130 | remove_prefix test 131 | ], 'xxx.test.scribeplugin') 132 | assert_equal 'xxx.test.scribeplugin', d.tag 133 | d.emit({"message" => "yyy testing first", "message2" => "yyy testing first another data"}, time) 134 | result = d.run 135 | assert_equal ResultCode::OK, result 136 | assert_equal [['xxx.test.scribeplugin', 'yyy testing first another data']], $handler.last 137 | 138 | d = create_driver(CONFIG + %[ 139 | field_ref message2 140 | remove_prefix test 141 | ], 'test') 142 | assert_equal 'test', d.tag 143 | d.emit({"message" => "zzz testing first", "message2" => "zzz testing first another data"}, time) 144 | result = d.run 145 | assert_equal ResultCode::OK, result 146 | assert_equal [[d.instance.default_category, 'zzz testing first another data']], $handler.last 147 | end 148 | 149 | def test_write_to_json 150 | time = Time.parse("2011-12-21 13:14:15 UTC").to_i 151 | 152 | d = create_driver(CONFIG_TO_JSON) 153 | e1 = {"message" => "testing first", "message2" => "testing first another data"} 154 | e2 = {"message" => "testing second", "message2" => "testing second another data"} 155 | e3 = {"message" => "testing third", "message2" => "testing third another data"} 156 | [e1,e2,e3].each{|e| d.emit(e, time)} 157 | result = d.run 158 | assert_equal ResultCode::OK, result 159 | 160 | events = $handler.last 161 | assert_equal d.tag, events[0][0] 162 | assert_equal e1, JSON.parse(events[0][1]) 163 | assert_equal d.tag, events[1][0] 164 | assert_equal e2, JSON.parse(events[1][1]) 165 | assert_equal d.tag, events[2][0] 166 | assert_equal e3, JSON.parse(events[2][1]) 167 | end 168 | 169 | def setup 170 | Fluent::Test.setup 171 | $handler = TestScribeServerHandler.new 172 | @dummy_server_thread = Thread.new do 173 | begin 174 | transport = Thrift::ServerSocket.new '127.0.0.1', 14630 175 | processor = Scribe::Processor.new $handler 176 | transport_factory = Thrift::FramedTransportFactory.new 177 | protocol_factory = Thrift::BinaryProtocolFactory.new 178 | protocol_factory.instance_eval {|obj| 179 | def get_protocol(trans) # override 180 | Thrift::BinaryProtocol.new(trans, strict_read=false, strict_write=false) 181 | end 182 | } 183 | server = Thrift::SimpleServer.new processor, transport, transport_factory, protocol_factory 184 | server.serve 185 | ensure 186 | transport.close unless transport.closed? 187 | end 188 | end 189 | sleep 0.1 # boo... 190 | end 191 | 192 | def teardown 193 | @dummy_server_thread.kill 194 | @dummy_server_thread.join 195 | end 196 | 197 | class TestScribeServerHandler 198 | attr :last 199 | def initialize 200 | @last = [] 201 | end 202 | def Log(msgs) 203 | @last = msgs.map{|msg| [msg.category, msg.message.force_encoding('UTF-8')]} 204 | ResultCode::OK 205 | end 206 | end 207 | end 208 | -------------------------------------------------------------------------------- /lib/fluent/plugin/thrift/facebook_service.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | 7 | require 'thrift' 8 | require 'fb303_types' 9 | 10 | module FacebookService 11 | class Client 12 | include ::Thrift::Client 13 | 14 | def getName() 15 | send_getName() 16 | return recv_getName() 17 | end 18 | 19 | def send_getName() 20 | send_message('getName', GetName_args) 21 | end 22 | 23 | def recv_getName() 24 | result = receive_message(GetName_result) 25 | return result.success unless result.success.nil? 26 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getName failed: unknown result') 27 | end 28 | 29 | def getVersion() 30 | send_getVersion() 31 | return recv_getVersion() 32 | end 33 | 34 | def send_getVersion() 35 | send_message('getVersion', GetVersion_args) 36 | end 37 | 38 | def recv_getVersion() 39 | result = receive_message(GetVersion_result) 40 | return result.success unless result.success.nil? 41 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getVersion failed: unknown result') 42 | end 43 | 44 | def getStatus() 45 | send_getStatus() 46 | return recv_getStatus() 47 | end 48 | 49 | def send_getStatus() 50 | send_message('getStatus', GetStatus_args) 51 | end 52 | 53 | def recv_getStatus() 54 | result = receive_message(GetStatus_result) 55 | return result.success unless result.success.nil? 56 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getStatus failed: unknown result') 57 | end 58 | 59 | def getStatusDetails() 60 | send_getStatusDetails() 61 | return recv_getStatusDetails() 62 | end 63 | 64 | def send_getStatusDetails() 65 | send_message('getStatusDetails', GetStatusDetails_args) 66 | end 67 | 68 | def recv_getStatusDetails() 69 | result = receive_message(GetStatusDetails_result) 70 | return result.success unless result.success.nil? 71 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getStatusDetails failed: unknown result') 72 | end 73 | 74 | def getCounters() 75 | send_getCounters() 76 | return recv_getCounters() 77 | end 78 | 79 | def send_getCounters() 80 | send_message('getCounters', GetCounters_args) 81 | end 82 | 83 | def recv_getCounters() 84 | result = receive_message(GetCounters_result) 85 | return result.success unless result.success.nil? 86 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getCounters failed: unknown result') 87 | end 88 | 89 | def getCounter(key) 90 | send_getCounter(key) 91 | return recv_getCounter() 92 | end 93 | 94 | def send_getCounter(key) 95 | send_message('getCounter', GetCounter_args, :key => key) 96 | end 97 | 98 | def recv_getCounter() 99 | result = receive_message(GetCounter_result) 100 | return result.success unless result.success.nil? 101 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getCounter failed: unknown result') 102 | end 103 | 104 | def setOption(key, value) 105 | send_setOption(key, value) 106 | recv_setOption() 107 | end 108 | 109 | def send_setOption(key, value) 110 | send_message('setOption', SetOption_args, :key => key, :value => value) 111 | end 112 | 113 | def recv_setOption() 114 | result = receive_message(SetOption_result) 115 | return 116 | end 117 | 118 | def getOption(key) 119 | send_getOption(key) 120 | return recv_getOption() 121 | end 122 | 123 | def send_getOption(key) 124 | send_message('getOption', GetOption_args, :key => key) 125 | end 126 | 127 | def recv_getOption() 128 | result = receive_message(GetOption_result) 129 | return result.success unless result.success.nil? 130 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getOption failed: unknown result') 131 | end 132 | 133 | def getOptions() 134 | send_getOptions() 135 | return recv_getOptions() 136 | end 137 | 138 | def send_getOptions() 139 | send_message('getOptions', GetOptions_args) 140 | end 141 | 142 | def recv_getOptions() 143 | result = receive_message(GetOptions_result) 144 | return result.success unless result.success.nil? 145 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getOptions failed: unknown result') 146 | end 147 | 148 | def getCpuProfile(profileDurationInSec) 149 | send_getCpuProfile(profileDurationInSec) 150 | return recv_getCpuProfile() 151 | end 152 | 153 | def send_getCpuProfile(profileDurationInSec) 154 | send_message('getCpuProfile', GetCpuProfile_args, :profileDurationInSec => profileDurationInSec) 155 | end 156 | 157 | def recv_getCpuProfile() 158 | result = receive_message(GetCpuProfile_result) 159 | return result.success unless result.success.nil? 160 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'getCpuProfile failed: unknown result') 161 | end 162 | 163 | def aliveSince() 164 | send_aliveSince() 165 | return recv_aliveSince() 166 | end 167 | 168 | def send_aliveSince() 169 | send_message('aliveSince', AliveSince_args) 170 | end 171 | 172 | def recv_aliveSince() 173 | result = receive_message(AliveSince_result) 174 | return result.success unless result.success.nil? 175 | raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'aliveSince failed: unknown result') 176 | end 177 | 178 | def reinitialize() 179 | send_reinitialize() 180 | end 181 | 182 | def send_reinitialize() 183 | send_message('reinitialize', Reinitialize_args) 184 | end 185 | def shutdown() 186 | send_shutdown() 187 | end 188 | 189 | def send_shutdown() 190 | send_message('shutdown', Shutdown_args) 191 | end 192 | end 193 | 194 | class Processor 195 | include ::Thrift::Processor 196 | 197 | def process_getName(seqid, iprot, oprot) 198 | args = read_args(iprot, GetName_args) 199 | result = GetName_result.new() 200 | result.success = @handler.getName() 201 | write_result(result, oprot, 'getName', seqid) 202 | end 203 | 204 | def process_getVersion(seqid, iprot, oprot) 205 | args = read_args(iprot, GetVersion_args) 206 | result = GetVersion_result.new() 207 | result.success = @handler.getVersion() 208 | write_result(result, oprot, 'getVersion', seqid) 209 | end 210 | 211 | def process_getStatus(seqid, iprot, oprot) 212 | args = read_args(iprot, GetStatus_args) 213 | result = GetStatus_result.new() 214 | result.success = @handler.getStatus() 215 | write_result(result, oprot, 'getStatus', seqid) 216 | end 217 | 218 | def process_getStatusDetails(seqid, iprot, oprot) 219 | args = read_args(iprot, GetStatusDetails_args) 220 | result = GetStatusDetails_result.new() 221 | result.success = @handler.getStatusDetails() 222 | write_result(result, oprot, 'getStatusDetails', seqid) 223 | end 224 | 225 | def process_getCounters(seqid, iprot, oprot) 226 | args = read_args(iprot, GetCounters_args) 227 | result = GetCounters_result.new() 228 | result.success = @handler.getCounters() 229 | write_result(result, oprot, 'getCounters', seqid) 230 | end 231 | 232 | def process_getCounter(seqid, iprot, oprot) 233 | args = read_args(iprot, GetCounter_args) 234 | result = GetCounter_result.new() 235 | result.success = @handler.getCounter(args.key) 236 | write_result(result, oprot, 'getCounter', seqid) 237 | end 238 | 239 | def process_setOption(seqid, iprot, oprot) 240 | args = read_args(iprot, SetOption_args) 241 | result = SetOption_result.new() 242 | @handler.setOption(args.key, args.value) 243 | write_result(result, oprot, 'setOption', seqid) 244 | end 245 | 246 | def process_getOption(seqid, iprot, oprot) 247 | args = read_args(iprot, GetOption_args) 248 | result = GetOption_result.new() 249 | result.success = @handler.getOption(args.key) 250 | write_result(result, oprot, 'getOption', seqid) 251 | end 252 | 253 | def process_getOptions(seqid, iprot, oprot) 254 | args = read_args(iprot, GetOptions_args) 255 | result = GetOptions_result.new() 256 | result.success = @handler.getOptions() 257 | write_result(result, oprot, 'getOptions', seqid) 258 | end 259 | 260 | def process_getCpuProfile(seqid, iprot, oprot) 261 | args = read_args(iprot, GetCpuProfile_args) 262 | result = GetCpuProfile_result.new() 263 | result.success = @handler.getCpuProfile(args.profileDurationInSec) 264 | write_result(result, oprot, 'getCpuProfile', seqid) 265 | end 266 | 267 | def process_aliveSince(seqid, iprot, oprot) 268 | args = read_args(iprot, AliveSince_args) 269 | result = AliveSince_result.new() 270 | result.success = @handler.aliveSince() 271 | write_result(result, oprot, 'aliveSince', seqid) 272 | end 273 | 274 | def process_reinitialize(seqid, iprot, oprot) 275 | args = read_args(iprot, Reinitialize_args) 276 | @handler.reinitialize() 277 | return 278 | end 279 | 280 | def process_shutdown(seqid, iprot, oprot) 281 | args = read_args(iprot, Shutdown_args) 282 | @handler.shutdown() 283 | return 284 | end 285 | 286 | end 287 | 288 | # HELPER FUNCTIONS AND STRUCTURES 289 | 290 | class GetName_args 291 | include ::Thrift::Struct, ::Thrift::Struct_Union 292 | 293 | FIELDS = { 294 | 295 | } 296 | 297 | def struct_fields; FIELDS; end 298 | 299 | def validate 300 | end 301 | 302 | ::Thrift::Struct.generate_accessors self 303 | end 304 | 305 | class GetName_result 306 | include ::Thrift::Struct, ::Thrift::Struct_Union 307 | SUCCESS = 0 308 | 309 | FIELDS = { 310 | SUCCESS => {:type => ::Thrift::Types::STRING, :name => 'success'} 311 | } 312 | 313 | def struct_fields; FIELDS; end 314 | 315 | def validate 316 | end 317 | 318 | ::Thrift::Struct.generate_accessors self 319 | end 320 | 321 | class GetVersion_args 322 | include ::Thrift::Struct, ::Thrift::Struct_Union 323 | 324 | FIELDS = { 325 | 326 | } 327 | 328 | def struct_fields; FIELDS; end 329 | 330 | def validate 331 | end 332 | 333 | ::Thrift::Struct.generate_accessors self 334 | end 335 | 336 | class GetVersion_result 337 | include ::Thrift::Struct, ::Thrift::Struct_Union 338 | SUCCESS = 0 339 | 340 | FIELDS = { 341 | SUCCESS => {:type => ::Thrift::Types::STRING, :name => 'success'} 342 | } 343 | 344 | def struct_fields; FIELDS; end 345 | 346 | def validate 347 | end 348 | 349 | ::Thrift::Struct.generate_accessors self 350 | end 351 | 352 | class GetStatus_args 353 | include ::Thrift::Struct, ::Thrift::Struct_Union 354 | 355 | FIELDS = { 356 | 357 | } 358 | 359 | def struct_fields; FIELDS; end 360 | 361 | def validate 362 | end 363 | 364 | ::Thrift::Struct.generate_accessors self 365 | end 366 | 367 | class GetStatus_result 368 | include ::Thrift::Struct, ::Thrift::Struct_Union 369 | SUCCESS = 0 370 | 371 | FIELDS = { 372 | SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success', :enum_class => Fb_status} 373 | } 374 | 375 | def struct_fields; FIELDS; end 376 | 377 | def validate 378 | unless @success.nil? || Fb_status::VALID_VALUES.include?(@success) 379 | raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field success!') 380 | end 381 | end 382 | 383 | ::Thrift::Struct.generate_accessors self 384 | end 385 | 386 | class GetStatusDetails_args 387 | include ::Thrift::Struct, ::Thrift::Struct_Union 388 | 389 | FIELDS = { 390 | 391 | } 392 | 393 | def struct_fields; FIELDS; end 394 | 395 | def validate 396 | end 397 | 398 | ::Thrift::Struct.generate_accessors self 399 | end 400 | 401 | class GetStatusDetails_result 402 | include ::Thrift::Struct, ::Thrift::Struct_Union 403 | SUCCESS = 0 404 | 405 | FIELDS = { 406 | SUCCESS => {:type => ::Thrift::Types::STRING, :name => 'success'} 407 | } 408 | 409 | def struct_fields; FIELDS; end 410 | 411 | def validate 412 | end 413 | 414 | ::Thrift::Struct.generate_accessors self 415 | end 416 | 417 | class GetCounters_args 418 | include ::Thrift::Struct, ::Thrift::Struct_Union 419 | 420 | FIELDS = { 421 | 422 | } 423 | 424 | def struct_fields; FIELDS; end 425 | 426 | def validate 427 | end 428 | 429 | ::Thrift::Struct.generate_accessors self 430 | end 431 | 432 | class GetCounters_result 433 | include ::Thrift::Struct, ::Thrift::Struct_Union 434 | SUCCESS = 0 435 | 436 | FIELDS = { 437 | SUCCESS => {:type => ::Thrift::Types::MAP, :name => 'success', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::I64}} 438 | } 439 | 440 | def struct_fields; FIELDS; end 441 | 442 | def validate 443 | end 444 | 445 | ::Thrift::Struct.generate_accessors self 446 | end 447 | 448 | class GetCounter_args 449 | include ::Thrift::Struct, ::Thrift::Struct_Union 450 | KEY = 1 451 | 452 | FIELDS = { 453 | KEY => {:type => ::Thrift::Types::STRING, :name => 'key'} 454 | } 455 | 456 | def struct_fields; FIELDS; end 457 | 458 | def validate 459 | end 460 | 461 | ::Thrift::Struct.generate_accessors self 462 | end 463 | 464 | class GetCounter_result 465 | include ::Thrift::Struct, ::Thrift::Struct_Union 466 | SUCCESS = 0 467 | 468 | FIELDS = { 469 | SUCCESS => {:type => ::Thrift::Types::I64, :name => 'success'} 470 | } 471 | 472 | def struct_fields; FIELDS; end 473 | 474 | def validate 475 | end 476 | 477 | ::Thrift::Struct.generate_accessors self 478 | end 479 | 480 | class SetOption_args 481 | include ::Thrift::Struct, ::Thrift::Struct_Union 482 | KEY = 1 483 | VALUE = 2 484 | 485 | FIELDS = { 486 | KEY => {:type => ::Thrift::Types::STRING, :name => 'key'}, 487 | VALUE => {:type => ::Thrift::Types::STRING, :name => 'value'} 488 | } 489 | 490 | def struct_fields; FIELDS; end 491 | 492 | def validate 493 | end 494 | 495 | ::Thrift::Struct.generate_accessors self 496 | end 497 | 498 | class SetOption_result 499 | include ::Thrift::Struct, ::Thrift::Struct_Union 500 | 501 | FIELDS = { 502 | 503 | } 504 | 505 | def struct_fields; FIELDS; end 506 | 507 | def validate 508 | end 509 | 510 | ::Thrift::Struct.generate_accessors self 511 | end 512 | 513 | class GetOption_args 514 | include ::Thrift::Struct, ::Thrift::Struct_Union 515 | KEY = 1 516 | 517 | FIELDS = { 518 | KEY => {:type => ::Thrift::Types::STRING, :name => 'key'} 519 | } 520 | 521 | def struct_fields; FIELDS; end 522 | 523 | def validate 524 | end 525 | 526 | ::Thrift::Struct.generate_accessors self 527 | end 528 | 529 | class GetOption_result 530 | include ::Thrift::Struct, ::Thrift::Struct_Union 531 | SUCCESS = 0 532 | 533 | FIELDS = { 534 | SUCCESS => {:type => ::Thrift::Types::STRING, :name => 'success'} 535 | } 536 | 537 | def struct_fields; FIELDS; end 538 | 539 | def validate 540 | end 541 | 542 | ::Thrift::Struct.generate_accessors self 543 | end 544 | 545 | class GetOptions_args 546 | include ::Thrift::Struct, ::Thrift::Struct_Union 547 | 548 | FIELDS = { 549 | 550 | } 551 | 552 | def struct_fields; FIELDS; end 553 | 554 | def validate 555 | end 556 | 557 | ::Thrift::Struct.generate_accessors self 558 | end 559 | 560 | class GetOptions_result 561 | include ::Thrift::Struct, ::Thrift::Struct_Union 562 | SUCCESS = 0 563 | 564 | FIELDS = { 565 | SUCCESS => {:type => ::Thrift::Types::MAP, :name => 'success', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}} 566 | } 567 | 568 | def struct_fields; FIELDS; end 569 | 570 | def validate 571 | end 572 | 573 | ::Thrift::Struct.generate_accessors self 574 | end 575 | 576 | class GetCpuProfile_args 577 | include ::Thrift::Struct, ::Thrift::Struct_Union 578 | PROFILEDURATIONINSEC = 1 579 | 580 | FIELDS = { 581 | PROFILEDURATIONINSEC => {:type => ::Thrift::Types::I32, :name => 'profileDurationInSec'} 582 | } 583 | 584 | def struct_fields; FIELDS; end 585 | 586 | def validate 587 | end 588 | 589 | ::Thrift::Struct.generate_accessors self 590 | end 591 | 592 | class GetCpuProfile_result 593 | include ::Thrift::Struct, ::Thrift::Struct_Union 594 | SUCCESS = 0 595 | 596 | FIELDS = { 597 | SUCCESS => {:type => ::Thrift::Types::STRING, :name => 'success'} 598 | } 599 | 600 | def struct_fields; FIELDS; end 601 | 602 | def validate 603 | end 604 | 605 | ::Thrift::Struct.generate_accessors self 606 | end 607 | 608 | class AliveSince_args 609 | include ::Thrift::Struct, ::Thrift::Struct_Union 610 | 611 | FIELDS = { 612 | 613 | } 614 | 615 | def struct_fields; FIELDS; end 616 | 617 | def validate 618 | end 619 | 620 | ::Thrift::Struct.generate_accessors self 621 | end 622 | 623 | class AliveSince_result 624 | include ::Thrift::Struct, ::Thrift::Struct_Union 625 | SUCCESS = 0 626 | 627 | FIELDS = { 628 | SUCCESS => {:type => ::Thrift::Types::I64, :name => 'success'} 629 | } 630 | 631 | def struct_fields; FIELDS; end 632 | 633 | def validate 634 | end 635 | 636 | ::Thrift::Struct.generate_accessors self 637 | end 638 | 639 | class Reinitialize_args 640 | include ::Thrift::Struct, ::Thrift::Struct_Union 641 | 642 | FIELDS = { 643 | 644 | } 645 | 646 | def struct_fields; FIELDS; end 647 | 648 | def validate 649 | end 650 | 651 | ::Thrift::Struct.generate_accessors self 652 | end 653 | 654 | class Reinitialize_result 655 | include ::Thrift::Struct, ::Thrift::Struct_Union 656 | 657 | FIELDS = { 658 | 659 | } 660 | 661 | def struct_fields; FIELDS; end 662 | 663 | def validate 664 | end 665 | 666 | ::Thrift::Struct.generate_accessors self 667 | end 668 | 669 | class Shutdown_args 670 | include ::Thrift::Struct, ::Thrift::Struct_Union 671 | 672 | FIELDS = { 673 | 674 | } 675 | 676 | def struct_fields; FIELDS; end 677 | 678 | def validate 679 | end 680 | 681 | ::Thrift::Struct.generate_accessors self 682 | end 683 | 684 | class Shutdown_result 685 | include ::Thrift::Struct, ::Thrift::Struct_Union 686 | 687 | FIELDS = { 688 | 689 | } 690 | 691 | def struct_fields; FIELDS; end 692 | 693 | def validate 694 | end 695 | 696 | ::Thrift::Struct.generate_accessors self 697 | end 698 | 699 | end 700 | 701 | --------------------------------------------------------------------------------