├── .gitignore ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── development ├── conf │ └── fluentd.conf └── run.sh ├── fluent-plugin-loki.gemspec └── lib └── fluent └── plugin └── out_loki.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /spec/examples.txt 9 | /test/tmp/ 10 | /test/version_tmp/ 11 | /tmp/ 12 | 13 | # Used by dotenv library to load environment variables. 14 | # .env 15 | 16 | ## Specific to RubyMotion: 17 | .dat* 18 | .repl_history 19 | build/ 20 | *.bridgesupport 21 | build-iPhoneOS/ 22 | build-iPhoneSimulator/ 23 | 24 | ## Specific to RubyMotion (use of CocoaPods): 25 | # 26 | # We recommend against adding the Pods directory to your .gitignore. However 27 | # you should judge for yourself, the pros and cons are mentioned at: 28 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 29 | # 30 | # vendor/Pods/ 31 | 32 | ## Documentation cache and generated files: 33 | /.yardoc/ 34 | /_yardoc/ 35 | /doc/ 36 | /rdoc/ 37 | 38 | ## Environment normalization: 39 | /.bundle/ 40 | /vendor/bundle 41 | /lib/bundler/man/ 42 | 43 | # for a library or gem, you might want to ignore these files since the code is 44 | # intended to run in multiple environments; otherwise, check them in: 45 | # Gemfile.lock 46 | # .ruby-version 47 | # .ruby-gemset 48 | 49 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 50 | .rvmrc 51 | 52 | Gemfile.lock 53 | test/* 54 | fluent/* 55 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in fluent-plugin-loki.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-plugin-loki 2 | 3 | [Fluentd](https://fluentd.org/) output plugin to [Grafana Loki](https://github.com/grafana/loki). 4 | 5 | - Can be used to ship docker logs to Loki (using [Fluentd docker logging driver](https://docs.docker.com/config/containers/logging/fluentd/)) 6 | - Enable easier transtion to Loki - an alternative to Loki's Promtail 7 | 8 | ## Installation 9 | 10 | ### RubyGems 11 | 12 | ``` 13 | $ gem install fluent-plugin-loki 14 | ``` 15 | 16 | ## Configuration 17 | sample configuration: 18 | ``` 19 | 20 | @type loki 21 | endpoint_url http://127.0.0.1:3100 22 | labels {"env":"prod","farm":"a"} # default: nil 23 | tenant abcd # default: nil 24 | rate_limit_msec 100 # default: 0 = no rate limiting 25 | raise_on_error false # default: true 26 | authentication basic # default: none 27 | username alice # default: '' 28 | password bobpop # default: '', secret: true 29 | buffered true # default: false. Switch non-buffered/buffered mode 30 | cacert_file /etc/ssl/endpoint1.cert # default: '' 31 | token tokent # default: '' 32 | custom_headers {"token":"arbitrary"} # default: nil 33 | 34 | ``` 35 | - **endpoint_url** - Loki's endpoint 36 | - **tenant** - Loki tenant id 37 | - **labels** - Labels for filtering in Grafana (currently they are static) 38 | 39 | (generated by running: ```fluent-plugin-config-format output loki -p lib/fluent/plugin```) 40 | 41 | ## Usage examples for common fluentd inputs: 42 | ### syslog 43 | *Setting up rsyslog* 44 | Open ```/etc/rsyslog.d/50-default.conf``` and append the following line: 45 | ```*.* @127.0.0.1:5140``` 46 | Then restart the rsyslogd service: 47 | ```sudo systemctl restart syslog``` 48 | *sample fluentd config:* 49 | ``` 50 | 51 | @type syslog 52 | port 5140 53 | bind 0.0.0.0 54 | tag system 55 | 56 | ``` 57 | *simulate data:* 58 | ```logger "came from syslog"``` 59 | ### file 60 | *sample fluentd config:* 61 | ``` 62 | ## File input 63 | ## read apache logs with tag=apache.access 64 | # 65 | # @type tail 66 | # format apache 67 | # path /var/log/httpd-access.log 68 | # tag apache.access 69 | # 70 | ``` 71 | ### HTTP 72 | *sample fluentd config:* 73 | ``` 74 | # HTTP input 75 | # http://localhost:8888/?json= 76 | # for ex: http://localhost:8888/baz?json={"src":"http"} 77 | 78 | @type http 79 | @id http_input 80 | 81 | port 8888 82 | 83 | ``` 84 | *simulate data:* 85 | ```bash 86 | curl -X POST -d 'json={"src":"http"}' http://localhost:8888 87 | ``` 88 | ### tcp 89 | *sample fluentd config:* 90 | ``` 91 | ## built-in TCP input 92 | ## $ echo | fluent-cat 93 | 94 | @type forward 95 | @id forward_input 96 | 97 | ``` 98 | *simulate data:* 99 | ```bash 100 | echo '{"src":"tcp"}' | fluent-cat tcp 101 | ``` 102 | 103 | ## Development (using Docker) 104 | - Set up Loki and Grafana (can be done by running their [docker-compose](https://github.com/grafana/loki/blob/master/production/docker-compose.yaml)) 105 | - Add loki data source to grafana 106 | - Run Fluentd container with volume mapping to the dummy configuration and to the plugin: 107 | ```bash 108 | docker run -d \ 109 | -p 9880:9880 \ 110 | -v $(pwd)/development/conf:/fluentd/etc \ 111 | -v $(pwd)/lib/fluent/plugin:/etc/fluent/plugin -e FLUENTD_CONF=fluentd.conf \ 112 | fluent/fluentd 113 | ``` 114 | - Execute to send log: ```curl -X POST -d 'json={"foo":"baz"}' http://localhost:9880``` 115 | 116 | 117 | ## Copyright 118 | 119 | * Copyright(c) 2018- Edan Shahmoon 120 | * License 121 | * Apache License, Version 2.0 122 | 123 | Heavily based on [fluent-plugin-out-http](https://github.com/fluent-plugins-nursery/fluent-plugin-out-http) 124 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler" 2 | Bundler::GemHelper.install_tasks 3 | 4 | require "rake/testtask" 5 | 6 | Rake::TestTask.new(:test) do |t| 7 | t.libs.push("lib", "test") 8 | t.test_files = FileList["test/**/test_*.rb"] 9 | t.verbose = true 10 | t.warning = true 11 | end 12 | 13 | task default: [:test] -------------------------------------------------------------------------------- /development/conf/fluentd.conf: -------------------------------------------------------------------------------- 1 | # /tmp/fluentd.conf 2 | 3 | @type http 4 | port 9880 5 | bind 0.0.0.0 6 | 7 | 8 | #@type stdout 9 | @type loki 10 | endpoint_url http://172.17.0.1:3100 11 | labels {"env":"prod","farm":"a"} 12 | tenant "fake" 13 | -------------------------------------------------------------------------------- /development/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker run -d \ 3 | -p 9880:9880 -v $(pwd)/development/conf:/fluentd/etc -v $(pwd)/lib/fluent/plugin:/etc/fluent/plugin -e FLUENTD_CONF=fluentd.conf \ 4 | fluent/fluentd -------------------------------------------------------------------------------- /fluent-plugin-loki.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | Gem::Specification.new do |gem| 4 | gem.name = "fluent-plugin-loki" 5 | gem.version = "0.3.0" 6 | gem.authors = ["Edan Shahmoon"] 7 | gem.email = ["edan100@gmail.com"] 8 | gem.summary = %q{A Fluentd output plugin to send logs to Grafana Loki} 9 | gem.description = gem.summary 10 | gem.homepage = "https://github.com/eeddaann/fluent-plugin-loki" 11 | gem.licenses = ["Apache-2.0"] 12 | 13 | gem.files = `git ls-files`.split($\) 14 | gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } 15 | gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) 16 | gem.require_paths = ["lib"] 17 | 18 | gem.required_ruby_version = '>= 2.1.0' 19 | 20 | gem.add_runtime_dependency "yajl-ruby", "~> 1.0" 21 | gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"] 22 | gem.add_development_dependency "bundler" 23 | gem.add_development_dependency "rake" 24 | gem.add_development_dependency "test-unit", ">= 3.1.0" 25 | end -------------------------------------------------------------------------------- /lib/fluent/plugin/out_loki.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'uri' 3 | require 'yajl' 4 | require 'json' 5 | require 'fluent/plugin/output' 6 | 7 | class Fluent::Plugin::LokiOutput < Fluent::Plugin::Output 8 | Fluent::Plugin.register_output('loki', self) 9 | 10 | helpers :compat_parameters 11 | 12 | DEFAULT_BUFFER_TYPE = "memory" 13 | 14 | def initialize 15 | super 16 | end 17 | 18 | # Loki's endpoint URL ex. http://localhost:3100 19 | config_param :endpoint_url, :string 20 | 21 | # Loki tenant id 22 | config_param :tenant, :string, :default => nil 23 | 24 | # labels 25 | config_param :labels, :hash, :default => nil 26 | 27 | # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE` 28 | config_param :ssl_no_verify, :bool, :default => false 29 | 30 | # Simple rate limiting: ignore any records within `rate_limit_msec` 31 | # since the last one. 32 | config_param :rate_limit_msec, :integer, :default => 0 33 | 34 | # Raise errors that were rescued during HTTP requests? 35 | config_param :raise_on_error, :bool, :default => true 36 | 37 | # ca file to use for https request 38 | config_param :cacert_file, :string, :default => '' 39 | 40 | # custom headers 41 | config_param :custom_headers, :hash, :default => nil 42 | 43 | # 'none' | 'basic' | 'jwt' | 'bearer' 44 | config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer], :default => :none 45 | config_param :username, :string, :default => '' 46 | config_param :password, :string, :default => '', :secret => true 47 | config_param :token, :string, :default => '' 48 | # Switch non-buffered/buffered plugin 49 | config_param :buffered, :bool, :default => false 50 | 51 | # Support multiple JSON encoders 52 | config_param :encoder, :enum, list: [:yajl, :json], default: :yajl 53 | 54 | config_section :buffer do 55 | config_set_default :@type, DEFAULT_BUFFER_TYPE 56 | config_set_default :chunk_keys, ['tag'] 57 | end 58 | 59 | def configure(conf) 60 | compat_parameters_convert(conf, :buffer) 61 | super 62 | 63 | @encoder = case @encoder 64 | when :yajl 65 | Yajl 66 | when :json 67 | JSON 68 | end 69 | 70 | @ssl_verify_mode = if @ssl_no_verify 71 | OpenSSL::SSL::VERIFY_NONE 72 | else 73 | OpenSSL::SSL::VERIFY_PEER 74 | end 75 | @ca_file = @cacert_file 76 | @last_request_time = nil 77 | raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered 78 | end 79 | 80 | def start 81 | super 82 | end 83 | 84 | def shutdown 85 | super 86 | end 87 | 88 | def format_url(tag, time, record) 89 | @endpoint_url 90 | end 91 | 92 | def set_body(req, tag, time, record) 93 | set_json_body(req, record) 94 | req 95 | end 96 | 97 | def set_header(req, tag, time, record) 98 | if @tenant 99 | req["X-Scope-OrgID"] = @tenant 100 | end 101 | if @custom_headers 102 | @custom_headers.each do |k,v| 103 | req[k] = v 104 | end 105 | req 106 | else 107 | req 108 | end 109 | end 110 | 111 | def set_json_body(req, data) 112 | req.body = @encoder.dump(data) 113 | req['Content-Type'] = 'application/json' 114 | end 115 | 116 | def create_request(tag, time, record) 117 | url = format_url(tag, time, record) 118 | uri = URI.parse(url+"/loki/api/v1/push") 119 | req = Net::HTTP::Post.new(uri.request_uri) 120 | set_body(req, tag, time, record) 121 | set_header(req, tag, time, record) 122 | return req, uri 123 | end 124 | 125 | def http_opts(uri) 126 | opts = { 127 | :use_ssl => uri.scheme == 'https' 128 | } 129 | opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl] 130 | opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file) 131 | opts 132 | end 133 | 134 | def proxies 135 | ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy'] 136 | end 137 | 138 | def send_request(req, uri) 139 | is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?) 140 | if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec) 141 | log.info('Dropped request due to rate limiting') 142 | return 143 | end 144 | 145 | res = nil 146 | 147 | begin 148 | if @authentication == :basic 149 | req.basic_auth(@username, @password) 150 | elsif @authentication == :bearer 151 | req['authorization'] = "bearer #{@token}" 152 | elsif @authentication == :jwt 153 | req['authorization'] = "jwt #{@token}" 154 | end 155 | @last_request_time = Time.now.to_f 156 | 157 | if proxy = proxies 158 | proxy_uri = URI.parse(proxy) 159 | 160 | res = Net::HTTP.start(uri.host, uri.port, 161 | proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password, 162 | **http_opts(uri)) {|http| http.request(req) } 163 | else 164 | res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) } 165 | end 166 | 167 | rescue => e # rescue all StandardErrors 168 | # server didn't respond 169 | log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'" 170 | raise e if @raise_on_error 171 | else 172 | unless res and res.is_a?(Net::HTTPSuccess) 173 | res_summary = if res 174 | "#{res.code} #{res.message} #{res.body}" 175 | else 176 | "res=nil" 177 | end 178 | log.warn "failed to #{req.method} #{uri} (#{res_summary})" 179 | end #end unless 180 | end # end begin 181 | end # end send_request 182 | def format_labels(labels) 183 | st = "{" 184 | labels.each do |key, val| 185 | st+= "#{key}=\"#{val}\"," 186 | end 187 | st[-1]="}" 188 | return st 189 | end 190 | def handle_record(tag, time, record) 191 | rec = {"streams"=>[{"labels"=>format_labels(@labels), "entries"=>[{"ts"=>Time.now.iso8601(3), "line"=>record.to_json}]}]} 192 | # I used time now instead of at 'time' because it cause 'Entry out of order' on loki's side 193 | req, uri = create_request(tag, time, rec) 194 | send_request(req, uri) 195 | end 196 | 197 | def prefer_buffered_processing 198 | @buffered 199 | end 200 | 201 | def format(tag, time, record) 202 | [time, record].to_msgpack 203 | end 204 | 205 | def formatted_to_msgpack_binary? 206 | true 207 | end 208 | 209 | def multi_workers_ready? 210 | true 211 | end 212 | 213 | def process(tag, es) 214 | es.each do |time, record| 215 | handle_record(tag, time, record) 216 | end 217 | end 218 | 219 | def write(chunk) 220 | tag = chunk.metadata.tag 221 | @endpoint_url = extract_placeholders(@endpoint_url, chunk.metadata) 222 | chunk.msgpack_each do |time, record| 223 | handle_record(tag, time, record) 224 | end 225 | end 226 | end --------------------------------------------------------------------------------