├── .github └── workflows │ └── push.yml ├── .gitignore ├── .rspec ├── .rubocop.yml ├── .ruby-version ├── Gemfile ├── LICENSE.md ├── README.md ├── Rakefile ├── bin └── console ├── config.ru ├── gemfiles ├── que_0.12.2.gemfile ├── que_0.12.3.gemfile ├── rails_4.1.gemfile ├── rails_4.2.gemfile ├── rails_5.2.gemfile ├── ruby_kafka.gemfile ├── shoryuken_4.0.gemfile ├── sidekiq_4.2.gemfile ├── sinatra_1.4.gemfile └── sinatra_2.0.gemfile ├── kiev.gemspec ├── lib ├── ext │ └── rack │ │ └── common_logger.rb ├── kiev.rb └── kiev │ ├── aws_sns.rb │ ├── aws_sns │ └── context_injector.rb │ ├── base.rb │ ├── base52.rb │ ├── config.rb │ ├── context_reader.rb │ ├── hanami.rb │ ├── her_ext │ └── client_request_id.rb │ ├── httparty.rb │ ├── json.rb │ ├── kafka.rb │ ├── kafka │ ├── context_extractor.rb │ ├── context_injector.rb │ └── message_context.rb │ ├── logger.rb │ ├── param_filter.rb │ ├── que │ └── job.rb │ ├── rack.rb │ ├── rack │ ├── request_id.rb │ ├── request_logger.rb │ ├── silence_action_dispatch_logger.rb │ └── store_request_details.rb │ ├── railtie.rb │ ├── request_body_filter.rb │ ├── request_body_filter │ ├── default.rb │ ├── form_data.rb │ ├── json.rb │ └── xml.rb │ ├── request_id.rb │ ├── request_logger.rb │ ├── request_store.rb │ ├── shoryuken.rb │ ├── shoryuken │ ├── context_reader.rb │ ├── middleware.rb │ └── middleware │ │ ├── message_tracer.rb │ │ ├── request_id.rb │ │ ├── request_logger.rb │ │ ├── request_store.rb │ │ ├── store_request_details.rb │ │ └── tree_path_suffix.rb │ ├── sidekiq.rb │ ├── sidekiq │ ├── client_request_id.rb │ ├── request_id.rb │ ├── request_logger.rb │ ├── request_store.rb │ └── store_request_details.rb │ ├── subrequest_helper.rb │ ├── test.rb │ ├── util.rb │ └── version.rb ├── spec ├── helpers │ ├── log_helper.rb │ └── que_helper.rb ├── lib │ ├── kiev │ │ ├── base52_spec.rb │ │ ├── config_spec.rb │ │ ├── json_spec.rb │ │ ├── kafka │ │ │ ├── context_extractor_spec.rb │ │ │ └── context_injector_spec.rb │ │ ├── logger_spec.rb │ │ ├── param_filter_spec.rb │ │ ├── que │ │ │ └── job_spec.rb │ │ ├── rack │ │ │ ├── request_id_spec.rb │ │ │ ├── request_logger_spec.rb │ │ │ └── store_request_details_spec.rb │ │ ├── request_body_filter_spec.rb │ │ ├── shoryuken_spec.rb │ │ └── subrequest_helper_spec.rb │ └── kiev_spec.rb └── spec_helper.rb └── test ├── data └── test.txt ├── helper.rb ├── her_ext_test.rb ├── rails_app ├── app │ ├── controllers │ │ ├── admin │ │ │ └── root_controller.rb │ │ └── root_controller.rb │ └── models │ │ └── user.rb ├── config │ ├── database.yml │ └── routes.rb ├── db │ ├── .gitignore │ └── schema.rb ├── log │ └── .gitignore └── public │ └── favicon.ico ├── rails_integration_test.rb ├── sidekiq_test.rb ├── sinatra_app └── test_app.rb └── sinatra_integration_test.rb /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake 6 | # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby 7 | 8 | name: Main CI 9 | 10 | on: 11 | push: 12 | branches: [ master ] 13 | pull_request: 14 | branches: [ master ] 15 | 16 | jobs: 17 | test: 18 | 19 | runs-on: ubuntu-latest 20 | strategy: 21 | matrix: 22 | ruby: ['2.5.1', '2.7.2', '2.6.5'] 23 | gemfile: 24 | - gemfiles/ruby_kafka.gemfile 25 | - gemfiles/que_0.12.2.gemfile 26 | - gemfiles/que_0.12.3.gemfile 27 | - gemfiles/rails_5.2.gemfile 28 | - gemfiles/shoryuken_4.0.gemfile 29 | - gemfiles/sidekiq_4.2.gemfile 30 | - gemfiles/sinatra_1.4.gemfile 31 | - gemfiles/sinatra_2.0.gemfile 32 | allow_failures: 33 | - false 34 | include: 35 | - os: ubuntu 36 | ruby-version: ruby-head 37 | gemfile: gemfiles/rails_5.2.gemfile 38 | allow_failures: true 39 | env: 40 | BUNDLE_GEMFILE: "${{ matrix.gemfile }}" 41 | ALLOW_FAILURES: "${{ matrix.allow_failures }}" 42 | REDIS_URL: "redis://localhost:6379/4" 43 | DATABASE_URL: ${{ (startsWith(matrix.gemfile,'gemfiles/que') && 'postgres://postgres:postgres@localhost:5432/que_test') || 'sqlite3:db/combustion_test.sqlite'}} 44 | continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' }} 45 | 46 | # Service containers to run with `container-job` 47 | services: 48 | redis: 49 | image: redis:latest 50 | ports: 51 | - 6379:6379 52 | # Label used to access the service container 53 | postgres: 54 | # Docker Hub image 55 | image: postgres:9.4 56 | # Provide the password for postgres 57 | env: 58 | POSTGRES_PASSWORD: postgres 59 | POSTGRES_DB: que_test 60 | # Set health checks to wait until postgres has started 61 | ports: 62 | - 5432:5432 63 | options: >- 64 | --health-cmd pg_isready 65 | --health-interval 10s 66 | --health-timeout 5s 67 | --health-retries 5 68 | steps: 69 | - uses: actions/checkout@v2 70 | - name: Set up Ruby 71 | # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, 72 | # change this to (see https://github.com/ruby/setup-ruby#versioning): 73 | # uses: ruby/setup-ruby@v1 74 | uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e 75 | with: 76 | ruby-version: ${{ matrix.ruby }} 77 | bundler-cache: true # runs 'bundle install' and caches installed gems automatically 78 | 79 | - name: Run tests 80 | run: bundle exec rake || $ALLOW_FAILURES 81 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /gemfiles/*.gemfile.lock 8 | /pkg/ 9 | /spec/reports/ 10 | /tmp/ 11 | .rubocop-http* 12 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require spec_helper 3 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: 2 | - https://raw.githubusercontent.com/blacklane/rubocop/master/rubocop.yml 3 | 4 | AllCops: 5 | TargetRubyVersion: 2.5 6 | Exclude: 7 | - test/rails_app/**/*.rb # auto-generated 8 | - spec/**/*.rb 9 | - test/**/*.rb 10 | - vendor/bundle/**/* 11 | Lint/SuppressedException: 12 | Exclude: 13 | - test/**/*.rb 14 | - spec/**/*.rb 15 | Lint/RescueException: 16 | Exclude: 17 | - lib/kiev/request_body_filter/json.rb 18 | - lib/kiev/sidekiq/request_logger.rb 19 | - lib/kiev/shoryuken/request_logger.rb 20 | - lib/kiev/rack/request_logger.rb 21 | - lib/kiev/json.rb 22 | Style/GlobalVars: 23 | Exclude: 24 | - test/helper.rb 25 | Style/GuardClause: 26 | Exclude: 27 | - lib/kiev/logger.rb 28 | Style/NestedParenthesizedCalls: 29 | Exclude: 30 | - spec/lib/kiev/json_spec.rb 31 | Style/BlockDelimiters: 32 | EnforcedStyle: line_count_based 33 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.5.1 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_5.2.gemfile")) 4 | 5 | gem "wwtd" 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2017 Blacklane 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kiev [![Build Status](https://github.com/blacklane/kiev/workflows/Main%20CI/badge.svg?branch=master)](https://github.com/blacklane/kiev/actions?query=workflow%3A%22Main+CI%22) [![Gem Version](https://badge.fury.io/rb/kiev.svg)](https://badge.fury.io/rb/kiev) 2 | 3 | Kiev is a comprehensive logging library aimed at covering a wide range of frameworks and tools from the Ruby ecosystem: 4 | 5 | - Rails 6 | - Sinatra 7 | - Rack and other Rack-based frameworks 8 | - Sidekiq 9 | - Que 10 | - Shoryuken 11 | - Her and other Faraday-based libraries 12 | - HTTParty 13 | 14 | The main goal of Kiev is consistent logging across distributed systems, like **tracking HTTP requests across various Ruby micro-services**. Kiev will generate and propagate request IDs and make it easy for you to identify service calls and branching requests, **including background jobs triggered by these requests**. 15 | 16 | Aside from web requests and background jobs, which are tracked out of the box, Kiev makes it easy to append additional information or introduce **custom events**. 17 | 18 | Kiev produces structured logs in the **JSON format**, which are ready to be ingested by ElasticSearch or other similar JSON-driven data stores. It eliminates the need for Logstash in a typical ELK stack. 19 | 20 | In **development mode**, Kiev can print human-readable logs - pretty much like the default Rails logger, but including all the additional information that you've provided via Kiev events. 21 | 22 | ## Install 23 | 24 | Add the gem to your `Gemfile`: 25 | 26 | ```ruby 27 | gem "kiev" 28 | ``` 29 | 30 | Don't forget to `bundle install`. 31 | 32 | ## Configure 33 | 34 | ### Rails 35 | 36 | Place your configuration under `config/initializers/kiev.rb`: 37 | 38 | ```ruby 39 | require "kiev" 40 | 41 | Kiev.configure do |config| 42 | config.app = :my_app 43 | config.development_mode = Rails.env.development? 44 | config.log_path = Rails.root.join("log", "structured.log") unless Rails.env.development? || $stdout.isatty 45 | end 46 | ``` 47 | 48 | The middleware stack is included automatically via a *Railtie*. 49 | 50 | ### Sinatra 51 | 52 | Somewhere in your code, ideally before the server configuration, add the following lines: 53 | 54 | ```ruby 55 | require "kiev" 56 | 57 | Kiev.configure do |config| 58 | config.app = :my_app 59 | config.log_path = File.join("log", "structured.log") 60 | end 61 | ``` 62 | 63 | Within your `Sinatra::Base` implementation, include the `Kiev::Rack` module, in order to register the middleware stack: 64 | 65 | ```ruby 66 | require "kiev" 67 | require "sinatra/base" 68 | 69 | class MyController < Sinatra::Base 70 | include Kiev::Rack 71 | 72 | use SomeOtherMiddleware 73 | 74 | get "/hello" do 75 | "world" 76 | end 77 | end 78 | ``` 79 | 80 | ### Rack 81 | 82 | Somewhere in your code, ideally before the server configuration, add the following lines: 83 | 84 | ```ruby 85 | require "kiev" 86 | 87 | Kiev.configure do |config| 88 | config.app = :my_app 89 | config.log_path = File.join("log", "structured.log") 90 | end 91 | ``` 92 | 93 | Within your `Rack::Builder` implementation, include the `Kiev::Rack` module, in order to register the middleware stack: 94 | 95 | ```ruby 96 | require "kiev" 97 | require "rack" 98 | 99 | app = Rack::Builder.new do 100 | include Kiev::Rack 101 | 102 | use SomeOtherMiddleware 103 | 104 | run labmda { |env| [ 200, {}, [ "hello world" ] ] } 105 | end 106 | 107 | run(app) 108 | ``` 109 | 110 | ### Hanami 111 | 112 | Place your configuration under `config/initializers/kiev.rb`: 113 | 114 | ```ruby 115 | require "kiev" 116 | 117 | Kiev.configure do |config| 118 | config.app = :my_app 119 | config.development_mode = Hanami.env?(:development) 120 | config.log_path = File.join("log", "structured.log") 121 | end 122 | ``` 123 | 124 | Within your `MyApp::Application` file, include the `Kiev::Hanami` module, in order to register the middleware stack. 125 | The `include` should be added before `configure` block. 126 | 127 | ```ruby 128 | module MyApp 129 | class Application < Hanami::Application 130 | include Kiev::Hanami 131 | 132 | configure do 133 | # ... 134 | end 135 | end 136 | end 137 | ``` 138 | 139 | ### Sidekiq 140 | 141 | Add the following lines to your initializer code: 142 | 143 | ```ruby 144 | Kiev::Sidekiq.enable 145 | ``` 146 | 147 | ### Shoryuken 148 | 149 | Add the following lines to your initializer code: 150 | 151 | ```ruby 152 | Kiev::Shoryuken.enable 153 | ``` 154 | 155 | The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this. 156 | 157 | ### AWS SNS 158 | 159 | To enhance messages published to SNS topics you can use the ContextInjector: 160 | 161 | ```ruby 162 | sns_message = { topic_arn: "...", message: "{...}" } 163 | Kiev::Kafka.inject_context(sns_message[:message_attributes]) 164 | 165 | ``` 166 | 167 | After this operation the message attributes will also include required context for the Kiev logger. 168 | 169 | ### Kafka 170 | 171 | To enhance messages published to Kafka topics you can use the ContextInjector: 172 | 173 | ```ruby 174 | Kiev::Kafka.inject_context(headers) 175 | ``` 176 | 177 | After this operation the headers variable will also include required context for the Kiev logger. 178 | 179 | If you have a consumed `Kafka::FetchedMessage` you can extract logger context with: 180 | 181 | ```ruby 182 | Kiev::Kafka.extract_context(message) 183 | ``` 184 | 185 | This will work regardless if headers are in HTTP format, e.g. `X-Tracking-Id` or plain field names: `tracking_id`. Plus the `message_key` field will contain the key of processed message. In case you want to log some more fields configure `persistent_log_fields` and `jobs_propagated_fields`. 186 | 187 | ### Que 188 | 189 | Add the following lines to your initializer code: 190 | 191 | ```ruby 192 | require "kiev/que/job" 193 | 194 | class MyJob < Kiev::Que::Job 195 | ... 196 | end 197 | ``` 198 | 199 | ### Her 200 | 201 | Add the following lines to your initializer code: 202 | 203 | ```ruby 204 | Her::API.setup(url: "https://api.example.com") do |c| 205 | c.use Kiev::HerExt::ClientRequestId 206 | # other middleware 207 | end 208 | ``` 209 | 210 | ## Loading only the required parts 211 | 212 | You can load only parts of the gem, if you don't want to use all features: 213 | 214 | ```ruby 215 | require "kiev/her_ext/client_request_id" 216 | ``` 217 | 218 | ## Logging 219 | 220 | ### Requests 221 | 222 | For web requests the Kiev middleware will log the following information by default: 223 | 224 | ```json 225 | { 226 | "application":"my_app", 227 | "event":"request_finished", 228 | "level":"INFO", 229 | "timestamp":"2017-01-27T16:11:44.123Z", 230 | "host":"localhost", 231 | "verb":"GET", 232 | "path":"/", 233 | "params":"{\"hello\":\"world\",\"password\":\"[FILTERED]\"}", 234 | "ip":"127.0.0.1", 235 | "request_id":"UUID", 236 | "request_depth":0, 237 | "route":"RootController#index", 238 | "user_agent":"curl/7.50.1", 239 | "status":200, 240 | "request_duration":62.3773, 241 | "body":"See #log_response_body_condition", 242 | "error_message": "...", 243 | "error_class": "...", 244 | "error_backtrace": "...", 245 | "tree_path": "ACE", 246 | "tree_leaf": true 247 | } 248 | ``` 249 | 250 | * `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option. 251 | 252 | * `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). (currently deprecated in favor of a new name: `tracking_id`) 253 | 254 | * `tracking_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). If not provided the value is seeded from deprecated `request_id`. 255 | 256 | * `request_depth` represents the position of the current request within a chain of requests. It starts with 0. 257 | 258 | * `route` attribute will be set to either the Rails route (`RootController#index`) or Sinatra route (`/`) or the path, depending on the context. 259 | 260 | * `request_duration` is measured in miliseconds. 261 | 262 | * `body` attribute coresponds to the response body and will be logged depending on the `#log_response_body_condition` option. 263 | 264 | * `tree_path` attribute can be used to follow the branching of requests within a chain of requests. It's a lexicographically sortable string. 265 | 266 | * `tree_leaf` points out that this request is a leaf in the request chain tree structure. 267 | 268 | ### Background jobs 269 | 270 | For background jobs, Kiev will log the following information by default: 271 | 272 | ```json 273 | { 274 | "application":"my_app", 275 | "event":"job_finished", 276 | "level":"INFO", 277 | "timestamp":"2017-01-27T16:11:44.123Z", 278 | "job_name":"name", 279 | "params": "...", 280 | "jid":123, 281 | "request_id":"UUID", 282 | "request_depth":0, 283 | "request_duration":0.000623773, 284 | "error_message": "...", 285 | "error_class": "...", 286 | "error_backtrace": "...", 287 | "tree_path": "BDF", 288 | "tree_leaf": true 289 | } 290 | ``` 291 | 292 | ### Appending data to the request log entry 293 | 294 | You can also append **arbitrary data** to the request log by calling: 295 | 296 | ```ruby 297 | # Append structured data (will be merged) 298 | Kiev.payload(first_name: "john", last_name: "smith") 299 | 300 | # Same thing 301 | Kiev[:first_name] = "john" 302 | Kiev[:last_name] = "smith" 303 | ``` 304 | 305 | ### Other events 306 | 307 | Kiev allows you to log custom events as well. 308 | 309 | The recommended way to do this is by using the `#event` method: 310 | 311 | ```ruby 312 | # Log event without any data 313 | Kiev.event(:my_event) 314 | 315 | # Log structured data (will be merged) 316 | Kiev.event(:my_event, { some_array: [1, 2, 3] }) 317 | 318 | # Log other data types (will be available under the `message` key) 319 | Kiev.event(:my_event, "hello world") 320 | 321 | # Log with given severity [debug, info, warn, error, fatal] 322 | Kiev.info(:my_event) 323 | Kiev.info(:my_event, { some_array: [1, 2, 3] }) 324 | Kiev.info(:my_event, "hello world") 325 | ``` 326 | 327 | However, `Kiev.logger` implements the Ruby `Logger` class, so all the other methods are available as well: 328 | 329 | ```ruby 330 | Kiev.logger.info("hello world") 331 | Kiev.logger.debug({ first_name: "john", last_name: "smith" }) 332 | ``` 333 | 334 | Note that, even when logging custom events, Kiev **will try to append request information** to the entries: the HTTP `verb` and `path` for web request or `job_name` and `jid` for background jobs. The payload, however, will be logged only for the `request_finished` or `job_finished` events. If you want to add a payload to a custom event, use the second argument of the `event` method. 335 | 336 | ## Advanced configuration 337 | 338 | ### development_mode 339 | 340 | Kiev offers human-readable logging for development purposes. You can enable it via the `development_mode` option: 341 | 342 | ```ruby 343 | Kiev.configure do |config| 344 | config.development_mode = Rails.env.development? 345 | end 346 | ``` 347 | 348 | ### filtered_params 349 | 350 | By default, Kiev filters out the values for the following parameters: 351 | 352 | - client_secret 353 | - token 354 | - password, 355 | - password_confirmation 356 | - old_password 357 | - credit_card_number 358 | - credit_card_cvv 359 | 360 | You can override this behaviour via the `filtered_params` option: 361 | 362 | ```ruby 363 | Kiev.configure do |config| 364 | config.filtered_params = %w(email first_name last_name) 365 | end 366 | ``` 367 | 368 | ### ignored_params 369 | 370 | By default, Kiev ignores the following parameters: 371 | 372 | - controller 373 | - action 374 | - format 375 | - authenticity_token 376 | - utf8 377 | 378 | You can override this behaviour via the `ignored_params` option: 379 | 380 | ```ruby 381 | Kiev.configure do |config| 382 | config.ignored_params = %w(some_field some_other_field) 383 | end 384 | ``` 385 | 386 | ### log_request_condition 387 | 388 | By default, Kiev doesn't log requests to `/ping`, `/health`, `/live` or `/ready` or requests to assets. 389 | 390 | You can override this behaviour via the `log_request_condition` option, which should be a `proc` returning a `boolean`: 391 | 392 | ```ruby 393 | Kiev.configure do |config| 394 | config.log_request_condition = proc do |request, response| 395 | !%r{(^(/ping|/health))|(\.(js|css|png|jpg|gif)$)}.match(request.path) 396 | end 397 | end 398 | ``` 399 | 400 | ### log_request_error_condition 401 | 402 | Kiev logs Ruby exceptions. By default, it won't log the exceptions produced by 404s. 403 | 404 | You can override this behaviour via the `log_request_error_condition` option, which should be a `proc` returning a `boolean`: 405 | 406 | ```ruby 407 | Kiev.configure do |config| 408 | config.log_request_error_condition = proc do |request, response| 409 | response.status != 404 410 | end 411 | end 412 | ``` 413 | 414 | ### log_response_body_condition 415 | 416 | Kiev can log the response body. By default, it will only log the response body when the status code is in the 4xx range and the content type is JSON or XML. 417 | 418 | You can override this behaviour via the `log_response_body_condition` option, which should be a `proc` returning a `boolean`: 419 | 420 | ```ruby 421 | Kiev.configure do |config| 422 | config.log_response_body_condition = proc do |request, response| 423 | response.status >= 400 && response.status < 500 && response.content_type =~ /(json|xml)/ 424 | end 425 | end 426 | ``` 427 | 428 | ### persistent_log_fields 429 | 430 | If you need to log some data for every event in the session (e.g. the user ID), you can do this via the `persistent_log_fields` option. 431 | 432 | ```ruby 433 | Kiev.configure do |config| 434 | config.persistent_log_fields = [:user_id] 435 | end 436 | 437 | # Somewhere in application 438 | before do 439 | Kiev[:user_id] = current_user.id 440 | end 441 | 442 | get "/" do 443 | "hello world" 444 | end 445 | ``` 446 | 447 | ### log_level 448 | You can specify log level. 449 | 450 | ```ruby 451 | Kiev.configure do |config| 452 | # One of: 0, 1, 2, 3, 4 (DEBUG, INFO, WARN, ERROR, FATAL) 453 | config.log_level = 0 454 | end 455 | ``` 456 | 457 | ### disable_filter_for_log_levels 458 | You can specify for which log levels personal identifying information filter will NOT be applied. 459 | 460 | ```ruby 461 | Kiev.configure do |config| 462 | # [DEBUG, INFO, WARN, ERROR, FATAL] 463 | config.disable_filter_for_log_levels = [0, 1, 2, 3, 4] 464 | end 465 | ``` 466 | 467 | **By default enabled for all suppported log levels.** 468 | 469 | ## nginx 470 | 471 | If you want to log 499 and 50x errors in nginx, which will not be captured by Ruby application, consider adding this to your nginx configuration: 472 | 473 | ``` 474 | log_format kiev '{"application":"app_name", "event":"request_finished",' 475 | '"timestamp":"$time_iso8601", "request_id":"$http_x_request_id",' 476 | '"user_agent":"$http_user_agent", "status":$status,' 477 | '"request_duration_seconds":$request_time, "host":"$host",' 478 | '"verb":"$request_method", "path":"$request_uri", "tree_path": "$http_x_tree_path"}'; 479 | 480 | log_format simple_log '$remote_addr - $remote_user [$time_local] ' 481 | '"$request" $status $bytes_sent ' 482 | '"$http_referer" "$http_user_agent"'; 483 | 484 | map $status $not_loggable { 485 | ~(499) 0; 486 | default 1; 487 | } 488 | 489 | map $status $loggable { 490 | ~(499) 1; 491 | default 0; 492 | } 493 | 494 | server { 495 | access_log /var/log/nginx/access.kiev.log kiev if=$loggable; 496 | access_log /var/log/nginx/access.log simple_log if=$not_loggable; 497 | 498 | location = /50x.html { 499 | access_log /var/log/nginx/access.kiev.log kiev; 500 | } 501 | } 502 | ``` 503 | 504 | If you'd like to measure nginx queue latency, add the following to your nginx configuration: 505 | 506 | ``` 507 | server { 508 | ... 509 | proxy_set_header X-Request-Start "${msec}"; 510 | ... 511 | } 512 | ``` 513 | 514 | Other libs/technologies using `X-Request-Start` are [rack-timeout](https://github.com/heroku/rack-timeout) and [NewRelic](https://docs.newrelic.com/docs/apm/applications-menu/features/request-queue-server-configuration-examples). There's no [support for ELB](https://forums.aws.amazon.com/message.jspa?messageID=396283) :( 515 | 516 | ## Logstash, Logrotate, Filebeat 517 | 518 | Kiev does not provide facilities to log directly to ElasticSearch. This is done for simplicity. Instead we recommend using [Filebeat](https://www.elastic.co/products/beats/filebeat) to deliver logs to ElasticSearch. 519 | 520 | When storing logs on disk, we recommend using Logrotate in truncate mode. 521 | 522 | You can use [jq](https://stedolan.github.io/jq/) to traverse JSON log files, when you're not running Kiev in *development mode*. 523 | 524 | ## Suffixing `tree_path` 525 | 526 | Kiev is built upon the assumption that one request is handled once. This isn't always true. 527 | 528 | A practical example: multiple Amazon SQS queues subscribed to one Amazon SNS topic. You send one message to SNS and queues receive identical copies that are impossible to distinguish in the trace without any help from the outside. 529 | 530 | You can solve this by adding a fixed unique suffix inside each queue processor. Preferably a single character with an even number in the alphabet (B, D, F and so on), to maintain the notion of "asynchronous processing" used throughout Kiev. 531 | 532 | For a combination of SNS and [Shoryuken](https://github.com/phstc/shoryuken) (SQS consumer). Here's how you can use it: 533 | 534 | * Enable "Raw Message Delivery" in your SQS-to-SNS subscriptions 535 | * On sender, write `Kiev::SubrequestHelper.payload` into the message attributes 536 | * On each receiver, use `Kiev::Shoryuken::suffix_tree_path` with a unique tag, like this: 537 | 538 | ```ruby 539 | # Suffix a single worker class: 540 | class MyWorker 541 | include Shoryuken::Worker 542 | Kiev::Shoryuken.suffix_tree_path(self, "B") 543 | # ... 544 | end 545 | 546 | # Or use a suffix process-wide: 547 | Shoryuken.configure_server do |config| 548 | Kiev::Shoryuken.suffix_tree_path(config, "B") 549 | end 550 | ``` 551 | 552 | Here's an example of the possble `tree_path` sequence you could get by configuring two consumers with suffixes `1` and `2` (note ordering by `tree_path`): 553 | 554 | | `tree_path` | Meaning | 555 | |-------------|-----------------------------------------------------------------| 556 | | `A` | An entry point into the system, a synchronous request | 557 | | `AB` | Background job caused by `A` executed | 558 | | `ABA` | Synchoronous request made from `AB` | 559 | | `ABD` | _(Not logged by Kiev itself)_ `AB` sends out an SNS message | 560 | | `ABD1` | Message `ABD` handled by susbcriber `1` | 561 | | `ABD1A` | Synchronous request sent by `1` when handling the message `ABD` | 562 | | `ABD1C` | Synchronous request sent by `1` when handling the message `ABD` | 563 | | `ABD2` | Message `ABD` handled by susbcriber `2` | 564 | | `ABD2A` | Synchronous request sent by `2` when handling the message `ABD` | 565 | | `ABF` | Another backgound job from `AB` executed | 566 | | `AD` | Background job caused by `A` executed | 567 | | `AE` | Synchronous request made from `A` | 568 | 569 | Without suffixing you won't see at a glance who made the request `ABDC` and you will have two entries for both `ABD` and `ABDA`. As different subscribers may log different fields, you might be able to tell apart `ABD`s. But both `ABDA`s could happen on the same node and be logged with the same lines of code. 570 | 571 | ## Alternatives 572 | 573 | ### Logging 574 | 575 | - [semantic_logger](http://rocketjob.github.io/semantic_logger/) 576 | - [lograge](https://github.com/roidrage/lograge) 577 | - [logging](https://github.com/TwP/logging) 578 | 579 | ### Request-Id 580 | 581 | - [Pliny::Middleware::RequestID](https://github.com/interagent/pliny/blob/master/lib/pliny/middleware/request_id.rb) 582 | - [ActionDispatch::RequestId](http://api.rubyonrails.org/classes/ActionDispatch/RequestId.html) 583 | - [request_id](https://github.com/remind101/request_id) 584 | 585 | ## Development 586 | 587 | Pull the code: 588 | 589 | ``` 590 | git clone git@github.com:blacklane/kiev.git 591 | ``` 592 | 593 | Run tests: 594 | 595 | ```sh 596 | bundle exec rake 597 | ``` 598 | 599 | Run tests for different rubies, frameworks and framework versions: 600 | 601 | ```sh 602 | # Create a Postgres test database for Que 603 | createdb que_test 604 | 605 | # Run the tests (replace myuser with your username) 606 | DATABASE_URL=postgres://myuser:@localhost/que_test bundle exec wwtd 607 | ``` 608 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "rspec/core/rake_task" 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | require "rake/testtask" 7 | Rake::TestTask.new do |t| 8 | t.libs << "test" 9 | t.pattern = "test/**/*_test.rb" 10 | end 11 | 12 | require "rubocop/rake_task" 13 | desc "Run RuboCop" 14 | RuboCop::RakeTask.new(:rubocop) do |task| 15 | task.options = ["--display-cop-names"] 16 | end 17 | 18 | task default: %w(spec test rubocop) 19 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "bundler/setup" 5 | require_relative "../lib/kiev" 6 | 7 | require "irb" 8 | IRB.start 9 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "rubygems" 4 | require "bundler" 5 | 6 | Bundler.require :default, :development 7 | 8 | Combustion.initialize!(:all) 9 | run Combustion::Application 10 | -------------------------------------------------------------------------------- /gemfiles/que_0.12.2.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "oj" 6 | 7 | # Build brakes with 0.12.3, TODO: handle this 8 | gem "que", "0.12.2" 9 | gem "pg" 10 | gem "sequel" 11 | 12 | gem "rack-test", require: false 13 | gem "rspec", require: false 14 | gem "minitest-reporters", require: false 15 | 16 | gemspec path: "../" 17 | -------------------------------------------------------------------------------- /gemfiles/que_0.12.3.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | # need it because of bug in https://github.com/chanks/que/issues/191 6 | gem "multi_json" 7 | gem "oj" 8 | 9 | gem "que", "0.12.3" 10 | gem "pg" 11 | gem "sequel" 12 | 13 | gem "rack-test", require: false 14 | gem "rspec", require: false 15 | gem "minitest-reporters", require: false 16 | 17 | gemspec path: "../" 18 | -------------------------------------------------------------------------------- /gemfiles/rails_4.1.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "oj" 6 | 7 | gem "rails", "4.1.16" 8 | gem "sqlite3", "1.3.13" 9 | 10 | gem "combustion" 11 | gem "rspec", require: false 12 | gem "rspec-rails", require: false 13 | gem "minitest-reporters", require: false 14 | 15 | gemspec path: "../" 16 | -------------------------------------------------------------------------------- /gemfiles/rails_4.2.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "oj" 6 | 7 | gem "rails", "4.2.7" 8 | gem "sqlite3", "1.3.13" 9 | 10 | gem "combustion" 11 | gem "rspec", require: false 12 | gem "rspec-rails", require: false 13 | gem "minitest-reporters", require: false 14 | 15 | gemspec path: "../" 16 | -------------------------------------------------------------------------------- /gemfiles/rails_5.2.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "oj" 6 | 7 | gem "rails", "~> 5.2.4" 8 | gem "sqlite3", "1.3.13" 9 | 10 | gem "combustion" 11 | gem "rspec", require: false 12 | gem "rspec-rails", require: false 13 | gem "minitest-reporters", require: false 14 | 15 | gemspec path: "../" 16 | -------------------------------------------------------------------------------- /gemfiles/ruby_kafka.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "ruby-kafka", "~> 0.7.10" 6 | 7 | gem "rack-test", require: false 8 | gem "rspec", require: false 9 | gem "minitest-reporters", require: false 10 | 11 | gemspec path: "../" 12 | -------------------------------------------------------------------------------- /gemfiles/shoryuken_4.0.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "aws-sdk", "~> 2.0" 6 | gem "shoryuken", "~> 4.0" 7 | 8 | gem "rack-test", require: false 9 | gem "minitest-reporters", require: false 10 | 11 | gemspec path: "../" 12 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_4.2.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "oj", "~> 2" 6 | 7 | gem "sidekiq", "~> 4.2.0" 8 | 9 | gem "rack-test", require: false 10 | gem "rspec", require: false 11 | gem "minitest-reporters", require: false 12 | 13 | gem "her" 14 | # We need to do it, since her gem doesn't lock upper boundry 15 | # https://github.com/remi/her/blob/master/her.gemspec#L26 16 | gem "faraday", "~> 1.9.3" 17 | 18 | gemspec path: "../" 19 | -------------------------------------------------------------------------------- /gemfiles/sinatra_1.4.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "oj" 6 | 7 | gem "xml-simple" 8 | 9 | gem "sinatra", "1.4.7" 10 | gem "sinatra-contrib" 11 | gem "rack-parser", require: "rack/parser" 12 | 13 | gem "rack-test", require: false 14 | gem "rspec", require: false 15 | gem "minitest-reporters", require: false 16 | 17 | gemspec path: "../" 18 | -------------------------------------------------------------------------------- /gemfiles/sinatra_2.0.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "oj" 6 | 7 | gem "xml-simple" 8 | 9 | gem "sinatra", "2.0.0" 10 | gem "sinatra-contrib" 11 | gem "rack-parser", require: "rack/parser" 12 | 13 | gem "rack-test", require: false 14 | gem "rspec", require: false 15 | gem "minitest-reporters", require: false 16 | 17 | gemspec path: "../" 18 | -------------------------------------------------------------------------------- /kiev.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.join(File.dirname(__FILE__), "lib/kiev/version") 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "kiev" 7 | spec.version = Kiev::VERSION 8 | spec.authors = ["Blacklane"] 9 | spec.licenses = ["MIT"] 10 | 11 | spec.summary = "Distributed logging to JSON integrated with various Ruby frameworks and tools" 12 | spec.description = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing "\ 13 | "human-readable output in development mode. It integrates nicely with Rails, Sinatra and other"\ 14 | " Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients." 15 | spec.homepage = "https://github.com/blacklane/kiev" 16 | 17 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 18 | spec.bindir = "exe" 19 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 20 | spec.require_paths = ["lib"] 21 | 22 | spec.required_ruby_version = ">= 2.5" 23 | spec.add_dependency "oga", "~> 2.2" 24 | spec.add_dependency "rack", ">= 1", "< 3" 25 | spec.add_dependency "request_store", ">= 1.0", "< 1.4" 26 | spec.add_dependency "ruby_dig", "~> 0.0.2" # to support ruby 2.2 27 | spec.add_development_dependency "rake", "~> 0" 28 | spec.add_development_dependency "rspec", "~> 3.10" 29 | spec.add_development_dependency "rubocop", "~> 0.54" 30 | end 31 | -------------------------------------------------------------------------------- /lib/ext/rack/common_logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Disable useless rack logger completely! 4 | # for some reason disable :logging doesn't work for sinatra 5 | module Rack 6 | class CommonLogger 7 | def call(env) 8 | # do nothing 9 | @app.call(env) 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/kiev.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "kiev/base" 4 | require_relative "kiev/aws_sns" if defined?(AWS::SNS) 5 | require_relative "kiev/kafka" if defined?(Kafka) 6 | require_relative "kiev/rack" if defined?(Rack) 7 | require_relative "kiev/railtie" if defined?(Rails) 8 | require_relative "kiev/sidekiq" if defined?(Sidekiq) 9 | require_relative "kiev/shoryuken" if defined?(Shoryuken) 10 | require_relative "kiev/her_ext/client_request_id" if defined?(Faraday) 11 | require_relative "kiev/httparty" if defined?(HTTParty) 12 | require_relative "kiev/que/job" if defined?(Que::Job) 13 | -------------------------------------------------------------------------------- /lib/kiev/aws_sns.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base" 4 | 5 | module Kiev 6 | module AwsSns 7 | require_relative "kafka/context_injector" 8 | 9 | class << self 10 | # @param [Hash] headers 11 | def inject_context(headers = {}) 12 | Kiev::AwsSns::ContextInjector.new.call(headers) 13 | end 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/kiev/aws_sns/context_injector.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "kiev/config" 4 | require "kiev/subrequest_helper" 5 | 6 | module Kiev 7 | module AwsSns 8 | class ContextInjector 9 | # @param [Hash] message_attributes Injects context headers 10 | # @return [Hash] 11 | def call(message_attributes = {}) 12 | Kiev::SubrequestHelper.payload.each do |key, value| 13 | message_attributes[key] = { 14 | data_type: "String", 15 | string_value: value.to_s 16 | } 17 | end 18 | message_attributes 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/kiev/base.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "request_store" 4 | require "ruby_dig" 5 | require_relative "request_store" 6 | require_relative "request_logger" 7 | require_relative "logger" 8 | require_relative "param_filter" 9 | require_relative "request_body_filter" 10 | require_relative "json" 11 | require_relative "version" 12 | require_relative "config" 13 | require_relative "util" 14 | require_relative "subrequest_helper" 15 | require_relative "hanami" 16 | require "forwardable" 17 | require "logger" 18 | 19 | module Kiev 20 | class << self 21 | extend Forwardable 22 | 23 | def_delegators :config, 24 | :logger, 25 | :filtered_params, 26 | :ignored_params, 27 | :log_level, 28 | :disable_filter_for_log_levels 29 | 30 | EMPTY_OBJ = {}.freeze 31 | 32 | def configure 33 | yield(Config.instance) 34 | end 35 | 36 | def config 37 | Config.instance 38 | end 39 | 40 | def event(log_name, data = EMPTY_OBJ, severity = log_level) 41 | logger.log(severity, logged_data(data), log_name) 42 | end 43 | 44 | Config.instance.supported_log_levels.each_pair do |key, value| 45 | define_method(key) do |log_name, data = EMPTY_OBJ| 46 | event(log_name, data, value) 47 | end 48 | end 49 | 50 | def []=(name, value) 51 | RequestStore.store[:payload] ||= {} 52 | RequestStore.store[:payload][name] = value 53 | end 54 | 55 | def payload(data) 56 | raise ArgumentError, "Hash expected" unless data.is_a?(Hash) 57 | 58 | RequestStore.store[:payload] ||= {} 59 | RequestStore.store[:payload].merge!(data) 60 | end 61 | 62 | def error=(value) 63 | RequestStore.store[:error] = value 64 | end 65 | 66 | def request_id 67 | RequestStore.store[:tracking_id] 68 | end 69 | 70 | alias_method :tracking_id, :request_id 71 | 72 | private 73 | 74 | def logged_data(data) 75 | return data unless config.filter_enabled? 76 | 77 | ParamFilter.filter(data, filtered_params, ignored_params) 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/kiev/base52.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: false 2 | 3 | module Kiev 4 | module Base52 5 | KEYS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".freeze 6 | BASE = KEYS.length 7 | 8 | def self.encode(num) 9 | return KEYS[0] if num == 0 10 | return nil if num < 0 11 | 12 | str = "" 13 | while num > 0 14 | str.prepend(KEYS[num % BASE]) 15 | num /= BASE 16 | end 17 | str 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/kiev/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "singleton" 4 | 5 | module Kiev 6 | class Config 7 | include Singleton 8 | 9 | DEFAULT_LOG_REQUEST_REGEXP = %r{(^(/ping|/health|/live|/ready))|(\.(js|css|png|jpg|gif|ico|svg)$)} 10 | private_constant :DEFAULT_LOG_REQUEST_REGEXP 11 | 12 | DEFAULT_LOG_REQUEST_CONDITION = proc do |request, _response| 13 | !DEFAULT_LOG_REQUEST_REGEXP.match(request.path) 14 | end 15 | 16 | DEFAULT_LOG_REQUEST_ERROR_CONDITION = proc do |_request, response| 17 | response.status != 404 18 | end 19 | 20 | DEFAULT_LOG_RESPONSE_BODY_REGEXP = /(json|xml)/ 21 | private_constant :DEFAULT_LOG_RESPONSE_BODY_REGEXP 22 | 23 | DEFAULT_LOG_RESPONSE_BODY_CONDITION = proc do |_request, response| 24 | !!(response.status >= 400 && response.status < 500 && response.content_type =~ DEFAULT_LOG_RESPONSE_BODY_REGEXP) 25 | end 26 | 27 | DEFAULT_LOG_REQUEST_BODY_CONDITION = proc do |request, _response| 28 | !!(request.content_type =~ /(application|text)\/xml/) 29 | end 30 | 31 | DEFAULT_IGNORED_RACK_EXCEPTIONS = 32 | %w( 33 | ActiveRecord::RecordNotFound 34 | Mongoid::Errors::DocumentNotFound 35 | Sequel::RecordNotFound 36 | ).freeze 37 | 38 | FILTERED_PARAMS = 39 | %w( 40 | client_secret 41 | token 42 | password 43 | password_confirmation 44 | old_password 45 | credit_card_number 46 | credit_card_cvv 47 | credit_card_holder 48 | credit_card_expiry_month 49 | credit_card_expiry_year 50 | CardNumber 51 | CardCVV 52 | CardExpires 53 | ).freeze 54 | 55 | IGNORED_PARAMS = 56 | (%w( 57 | controller 58 | action 59 | format 60 | authenticity_token 61 | utf8 62 | tempfile 63 | ) << :tempfile).freeze 64 | 65 | DEFAULT_HTTP_PROPAGATED_FIELDS = { 66 | tracking_id: "X-Tracking-Id", 67 | request_id: "X-Request-Id", 68 | request_depth: "X-Request-Depth", 69 | tree_path: "X-Tree-Path" 70 | }.freeze 71 | 72 | DEFAULT_PRE_RACK_HOOK = proc do |env| 73 | Config.instance.http_propagated_fields.each do |key, http_key| 74 | Kiev[key] = Util.sanitize(env[Util.to_http(http_key)]) 75 | end 76 | end 77 | 78 | SUPPORTED_LOG_LEVELS = { 79 | debug: ::Logger::DEBUG, 80 | info: ::Logger::INFO, 81 | warn: ::Logger::WARN, 82 | error: ::Logger::ERROR, 83 | fatal: ::Logger::FATAL 84 | }.freeze 85 | 86 | private_constant :SUPPORTED_LOG_LEVELS 87 | 88 | attr_accessor :app, 89 | :log_request_condition, 90 | :log_request_error_condition, 91 | :log_response_body_condition, 92 | :log_request_body_condition, 93 | :filtered_params, 94 | :ignored_params, 95 | :ignored_rack_exceptions, 96 | :disable_default_logger, 97 | :persistent_log_fields, 98 | :pre_rack_hook 99 | 100 | attr_reader :development_mode, 101 | :logger, 102 | :log_level, 103 | :http_propagated_fields, 104 | :jobs_propagated_fields, 105 | :all_http_propagated_fields, # for internal use 106 | :all_jobs_propagated_fields, # for internal use 107 | :disable_filter_for_log_levels 108 | 109 | def initialize 110 | @log_request_condition = DEFAULT_LOG_REQUEST_CONDITION 111 | @log_request_error_condition = DEFAULT_LOG_REQUEST_ERROR_CONDITION 112 | @log_response_body_condition = DEFAULT_LOG_RESPONSE_BODY_CONDITION 113 | @log_request_body_condition = DEFAULT_LOG_REQUEST_BODY_CONDITION 114 | @filtered_params = FILTERED_PARAMS 115 | @ignored_params = IGNORED_PARAMS 116 | @disable_default_logger = true 117 | @development_mode = false 118 | @ignored_rack_exceptions = DEFAULT_IGNORED_RACK_EXCEPTIONS.dup 119 | @logger = Kiev::Logger.new(STDOUT) 120 | @log_level = default_log_level 121 | @persistent_log_fields = [] 122 | @pre_rack_hook = DEFAULT_PRE_RACK_HOOK 123 | @disable_filter_for_log_levels = [] 124 | self.propagated_fields = {} 125 | update_logger_settings 126 | end 127 | 128 | def http_propagated_fields=(value) 129 | @all_http_propagated_fields = DEFAULT_HTTP_PROPAGATED_FIELDS.merge(value) 130 | @http_propagated_fields = @all_http_propagated_fields.dup 131 | DEFAULT_HTTP_PROPAGATED_FIELDS.keys.each do |key| 132 | @http_propagated_fields.delete(key) 133 | end 134 | @http_propagated_fields.freeze 135 | end 136 | 137 | def jobs_propagated_fields=(value) 138 | @all_jobs_propagated_fields = (DEFAULT_HTTP_PROPAGATED_FIELDS.keys + value).uniq.freeze 139 | @jobs_propagated_fields = (@all_jobs_propagated_fields - DEFAULT_HTTP_PROPAGATED_FIELDS.keys).freeze 140 | end 141 | 142 | # shortcut 143 | def propagated_fields=(value) 144 | self.http_propagated_fields = value 145 | self.jobs_propagated_fields = value.keys 146 | end 147 | 148 | def log_path=(value) 149 | logger.path = value 150 | update_logger_settings 151 | end 152 | 153 | def log_level=(value) 154 | raise ArgumentError, "Unsupported log level #{value}" unless supported_log_level?(value) 155 | 156 | @log_level = value 157 | update_logger_settings 158 | end 159 | 160 | def disable_filter_for_log_levels=(log_levels) 161 | raise ArgumentError, "Unsupported log levels" unless array_with_log_levels?(log_levels) 162 | 163 | @disable_filter_for_log_levels = log_levels 164 | end 165 | 166 | def development_mode=(value) 167 | @development_mode = value 168 | update_logger_settings 169 | end 170 | 171 | def supported_log_levels 172 | SUPPORTED_LOG_LEVELS 173 | end 174 | 175 | def filter_enabled? 176 | !disable_filter_for_log_levels.include?(log_level) 177 | end 178 | 179 | private 180 | 181 | def update_logger_settings 182 | @logger.formatter = formatter 183 | @logger.level = @log_level 184 | end 185 | 186 | def formatter 187 | development_mode ? Logger::DEVELOPMENT_FORMATTER : Logger::FORMATTER 188 | end 189 | 190 | def default_log_level 191 | development_mode ? ::Logger::DEBUG : ::Logger::INFO 192 | end 193 | 194 | def array_with_log_levels?(log_levels) 195 | return false unless log_levels.is_a?(Array) 196 | 197 | log_levels.all? { |level| supported_log_level?(level) } 198 | end 199 | 200 | def supported_log_level?(log_level) 201 | supported_log_levels.value?(log_level) 202 | end 203 | end 204 | end 205 | -------------------------------------------------------------------------------- /lib/kiev/context_reader.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | # Abstracts common details about reading tracing context 5 | # into Kiev's request store. Subclass and override #[] to 6 | # change field lookup. 7 | class ContextReader 8 | REQUEST_ID = "request_id" 9 | TRACKING_ID = "tracking_id" 10 | REQUEST_DEPTH = "request_depth" 11 | TREE_PATH = "tree_path" 12 | 13 | def initialize(subject) 14 | @subject = subject 15 | end 16 | 17 | def [](key) 18 | subject[key] 19 | end 20 | 21 | def tracking_id 22 | presence(self[TRACKING_ID]) || presence(self[REQUEST_ID]) || SecureRandom.uuid 23 | end 24 | 25 | alias_method :request_id, :tracking_id 26 | 27 | def tree_root? 28 | !self[TRACKING_ID] && !self[REQUEST_ID] 29 | end 30 | 31 | def request_depth 32 | tree_root? ? 0 : (self[REQUEST_DEPTH].to_i + 1) 33 | end 34 | 35 | def tree_path 36 | if tree_root? 37 | SubrequestHelper.root_path(synchronous: false) 38 | else 39 | self[TREE_PATH] 40 | end 41 | end 42 | 43 | private 44 | 45 | attr_reader :subject 46 | 47 | def presence(value) 48 | return value if value && !value.empty? 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/kiev/hanami.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "rack/request_logger" 4 | require_relative "rack/store_request_details" 5 | require_relative "rack/request_id" 6 | 7 | module Kiev 8 | module Hanami 9 | def self.included(base) 10 | base.configure do 11 | # The order is important 12 | middleware.use(::RequestStore::Middleware) 13 | middleware.use(Kiev::Rack::RequestLogger) 14 | middleware.use(Kiev::Rack::StoreRequestDetails) 15 | middleware.use(Kiev::Rack::RequestId) 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/kiev/her_ext/client_request_id.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../base" 4 | 5 | module Kiev 6 | module HerExt 7 | class ClientRequestId < Faraday::Middleware 8 | def call(env) 9 | env[:request_headers].merge!(SubrequestHelper.headers) 10 | @app.call(env) 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/kiev/httparty.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base" 4 | 5 | module Kiev 6 | module HTTParty 7 | def self.headers 8 | SubrequestHelper.headers 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/kiev/json.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "time" 4 | 5 | module Kiev 6 | class JSON 7 | class << self 8 | attr_accessor :engine 9 | end 10 | end 11 | end 12 | 13 | begin 14 | require "oj" 15 | Kiev::JSON.engine = :oj 16 | rescue LoadError 17 | require "json" 18 | 19 | if defined?(ActiveSupport::JSON) 20 | Kiev::JSON.engine = :activesupport 21 | elsif defined?(::JSON) 22 | Kiev::JSON.engine = :json 23 | end 24 | end 25 | 26 | module Kiev 27 | class JSON 28 | OJ_OPTIONS_3 = { 29 | mode: :rails, 30 | use_as_json: true, 31 | use_to_json: true 32 | } # do not do freeze for Oj3 and Rails 4.1 33 | 34 | OJ_OPTIONS_2 = { 35 | float_precision: 16, 36 | bigdecimal_as_decimal: false, 37 | nan: :null, 38 | time_format: :xmlschema, 39 | second_precision: 3, 40 | mode: :compat, 41 | use_as_json: true, 42 | use_to_json: true 43 | }.freeze 44 | 45 | OJ_OPTIONS = (defined?(Oj::VERSION) && Oj::VERSION >= "3") ? OJ_OPTIONS_3 : OJ_OPTIONS_2 46 | 47 | FAIL_JSON = "{\"error_json\":\"failed to generate json\"}" 48 | NO_JSON = "{\"error_json\":\"no json backend\"}" 49 | 50 | class << self 51 | def generate(obj) 52 | if engine == :oj 53 | oj_generate(obj) 54 | elsif engine == :activesupport 55 | activesupport_generate(obj) 56 | elsif engine == :json 57 | json_generate(obj) 58 | else 59 | NO_JSON.dup 60 | end 61 | end 62 | 63 | def logstash(entry) 64 | entry.each do |key, value| 65 | entry[key] = if value.respond_to?(:iso8601) 66 | value.iso8601(3) 67 | elsif !scalar?(value) 68 | generate(value) 69 | elsif value.is_a?(String) && value.encoding != Encoding::UTF_8 70 | value.encode( 71 | Encoding::UTF_8, 72 | invalid: :replace, 73 | undef: :replace, 74 | replace: "?" 75 | ) 76 | elsif value.respond_to?(:infinite?) && value.infinite? 77 | nil 78 | else 79 | value 80 | end 81 | end 82 | 83 | generate(entry) << "\n" 84 | end 85 | 86 | private 87 | 88 | # Arrays excluded here because Elastic indexes very picky: 89 | # if you have array of mixed things it will complain 90 | def scalar?(value) 91 | value.is_a?(String) || 92 | value.is_a?(Numeric) || 93 | value.is_a?(Symbol) || 94 | value.is_a?(TrueClass) || 95 | value.is_a?(FalseClass) || 96 | value.is_a?(NilClass) 97 | end 98 | 99 | def oj_generate(obj) 100 | Oj.dump(obj, OJ_OPTIONS) 101 | rescue Exception 102 | FAIL_JSON.dup 103 | end 104 | 105 | def activesupport_generate(obj) 106 | ActiveSupport::JSON.encode(obj) 107 | rescue Exception 108 | FAIL_JSON.dup 109 | end 110 | 111 | def json_generate(obj) 112 | ::JSON.generate(obj, quirks_mode: true) 113 | rescue Exception 114 | FAIL_JSON.dup 115 | end 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/kiev/kafka.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base" 4 | 5 | module Kiev 6 | module Kafka 7 | require_relative "kafka/context_extractor" 8 | require_relative "kafka/context_injector" 9 | 10 | class << self 11 | # @param [Kafka::FetchedMessage] message 12 | def extract_context(message) 13 | Kiev::Kafka::ContextExtractor.new.call(message) 14 | end 15 | 16 | # @param [Hash] headers 17 | def inject_context(headers = {}) 18 | Kiev::Kafka::ContextInjector.new.call(headers) 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/kiev/kafka/context_extractor.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "message_context" 4 | require "kiev/request_id" 5 | require "kiev/context_reader" 6 | 7 | module Kiev 8 | module Kafka 9 | class ContextExtractor 10 | include Kiev::RequestId::Mixin 11 | 12 | # @param [Kafka::FetchedMessage] message 13 | def call(message) 14 | context = Kiev::Kafka::MessageContext.new(message) 15 | context_reader = Kiev::ContextReader.new(context) 16 | wrap_request_id(context_reader) {} 17 | 18 | Kiev[:message_key] = message.key 19 | 20 | Config.instance.jobs_propagated_fields.each do |key| 21 | Kiev[key] = context_reader[key] 22 | end 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/kiev/kafka/context_injector.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "kiev/config" 4 | require "kiev/subrequest_helper" 5 | 6 | module Kiev 7 | module Kafka 8 | class ContextInjector 9 | # @param [Hash] headers Injects context headers 10 | # @return [Hash] 11 | def call(headers = {}) 12 | Kiev::SubrequestHelper.payload.each do |key, value| 13 | field_key = Kiev::Config::DEFAULT_HTTP_PROPAGATED_FIELDS.fetch(key.to_sym, key) 14 | headers[field_key] = value 15 | end 16 | headers 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/kiev/kafka/message_context.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Kafka 5 | class MessageContext 6 | # @param [Kafka::FetchedMessage] message 7 | def initialize(message) 8 | @headers = message.headers 9 | end 10 | 11 | def value(field) 12 | headers[header_key(field)] || headers[field.to_s] 13 | end 14 | 15 | alias_method :[], :value 16 | 17 | private 18 | 19 | attr_reader :headers 20 | 21 | # @param [String] field 22 | def header_key(field) 23 | "x_#{field}".gsub("_", " ").split.map(&:capitalize).join("-") 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/kiev/logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "logger" 4 | require "time" 5 | require "forwardable" 6 | 7 | # Keep this class minimal and compatible with Ruby Logger. 8 | # If you add custom methods to this class and they will be used by developer, 9 | # it will be hard to swap this class with any other Logger implementation. 10 | module Kiev 11 | class Logger 12 | extend Forwardable 13 | def_delegators(*([:@logger] + ::Logger.instance_methods(false))) 14 | 15 | DEFAULT_LOG_NAME = "log" 16 | DEFAULT_MESSAGE = "log" 17 | LOG_ERROR = "ERROR" 18 | ERROR_STATUS = 500 19 | 20 | FORMATTER = proc do |severity, time, log_name, data| 21 | entry = 22 | { 23 | application: Config.instance.app, 24 | log_name: log_name || DEFAULT_LOG_NAME, 25 | level: severity, 26 | timestamp: time.utc, 27 | message: log_name || DEFAULT_MESSAGE, 28 | tracking_id: RequestStore.store[:tracking_id], 29 | request_id: RequestStore.store[:request_id], 30 | request_depth: RequestStore.store[:request_depth], 31 | tree_path: RequestStore.store[:tree_path] 32 | } 33 | 34 | # data required to restore source of log entry 35 | if RequestStore.store[:web] 36 | entry[:verb] = RequestStore.store[:request_verb] 37 | entry[:path] = RequestStore.store[:request_path] 38 | end 39 | if RequestStore.store[:background_job] 40 | entry[:job_name] = RequestStore.store[:job_name] 41 | entry[:jid] = RequestStore.store[:jid] 42 | end 43 | 44 | if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(log_name) 45 | entry[:tree_leaf] = true 46 | end 47 | 48 | if RequestStore.store[:payload] 49 | if %i(request_finished job_finished).include?(log_name) 50 | entry.merge!(RequestStore.store[:payload]) 51 | else 52 | Config.instance.persistent_log_fields.each do |field| 53 | entry[field] = RequestStore.store[:payload][field] 54 | end 55 | end 56 | end 57 | 58 | if data.is_a?(Hash) 59 | entry.merge!(data) 60 | elsif !data.nil? 61 | entry[:message] = data.to_s 62 | entry[:status] = ERROR_STATUS if data.to_s.downcase.include?(LOG_ERROR) 63 | end 64 | 65 | entry[:level] = LOG_ERROR if entry[:status].to_i.between?(400, 599) 66 | 67 | # Save some disk space 68 | entry.reject! { |_, value| value.nil? } 69 | 70 | JSON.logstash(entry) 71 | end 72 | 73 | DEVELOPMENT_FORMATTER = proc do |severity, time, log_name, data| 74 | entry = [] 75 | 76 | entry << time.iso8601 77 | entry << (log_name || severity).upcase 78 | 79 | if data.is_a?(String) 80 | entry << "#{data}\n" 81 | end 82 | 83 | if %i(request_finished job_finished).include?(log_name) 84 | verb = RequestStore.store[:request_verb] 85 | path = RequestStore.store[:request_path] 86 | entry << "#{verb} #{path}" if verb && path 87 | 88 | job_name = RequestStore.store[:job_name] 89 | jid = RequestStore.store[:jid] 90 | entry << "#{job_name} #{jid}" if job_name && jid 91 | 92 | status = data.is_a?(Hash) ? data.delete(:status) : nil 93 | entry << "- #{status}" if status 94 | duration = data.is_a?(Hash) ? data.delete(:request_duration) : nil 95 | entry << "(#{duration}ms)" if duration 96 | entry << "\n" 97 | 98 | meta = RequestStore.store.slice(:trakcing_id, :request_id, :request_depth) 99 | .reverse_merge!(Hash(RequestStore.store[:payload])) 100 | 101 | meta.reject! { |_, value| value.nil? } 102 | 103 | entry << " Meta: #{meta.inspect}\n" 104 | 105 | entry << " Params: #{data[:params].inspect}\n" if data.is_a?(Hash) && data[:params] 106 | 107 | if data.is_a?(Hash) && data[:body] 108 | entry << " Response: #{data[:body]}\n" 109 | end 110 | end 111 | 112 | entry.join(" ") 113 | end 114 | 115 | def initialize(log_path) 116 | @logger = ::Logger.new(log_path) 117 | end 118 | 119 | def path=(log_path) 120 | previous_logger = @logger 121 | @logger = ::Logger.new(log_path) 122 | if previous_logger 123 | @logger.level = previous_logger.level 124 | @logger.formatter = previous_logger.formatter 125 | end 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /lib/kiev/param_filter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | class ParamFilter 5 | FILTERED = "[FILTERED]" 6 | 7 | def self.filter(params, filtered_params, ignored_params) 8 | new(filtered_params, ignored_params).call(params) 9 | end 10 | 11 | def initialize(filtered_params, ignored_params) 12 | @filtered_params = normalize(filtered_params) 13 | @ignored_params = normalize(ignored_params) 14 | end 15 | 16 | def call(params) 17 | return params unless filterable?(params) 18 | 19 | params.each_with_object({}) do |(key, value), acc| 20 | next if ignored_params.include?(key.to_s) 21 | 22 | if defined?(ActionDispatch) && value.is_a?(ActionDispatch::Http::UploadedFile) 23 | value = { 24 | original_filename: value.original_filename, 25 | content_type: value.content_type, 26 | headers: value.headers 27 | } 28 | end 29 | 30 | acc[key] = 31 | if filtered_params.include?(key.to_s) && !value.is_a?(Hash) 32 | FILTERED 33 | elsif value.is_a?(Hash) 34 | call(value) 35 | else 36 | value 37 | end 38 | end 39 | end 40 | 41 | private 42 | 43 | attr_reader :filtered_params, :ignored_params 44 | 45 | def filterable?(params) 46 | params.respond_to?(:each_with_object) 47 | end 48 | 49 | def normalize(params) 50 | Set.new(params.map(&:to_s)) 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/kiev/que/job.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../base" 4 | 5 | module Kiev 6 | module Que 7 | # Original implementation https://github.com/chanks/que/blob/master/lib/que/job.rb 8 | class Job < ::Que::Job 9 | include Kiev::RequestStore::Mixin 10 | 11 | def self.enqueue(*args) 12 | if ::Que.mode == :async 13 | super(*args.unshift(SubrequestHelper.payload)) 14 | else 15 | super 16 | end 17 | end 18 | 19 | def _run 20 | if ::Que.mode == :async 21 | wrap_request_store { kiev_run } 22 | else 23 | kiev_run 24 | end 25 | end 26 | 27 | private 28 | 29 | NEW_LINE = "\n" 30 | LOG_ERROR = "ERROR" 31 | 32 | def kiev_run 33 | args = attrs[:args] 34 | payload = {} 35 | 36 | if args.first.is_a?(Hash) 37 | options = args.shift 38 | payload = Config.instance.all_jobs_propagated_fields.map do |key| 39 | # sometimes JSON decoder is overridden and it can be instructed to symbolize keys 40 | [key, options.delete(key.to_s) || options.delete(key)] 41 | end.to_h 42 | args.unshift(options) if options.any? 43 | end 44 | 45 | if ::Que.mode == :async 46 | Config.instance.jobs_propagated_fields.each do |key| 47 | Kiev[key] = payload[key] 48 | end 49 | request_store = Kiev::RequestStore.store 50 | request_store[:tracking_id] = payload[:tracking_id] 51 | request_store[:request_id] = payload[:tracking_id] || payload[:request_id] 52 | request_store[:request_depth] = payload[:request_depth].to_i + 1 53 | request_store[:tree_path] = payload[:tree_path] 54 | 55 | request_store[:background_job] = true 56 | request_store[:job_name] = attrs[:job_class] 57 | end 58 | 59 | began_at = Time.now 60 | 61 | ::Que::Job.instance_method(:_run).bind(self).call 62 | 63 | data = { 64 | params: attrs[:args], 65 | request_duration: ((Time.now - began_at) * 1000).round(3) 66 | } 67 | 68 | error ||= _error 69 | 70 | if error 71 | data[:error_class] = error.class.name 72 | data[:error_message] = error.message[0..5000] 73 | data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000] 74 | data[:level] = LOG_ERROR 75 | end 76 | 77 | Kiev.event(:job_finished, data) 78 | end 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/kiev/rack.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base" 4 | require_relative "rack/request_logger" 5 | require_relative "rack/request_id" 6 | require_relative "rack/store_request_details" 7 | require_relative "rack/silence_action_dispatch_logger" 8 | require_relative "../ext/rack/common_logger" 9 | 10 | module Kiev 11 | module Rack 12 | def self.included(base) 13 | # The order is important 14 | base.use(::RequestStore::Middleware) 15 | base.use(Kiev::Rack::RequestLogger) 16 | base.use(Kiev::Rack::StoreRequestDetails) 17 | base.use(Kiev::Rack::RequestId) 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/kiev/rack/request_id.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "securerandom" 4 | 5 | module Kiev 6 | module Rack 7 | class RequestId 8 | # for Rails 4 9 | RAILS_REQUEST_ID = "action_dispatch.request_id" 10 | 11 | def initialize(app) 12 | @app = app 13 | end 14 | 15 | def call(env) 16 | request_id_header_out = to_rack(:request_id) 17 | tracking_id_header_out = to_rack(:tracking_id) 18 | 19 | tracking_id = make_tracking_id(env[to_http(:tracking_id)] || env[RAILS_REQUEST_ID] || env[to_http(:request_id)]) 20 | RequestStore.store[:tracking_id] = tracking_id 21 | RequestStore.store[:request_id] = tracking_id 22 | RequestStore.store[:request_depth] = request_depth(env) 23 | RequestStore.store[:tree_path] = tree_path(env) 24 | 25 | @app.call(env).tap do |_status, headers, _body| 26 | headers[tracking_id_header_out] = tracking_id 27 | headers[request_id_header_out] = tracking_id 28 | end 29 | end 30 | 31 | private 32 | 33 | def tree_root?(env) 34 | tracking_id_header_in = to_http(:tracking_id) 35 | request_id_header_in = to_http(:request_id) 36 | !env[tracking_id_header_in] && !env[request_id_header_in] 37 | end 38 | 39 | def request_depth(env) 40 | request_depth_header = to_http(:request_depth) 41 | tree_root?(env) ? 0 : (env[request_depth_header].to_i + 1) 42 | end 43 | 44 | def tree_path(env) 45 | tree_path_header = to_http(:tree_path) 46 | tree_root?(env) ? SubrequestHelper.root_path(synchronous: true) : Util.sanitize(env[tree_path_header]) 47 | end 48 | 49 | def to_http(value) 50 | Util.to_http(to_rack(value)) 51 | end 52 | 53 | def to_rack(value) 54 | Config.instance.all_http_propagated_fields[value] 55 | end 56 | 57 | def make_tracking_id(tracking_id) 58 | if tracking_id.nil? || tracking_id.empty? 59 | internal_tracking_id 60 | else 61 | Util.sanitize(tracking_id) 62 | end 63 | end 64 | 65 | def internal_tracking_id 66 | SecureRandom.uuid 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/kiev/rack/request_logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "zlib" 4 | 5 | module Kiev 6 | module Rack 7 | class RequestLogger 8 | ERROR_STATUS = 500 9 | ERROR_HEADERS = [].freeze 10 | ERROR_BODY = [""].freeze 11 | LOG_ERROR = "ERROR" 12 | 13 | def initialize(app) 14 | @app = app 15 | end 16 | 17 | def call(env) 18 | rescued_exception = nil 19 | began_at = Time.now.to_f 20 | 21 | request = ::Rack::Request.new(env) 22 | 23 | begin 24 | status, headers, body = @app.call(env) 25 | rescue Exception => e 26 | rescued_exception = e 27 | 28 | status = ERROR_STATUS 29 | headers = ERROR_HEADERS 30 | body = ERROR_BODY 31 | 32 | if defined?(ActionDispatch::ExceptionWrapper) 33 | status = ::ActionDispatch::ExceptionWrapper.status_code_for_exception(rescued_exception.class.name) 34 | end 35 | end 36 | 37 | response = ::Rack::Response.new(body, status, headers) 38 | 39 | rack_exception = log_rack_exception?(env[SINATRA_ERROR]) ? env[SINATRA_ERROR] : nil 40 | log_exception = RequestStore.store[:error] 41 | exception = rescued_exception || rack_exception || log_exception 42 | 43 | if exception || Config.instance.log_request_condition.call(request, response) 44 | Kiev.event( 45 | :request_finished, 46 | form_data( 47 | began_at: began_at, 48 | env: env, 49 | request: request, 50 | response: response, 51 | status: status, 52 | body: body, 53 | exception: exception 54 | ) 55 | ) 56 | end 57 | 58 | raise rescued_exception if rescued_exception 59 | 60 | [status, headers, body] 61 | end 62 | 63 | private 64 | 65 | HTTP_USER_AGENT = "HTTP_USER_AGENT" 66 | ACTION_REQUEST_PARAMETERS = "action_dispatch.request.request_parameters" 67 | ACTION_QUERY_PARAMETERS = "action_dispatch.request.query_parameters" 68 | NEW_LINE = "\n" 69 | HTTP_X_REQUEST_START = "HTTP_X_REQUEST_START" 70 | SINATRA_ERROR = "sinatra.error" 71 | 72 | def log_rack_exception?(exception) 73 | !Config.instance.ignored_rack_exceptions.include?(exception.class.name) 74 | end 75 | 76 | def form_data(request:, began_at:, status:, env:, body:, response:, exception:) 77 | config = Config.instance 78 | 79 | params = 80 | if env[ACTION_REQUEST_PARAMETERS] && env[ACTION_QUERY_PARAMETERS] 81 | env[ACTION_REQUEST_PARAMETERS].merge(env[ACTION_QUERY_PARAMETERS]) 82 | elsif env[ACTION_REQUEST_PARAMETERS] 83 | env[ACTION_REQUEST_PARAMETERS] 84 | else 85 | request.params 86 | end 87 | 88 | data = { 89 | host: request.host, # env["HTTP_HOST"] || env["HTTPS_HOST"], 90 | params: params.empty? ? nil : params, # env[Rack::QUERY_STRING], 91 | ip: request.ip, # split_http_x_forwarded_headers(env) || env["REMOTE_ADDR"] 92 | user_agent: env[HTTP_USER_AGENT], 93 | status: status, 94 | request_duration: ((Time.now.to_f - began_at) * 1000).round(3), 95 | route: extract_route(env) 96 | } 97 | 98 | data[:level] = LOG_ERROR if data[:status].to_i.between?(400, 599) 99 | 100 | if env[HTTP_X_REQUEST_START] 101 | data[:request_latency] = ((began_at - env[HTTP_X_REQUEST_START].to_f) * 1000).round(3) 102 | end 103 | 104 | if config.log_request_body_condition.call(request, response) 105 | data[:request_body] = 106 | RequestBodyFilter.filter( 107 | request.content_type, 108 | request.body, 109 | config.filtered_params, 110 | config.ignored_params 111 | ) 112 | end 113 | 114 | if config.log_response_body_condition.call(request, response) 115 | # it should always respond to each, but this code is not streaming friendly 116 | full_body = [] 117 | body.each do |str| 118 | full_body << str 119 | end 120 | data[:body] = full_body.join 121 | if data[:body] && !data[:body].empty? && response.headers["Content-Encoding"] == "gzip" 122 | begin 123 | sio = StringIO.new(data[:body]) 124 | gz = Zlib::GzipReader.new(sio) 125 | data[:body] = gz.read 126 | rescue Zlib::GzipFile::Error => e 127 | data[:gzip_parse_error] = e.message 128 | end 129 | end 130 | end 131 | 132 | should_log_errors = config.log_request_error_condition.call(request, response) 133 | if should_log_errors && exception.is_a?(Exception) 134 | data[:error_class] = exception.class.name 135 | data[:error_message] = exception.message[0..5000] 136 | data[:error_backtrace] = Array(exception.backtrace).join(NEW_LINE)[0..5000] 137 | data[:level] = LOG_ERROR 138 | end 139 | data 140 | end 141 | 142 | def extract_route(env) 143 | action_params = env["action_dispatch.request.parameters"] 144 | if action_params 145 | "#{action_params['controller']}##{action_params['action']}" 146 | else 147 | env["sinatra.route"] 148 | end 149 | end 150 | end 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /lib/kiev/rack/silence_action_dispatch_logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Rack 5 | class SilenceActionDispatchLogger 6 | class << self 7 | attr_accessor :disabled 8 | end 9 | 10 | NULL_LOGGER = ::Logger.new("/dev/null") 11 | 12 | def initialize(app) 13 | @app = app 14 | end 15 | 16 | def call(env) 17 | env["action_dispatch.logger"] = NULL_LOGGER unless self.class.disabled 18 | @app.call(env) 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/kiev/rack/store_request_details.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Rack 5 | class StoreRequestDetails 6 | def initialize(app) 7 | @app = app 8 | end 9 | 10 | def call(env) 11 | request = ::Rack::Request.new(env) 12 | RequestStore.store[:web] = true 13 | RequestStore.store[:request_verb] = request.request_method 14 | RequestStore.store[:request_path] = request.path 15 | 16 | Config.instance.pre_rack_hook.call(env) 17 | @app.call(env) 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/kiev/railtie.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base" 4 | require "action_view/log_subscriber" 5 | require "action_controller/log_subscriber" 6 | 7 | module Kiev 8 | class Railtie < Rails::Railtie 9 | initializer("kiev.insert_middleware") do |app| 10 | app.config.middleware.insert_after(::RequestStore::Middleware, Kiev::Rack::RequestId) 11 | app.config.middleware.insert_after(Kiev::Rack::RequestId, Kiev::Rack::StoreRequestDetails) 12 | app.config.middleware.insert_after(ActionDispatch::ShowExceptions, Kiev::Rack::RequestLogger) 13 | end 14 | 15 | if Config.instance.disable_default_logger 16 | initializer("kiev.disable_default_logger") do |app| 17 | app.config.middleware.delete(Rails::Rack::Logger) 18 | app.config.middleware.insert_before(ActionDispatch::DebugExceptions, Kiev::Rack::SilenceActionDispatchLogger) 19 | Rails.logger = Config.instance.logger 20 | app.config.after_initialize do 21 | Kiev::Rack::SilenceActionDispatchLogger.disabled = app.config.consider_all_requests_local 22 | remove_existing_log_subscriptions unless Kiev::Config.instance.development_mode 23 | end 24 | end 25 | end 26 | 27 | private 28 | 29 | def remove_existing_log_subscriptions 30 | ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber| 31 | case subscriber 32 | when ActionView::LogSubscriber 33 | unsubscribe(:action_view, subscriber) 34 | when ActionController::LogSubscriber 35 | unsubscribe(:action_controller, subscriber) 36 | when defined?(ActiveRecord::LogSubscriber) && ActiveRecord::LogSubscriber 37 | unsubscribe(:active_record, subscriber) 38 | when defined?(SequelRails::Railties::LogSubscriber) && SequelRails::Railties::LogSubscriber 39 | unsubscribe(:sequel, subscriber) 40 | end 41 | end 42 | end 43 | 44 | def unsubscribe(component, subscriber) 45 | events = subscriber.public_methods(false).reject { |method| method.to_s == "call" } 46 | events.each do |event| 47 | ActiveSupport::Notifications.notifier.listeners_for("#{event}.#{component}").each do |listener| 48 | if listener.instance_variable_get("@delegate") == subscriber 49 | ActiveSupport::Notifications.unsubscribe(listener) 50 | end 51 | end 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/kiev/request_body_filter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "request_body_filter/default" 4 | require_relative "request_body_filter/xml" 5 | require_relative "request_body_filter/json" 6 | require_relative "request_body_filter/form_data" 7 | 8 | module Kiev 9 | module RequestBodyFilter 10 | FILTERED = "[FILTERED]" 11 | 12 | JSON_CONTENT_TYPE = %w(text/json application/json) 13 | XML_CONTENT_TYPES = %w(text/xml application/xml) 14 | FORM_DATA_CONTENT_TYPES = %w(application/x-www-form-urlencoded multipart/form-data) 15 | 16 | def self.for_content_type(content_type) 17 | case content_type 18 | when *JSON_CONTENT_TYPE 19 | Json 20 | when *XML_CONTENT_TYPES 21 | Xml 22 | when *FORM_DATA_CONTENT_TYPES 23 | FormData 24 | else 25 | Default 26 | end 27 | end 28 | 29 | def self.filter(content_type, request_body, filtered_params, ignored_params) 30 | body = request_body.read 31 | request_body.rewind 32 | 33 | return body unless Kiev.config.filter_enabled? 34 | 35 | body_filter = for_content_type(content_type) 36 | body_filter.call(body, filtered_params, ignored_params) 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/kiev/request_body_filter/default.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module RequestBodyFilter 5 | class Default 6 | def self.call(request_body, _filtered_params, _ignored_params) 7 | request_body 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/kiev/request_body_filter/form_data.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module RequestBodyFilter 5 | class FormData 6 | def self.call(request_body, filtered_params, ignored_params) 7 | params = ::Rack::Utils.parse_nested_query(request_body) 8 | ParamFilter.filter(params, filtered_params, ignored_params) 9 | end 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/kiev/request_body_filter/json.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module RequestBodyFilter 5 | class Json 6 | def self.call(request_body, filtered_params, ignored_params) 7 | params = ::JSON.parse(request_body) 8 | ParamFilter.filter(params, filtered_params, ignored_params) 9 | rescue Exception 10 | request_body 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/kiev/request_body_filter/xml.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "oga" 4 | 5 | module Kiev 6 | module RequestBodyFilter 7 | class Xml 8 | def self.call(request_body, filtered_params, _ignored_params) 9 | document = Oga.parse_xml(request_body) 10 | filtered_params.each do |param| 11 | sensitive_param = document.at_xpath("//#{param}/text()") 12 | sensitive_param.text = FILTERED if sensitive_param.respond_to?(:text=) 13 | end 14 | document.to_xml 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/kiev/request_id.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module RequestId 5 | module Mixin 6 | NEW_LINE = "\n" 7 | 8 | def wrap_request_id(context_reader, &_block) 9 | request_store = Kiev::RequestStore.store 10 | request_store[:tracking_id] = context_reader.tracking_id || context_reader.request_id 11 | request_store[:request_id] = request_store[:tracking_id] 12 | request_store[:request_depth] = context_reader.request_depth 13 | request_store[:tree_path] = context_reader.tree_path 14 | yield 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/kiev/request_logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module RequestLogger 5 | module Mixin 6 | NEW_LINE = "\n" 7 | LOG_ERROR = "ERROR" 8 | 9 | def wrap_request_logger(event, **data, &_block) 10 | began_at = Time.now 11 | error = nil 12 | 13 | begin 14 | return_value = yield 15 | rescue StandardError => e 16 | error = e 17 | end 18 | 19 | begin 20 | data[:request_duration] = ((Time.now - began_at) * 1000).round(3) 21 | if error 22 | data[:error_class] = error.class.name 23 | data[:error_message] = error.message[0..5000] 24 | data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000] 25 | data[:level] = LOG_ERROR 26 | end 27 | 28 | Kiev.event(event, data) 29 | ensure 30 | raise error if error 31 | 32 | return_value 33 | end 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/kiev/request_store.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module RequestStore 5 | def self.store 6 | ::RequestStore.store[:kiev] ||= {} 7 | end 8 | 9 | module Mixin 10 | def wrap_request_store_13 11 | ::RequestStore.begin! 12 | yield 13 | ensure 14 | ::RequestStore.end! 15 | ::RequestStore.clear! 16 | end 17 | 18 | def wrap_request_store_10 19 | ::RequestStore.clear! 20 | yield 21 | ensure 22 | ::RequestStore.clear! 23 | end 24 | 25 | if ::RequestStore::VERSION >= "1.3" 26 | alias_method :wrap_request_store, :wrap_request_store_13 27 | else 28 | alias_method :wrap_request_store, :wrap_request_store_10 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base" 4 | 5 | module Kiev 6 | module Shoryuken 7 | require_relative "shoryuken/middleware" 8 | 9 | class << self 10 | def enable(base = nil) 11 | base ||= ::Shoryuken 12 | base.configure_client do |config| 13 | enable_client_middleware(config) 14 | end 15 | base.configure_server do |config| 16 | enable_client_middleware(config) 17 | enable_server_middleware(config) 18 | end 19 | end 20 | 21 | def enable_server_middleware(config) 22 | server_mw_enabled = false 23 | config.server_middleware do |chain| 24 | chain.add(Middleware::RequestStore) 25 | chain.add(Middleware::RequestId) 26 | chain.add(Middleware::StoreRequestDetails) 27 | chain.add(Middleware::RequestLogger) 28 | server_mw_enabled = true 29 | end 30 | server_mw_enabled # Shoryuken configuration may skip that block in non-worker setups 31 | end 32 | 33 | def enable_client_middleware(config) 34 | config.client_middleware do |chain| 35 | chain.add(Middleware::MessageTracer) 36 | end 37 | end 38 | 39 | def suffix_tree_path(config, tag) 40 | config.server_middleware do |chain| 41 | chain.insert_after(Middleware::RequestId, Middleware::TreePathSuffix, tag) 42 | end 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/context_reader.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "kiev/context_reader" 4 | 5 | module Kiev 6 | module Shoryuken 7 | class ContextReader < Kiev::ContextReader 8 | def initialize(message) 9 | super 10 | @message_attributes = message.message_attributes 11 | end 12 | 13 | def [](key) 14 | return unless @message_attributes.key?(key) 15 | 16 | attribute_value = @message_attributes[key] 17 | return unless attribute_value.data_type == "String" 18 | 19 | attribute_value.string_value 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/middleware.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Client middleware 4 | require_relative "middleware/message_tracer" 5 | 6 | # Server middleware 7 | require_relative "middleware/request_id" 8 | require_relative "middleware/request_logger" 9 | require_relative "middleware/request_store" 10 | require_relative "middleware/store_request_details" 11 | require_relative "middleware/tree_path_suffix" 12 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/middleware/message_tracer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "kiev/aws_sns/context_injector" 4 | 5 | module Kiev 6 | module Shoryuken 7 | module Middleware 8 | class MessageTracer 9 | def call(options) 10 | options[:message_attributes] ||= {} 11 | Kiev::AwsSns::ContextInjector.new.call(options[:message_attributes]) 12 | yield 13 | end 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/middleware/request_id.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "securerandom" 4 | require "kiev/request_id" 5 | require "kiev/shoryuken/context_reader" 6 | 7 | module Kiev 8 | module Shoryuken 9 | module Middleware 10 | class RequestId 11 | include Kiev::RequestId::Mixin 12 | 13 | def call(_worker, _queue, message, _body, &block) 14 | context_reader = Kiev::Shoryuken::ContextReader.new(message) 15 | wrap_request_id(context_reader, &block) 16 | end 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/middleware/request_logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Shoryuken 5 | module Middleware 6 | class RequestLogger 7 | include Kiev::RequestLogger::Mixin 8 | 9 | def call(_worker, _queue, _message, body, &block) 10 | wrap_request_logger(:job_finished, body: body, &block) 11 | end 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/middleware/request_store.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Shoryuken 5 | module Middleware 6 | class RequestStore 7 | include Kiev::RequestStore::Mixin 8 | 9 | def call(_worker, _queue, _message, _body, &block) 10 | wrap_request_store(&block) 11 | end 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/middleware/store_request_details.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "kiev/shoryuken/context_reader" 4 | 5 | module Kiev 6 | module Shoryuken 7 | module Middleware 8 | class StoreRequestDetails 9 | def call(_worker, _queue, message, _body) 10 | context_reader = Kiev::Shoryuken::ContextReader.new(message) 11 | Config.instance.jobs_propagated_fields.each do |key| 12 | Kiev[key] = context_reader[key] 13 | end 14 | request_store = Kiev::RequestStore.store 15 | request_store[:background_job] = true 16 | request_store[:message_id] = message.message_id 17 | yield 18 | end 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/kiev/shoryuken/middleware/tree_path_suffix.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Shoryuken 5 | module Middleware 6 | class TreePathSuffix 7 | def initialize(tag) 8 | @tag = tag.dup.freeze 9 | end 10 | 11 | def call(_worker, _queue, _message, _body) 12 | request_store = Kiev::RequestStore.store 13 | request_store[:tree_path] ||= "" 14 | request_store[:tree_path] += @tag 15 | yield 16 | end 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/kiev/sidekiq.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base" 4 | 5 | module Kiev 6 | module Sidekiq 7 | require_relative "sidekiq/client_request_id" 8 | require_relative "sidekiq/request_store" 9 | require_relative "sidekiq/request_logger" 10 | require_relative "sidekiq/request_id" 11 | require_relative "sidekiq/store_request_details" 12 | 13 | class << self 14 | def enable(base = nil) 15 | base ||= ::Sidekiq 16 | base.configure_client do |config| 17 | enable_client_middleware(config) 18 | end 19 | base.configure_server do |config| 20 | enable_client_middleware(config) 21 | enable_server_middleware(config) 22 | end 23 | end 24 | 25 | def enable_server_middleware(config) 26 | config.server_middleware do |chain| 27 | chain.prepend(Kiev::Sidekiq::RequestLogger) 28 | chain.prepend(Kiev::Sidekiq::StoreRequestDetails) 29 | chain.prepend(Kiev::Sidekiq::RequestId) 30 | chain.prepend(Kiev::Sidekiq::RequestStore) 31 | end 32 | end 33 | 34 | def enable_client_middleware(config) 35 | config.client_middleware do |chain| 36 | chain.prepend(Kiev::Sidekiq::ClientRequestId) 37 | end 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/kiev/sidekiq/client_request_id.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Sidekiq 5 | class ClientRequestId 6 | def call(_worker_class, job, _queue, _redis_pool) 7 | job.merge!(SubrequestHelper.payload) 8 | yield 9 | end 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/kiev/sidekiq/request_id.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "securerandom" 4 | require "kiev/request_id" 5 | require "kiev/context_reader" 6 | 7 | module Kiev 8 | module Sidekiq 9 | class RequestId 10 | include Kiev::RequestId::Mixin 11 | 12 | def call(_worker, job, _queue, &block) 13 | context_reader = Kiev::ContextReader.new(job) 14 | wrap_request_id(context_reader, &block) 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/kiev/sidekiq/request_logger.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Sidekiq 5 | class RequestLogger 6 | include Kiev::RequestLogger::Mixin 7 | 8 | ARGS = "args" 9 | 10 | def call(_worker, job, _queue, &block) 11 | wrap_request_logger(:job_finished, params: job[ARGS], &block) 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/kiev/sidekiq/request_store.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Sidekiq 5 | class RequestStore 6 | include Kiev::RequestStore::Mixin 7 | 8 | def call(_worker, _job, _queue, &block) 9 | wrap_request_store(&block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/kiev/sidekiq/store_request_details.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Sidekiq 5 | class StoreRequestDetails 6 | JID = "jid" 7 | WRAPPED = "wrapped" 8 | 9 | def call(worker, job, _queue) 10 | Config.instance.jobs_propagated_fields.each do |key| 11 | Kiev[key] = job[key.to_s] 12 | end 13 | request_store = Kiev::RequestStore.store 14 | request_store[:background_job] = true 15 | request_store[:job_name] = expand_worker_name(worker, job) 16 | request_store[:jid] = job[JID] 17 | yield 18 | end 19 | 20 | private 21 | 22 | def expand_worker_name(worker, job) 23 | job[WRAPPED] || worker.class.name 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/kiev/subrequest_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "base52" 4 | 5 | module Kiev 6 | class SubrequestHelper 7 | class << self 8 | def headers(metadata: false) 9 | Config.instance.all_http_propagated_fields.map do |key, http_key| 10 | field = field_value(key, true) 11 | [metadata ? key : http_key, field.to_s] if field 12 | end.compact.to_h 13 | end 14 | 15 | def payload 16 | Config.instance.all_jobs_propagated_fields.map do |key| 17 | field = field_value(key, false) 18 | [key.to_s, field] if field 19 | end.compact.to_h 20 | end 21 | 22 | def root_path(synchronous:) 23 | encode(0, synchronous) 24 | end 25 | 26 | def subrequest_path(synchronous:) 27 | current_path + encode(counter, synchronous) 28 | end 29 | 30 | private 31 | 32 | def field_value(key, synchronous) 33 | if key == :tree_path 34 | subrequest_path(synchronous: synchronous) 35 | else 36 | request_store = Kiev::RequestStore.store 37 | request_store.dig(key) || request_store.dig(:payload, key) 38 | end 39 | end 40 | 41 | def encode(value, synchronous) 42 | # this scheme can encode up to 26 consequent requests (synchronous or asynchronous) 43 | Base52.encode(value * 2 + (synchronous ? 0 : 1)) 44 | end 45 | 46 | def current_path 47 | RequestStore.store[:tree_path] || "" 48 | end 49 | 50 | def counter 51 | if RequestStore.store[:subrequest_count] 52 | # generally this is not atomic operation, 53 | # but because RequestStore.store is tied to current thread this is ok 54 | RequestStore.store[:subrequest_count] += 1 55 | else 56 | RequestStore.store[:subrequest_count] = 0 57 | end 58 | end 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/kiev/test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "json" 4 | 5 | module Kiev 6 | # Test helpers for testing both Kiev itself and products that use it. 7 | module Test 8 | module Log 9 | STREAM = StringIO.new 10 | 11 | module_function 12 | 13 | def configure 14 | @logs = [] 15 | Kiev.configure do |c| 16 | c.log_path = STREAM 17 | end 18 | end 19 | 20 | def clear 21 | STREAM.rewind 22 | STREAM.truncate(0) 23 | @logs = [] 24 | end 25 | 26 | def entries 27 | return @logs unless @logs.empty? 28 | 29 | @logs = raw_logs.each_line.map(&::JSON.method(:parse)) 30 | rescue StandardError 31 | puts raw_logs 32 | raise 33 | end 34 | 35 | def raw_logs 36 | STREAM.string 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/kiev/util.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | module Util 5 | def self.sanitize(value) 6 | return unless value 7 | 8 | value.gsub(/[^\w\-]/, "")[0...255] 9 | end 10 | 11 | def self.to_http(value) 12 | "HTTP_#{value.tr('-', '_').upcase}" 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/kiev/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Kiev 4 | VERSION = "4.9.0" 5 | end 6 | -------------------------------------------------------------------------------- /spec/helpers/log_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | LOG_IO = StringIO.new 4 | ROOT_FOLDER = File.expand_path(File.dirname(__FILE__)).to_s.freeze 5 | DATA_FOLDER = "#{ROOT_FOLDER}/data" 6 | 7 | require "json" 8 | module KievHelper 9 | LOG_IO = StringIO.new 10 | 11 | def enable_log_tracking 12 | Kiev.configure do |c| 13 | c.log_path = LOG_IO 14 | end 15 | end 16 | 17 | def disable_log_tracking 18 | Kiev.configure do |c| 19 | c.log_path = "/dev/null" 20 | end 21 | end 22 | 23 | def reset_logs 24 | LOG_IO.rewind 25 | LOG_IO.truncate(0) 26 | @logs = nil 27 | end 28 | 29 | def logs 30 | return @logs if @logs 31 | LOG_IO.rewind 32 | @logs = LOG_IO.read.split("\n").map(&JSON.method(:parse)) 33 | end 34 | 35 | def log_first 36 | logs.first 37 | end 38 | 39 | def log_last 40 | logs.last 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /spec/helpers/que_helper.rb: -------------------------------------------------------------------------------- 1 | # https://github.com/chanks/que/blob/master/spec/spec_helper.rb 2 | # frozen_string_literal: true 3 | 4 | require "uri" 5 | require "pg" 6 | 7 | # Handy constants for initializing PG connections: 8 | QUE_URL = ENV["DATABASE_URL"] || "postgres://postgres:@localhost/que_test" 9 | 10 | NEW_PG_CONNECTION = proc do 11 | uri = URI.parse(QUE_URL) 12 | pg = PG::Connection.open( 13 | host: uri.host, 14 | user: uri.user, 15 | password: uri.password, 16 | port: uri.port || 5432, 17 | dbname: uri.path[1..-1] 18 | ) 19 | 20 | # Avoid annoying NOTICE messages in specs. 21 | pg.async_exec("SET client_min_messages TO 'warning'") 22 | pg 23 | end 24 | 25 | Que.connection = NEW_PG_CONNECTION.call 26 | QUE_ADAPTERS = { pg: Que.adapter } 27 | 28 | # We use Sequel to examine the database in specs. 29 | require "sequel" 30 | DB = Sequel.connect(QUE_URL) 31 | # DB.loggers << Logger.new($stdout) 32 | 33 | if ENV["CI"] 34 | DB.synchronize do |conn| 35 | puts "Ruby #{RUBY_VERSION}" 36 | puts "Sequel #{Sequel::VERSION}" 37 | puts conn.async_exec("SELECT version()").to_a.first["version"] 38 | end 39 | end 40 | 41 | # Reset the table to the most up-to-date version. 42 | DB.drop_table?(:que_jobs) 43 | Que::Migrations.migrate! 44 | -------------------------------------------------------------------------------- /spec/lib/kiev/base52_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | describe Kiev::Base52 do 6 | describe "encode" do 7 | subject { (0..51).map(&Kiev::Base52.method(:encode)) } 8 | 9 | it "can encode 52 numbes in unique way" do 10 | expect(subject.uniq.length).to eq(52) 11 | end 12 | 13 | it "can encode 52 numbes with one char" do 14 | expect(subject.map(&:length).reduce(:+)).to eq(52) 15 | end 16 | 17 | it "produces 52 lexicographically sortable values" do 18 | (0..50).each do |x| 19 | expect(Kiev::Base52.encode(x) < Kiev::Base52.encode(x + 1)).to eq(true) 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/lib/kiev/config_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "ostruct" 5 | 6 | describe Kiev::Config do 7 | describe "constants" do 8 | it do 9 | expect(described_class::FILTERED_PARAMS).to eq( 10 | %w( 11 | client_secret 12 | token 13 | password 14 | password_confirmation 15 | old_password 16 | credit_card_number 17 | credit_card_cvv 18 | credit_card_holder 19 | credit_card_expiry_month 20 | credit_card_expiry_year 21 | CardNumber 22 | CardCVV 23 | CardExpires 24 | ) 25 | ) 26 | end 27 | it do 28 | expect(described_class::IGNORED_PARAMS).to eq( 29 | %w( 30 | controller 31 | action 32 | format 33 | authenticity_token 34 | utf8 35 | tempfile 36 | ) << :tempfile 37 | ) 38 | end 39 | end 40 | 41 | describe "DEFAULT_LOG_REQUEST_ERROR_CONDITION" do 42 | let(:request) { nil } 43 | subject { described_class::DEFAULT_LOG_REQUEST_ERROR_CONDITION } 44 | context "404 status code" do 45 | let(:response) { OpenStruct.new(status: 404) } 46 | it { expect(subject.call(request, response)).to eq(false) } 47 | end 48 | context "400 status code" do 49 | let(:response) { OpenStruct.new(status: 400) } 50 | it { expect(subject.call(request, response)).to eq(true) } 51 | end 52 | context "200 status code" do 53 | let(:response) { OpenStruct.new(status: 200) } 54 | it { expect(subject.call(request, response)).to eq(true) } 55 | end 56 | end 57 | 58 | describe "DEFAULT_LOG_REQUEST_CONDITION" do 59 | let(:response) { nil } 60 | subject { described_class::DEFAULT_LOG_REQUEST_CONDITION } 61 | context "simple request" do 62 | let(:request) { OpenStruct.new(path: "/") } 63 | it { expect(subject.call(request, response)).to eq(true) } 64 | end 65 | context "health request" do 66 | let(:request) { OpenStruct.new(path: "/health") } 67 | it { expect(subject.call(request, response)).to eq(false) } 68 | end 69 | context "ping request" do 70 | let(:request) { OpenStruct.new(path: "/ping") } 71 | it { expect(subject.call(request, response)).to eq(false) } 72 | end 73 | context "ready request" do 74 | let(:request) { OpenStruct.new(path: "/ready") } 75 | it { expect(subject.call(request, response)).to eq(false) } 76 | end 77 | context "live request" do 78 | let(:request) { OpenStruct.new(path: "/live") } 79 | it { expect(subject.call(request, response)).to eq(false) } 80 | end 81 | context "/something/ping request" do 82 | let(:request) { OpenStruct.new(path: "/something/ping") } 83 | it { expect(subject.call(request, response)).to eq(true) } 84 | end 85 | context "media request" do 86 | let(:request) { OpenStruct.new(path: "/test.jpg") } 87 | it { expect(subject.call(request, response)).to eq(false) } 88 | end 89 | end 90 | 91 | describe "DEFAULT_LOG_RESPONSE_BODY_CONDITION" do 92 | let(:request) { nil } 93 | subject { described_class::DEFAULT_LOG_RESPONSE_BODY_CONDITION } 94 | context "500 status code for json response" do 95 | let(:response) { OpenStruct.new(status: 500, content_type: "json/something") } 96 | it { expect(subject.call(request, response)).to eq(false) } 97 | end 98 | context "400 status code for json response" do 99 | let(:response) { OpenStruct.new(status: 400, content_type: "json/something") } 100 | it { expect(subject.call(request, response)).to eq(true) } 101 | end 102 | context "200 status code for json response" do 103 | let(:response) { OpenStruct.new(status: 200, content_type: "json/something") } 104 | it { expect(subject.call(request, response)).to eq(false) } 105 | end 106 | context "500 status code for xml response" do 107 | let(:response) { OpenStruct.new(status: 500, content_type: "xml/something") } 108 | it { expect(subject.call(request, response)).to eq(false) } 109 | end 110 | context "400 status code for xml response" do 111 | let(:response) { OpenStruct.new(status: 400, content_type: "xml/something") } 112 | it { expect(subject.call(request, response)).to eq(true) } 113 | end 114 | context "200 status code for xml response" do 115 | let(:response) { OpenStruct.new(status: 200, content_type: "xml/something") } 116 | it { expect(subject.call(request, response)).to eq(false) } 117 | end 118 | context "400 status code for html response" do 119 | let(:response) { OpenStruct.new(status: 400, content_type: "html/something") } 120 | it { expect(subject.call(request, response)).to eq(false) } 121 | end 122 | end 123 | 124 | describe "DEFAULT_PRE_RACK_HOOK" do 125 | let(:env) { { "HTTP_FIELD" => "f" } } 126 | subject { described_class::DEFAULT_PRE_RACK_HOOK } 127 | before do 128 | allow_any_instance_of(described_class).to receive(:http_propagated_fields) { { field: "field" } } 129 | allow(Kiev::Util).to receive(:sanitize).and_call_original 130 | allow(Kiev::Util).to receive(:to_http).and_call_original 131 | allow(Kiev).to receive(:[]=) 132 | end 133 | it do 134 | expect(Kiev::Util).to receive(:sanitize).with("f") 135 | expect(Kiev::Util).to receive(:to_http).with("field") 136 | expect(Kiev).to receive(:[]=).with(:field, "f") 137 | subject.call(env) 138 | end 139 | end 140 | 141 | describe "http_propagated_fields" do 142 | subject { described_class.instance } 143 | before { subject.http_propagated_fields = {} } 144 | after { subject.http_propagated_fields = {} } 145 | it do 146 | subject.http_propagated_fields = { 147 | request_id: "Request-Id", 148 | http_field: "X-Field" 149 | } 150 | expect(subject.http_propagated_fields).to eq(http_field: "X-Field") 151 | expect(subject.all_http_propagated_fields).to eq( 152 | tracking_id: "X-Tracking-Id", 153 | request_id: "Request-Id", 154 | request_depth: "X-Request-Depth", 155 | tree_path: "X-Tree-Path", 156 | http_field: "X-Field" 157 | ) 158 | end 159 | end 160 | 161 | describe "jobs_propagated_fields" do 162 | subject { described_class.instance } 163 | before { subject.jobs_propagated_fields = [] } 164 | after { subject.jobs_propagated_fields = [] } 165 | it do 166 | subject.jobs_propagated_fields = [:request_id, :jobs_field] 167 | expect(subject.jobs_propagated_fields).to eq([:jobs_field]) 168 | expect(subject.all_jobs_propagated_fields).to eq( 169 | [:tracking_id, :request_id, :request_depth, :tree_path, :jobs_field] 170 | ) 171 | end 172 | end 173 | 174 | describe "log_level" do 175 | subject { described_class.instance } 176 | 177 | context "when unsupported log level provided" do 178 | it { expect { subject.log_level = 123 }.to raise_error(ArgumentError, "Unsupported log level 123") } 179 | end 180 | end 181 | 182 | describe "disable_filter_for_log_levels" do 183 | subject { described_class.instance } 184 | 185 | context "when unsupported log levels provided" do 186 | context "when not array" do 187 | it do 188 | expect { 189 | subject.disable_filter_for_log_levels = 1 190 | }.to raise_error(ArgumentError, "Unsupported log levels") 191 | end 192 | end 193 | 194 | context "with unsupported level" do 195 | it do 196 | expect { 197 | subject.disable_filter_for_log_levels = [0, 1, 2, 8] 198 | }.to raise_error(ArgumentError, "Unsupported log levels") 199 | end 200 | end 201 | end 202 | end 203 | end 204 | -------------------------------------------------------------------------------- /spec/lib/kiev/json_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "bigdecimal" 5 | require "zlib" 6 | 7 | class AsJson 8 | def as_json(_options = nil) 9 | { a: 1 } 10 | end 11 | 12 | def to_json(_options = nil) 13 | '{"a":1}' 14 | end 15 | end 16 | 17 | describe Kiev::JSON do 18 | TEST_DATA = { 19 | Regexp: /test/, 20 | StringChinese: "二胡", 21 | StringSpecial: "\u2028\u2029><&", 22 | StringSpecial2: "\/", 23 | StringSpecial3: "\\\b\f\n\r\t", 24 | Time: Time.new(2012, 1, 5, 23, 58, 7.99996, 32_400), 25 | Date: Date.new(2012, 1, 5, 23), 26 | DateTime: DateTime.new(2012, 1, 5, 23, 58, 7.99996, 32_400), 27 | BigDecimal: BigDecimal("1") / 3, 28 | BigDecimalInfinity: BigDecimal("0.5") / 0, 29 | Float: 1.0 / 3, 30 | FloatInfinity: 0.5 / 0, 31 | Range: (1..10), 32 | Complex: Complex("0.3-0.5i"), 33 | Exception: Exception.new, 34 | OpenStruct: OpenStruct.new(country: "Australia", population: 20_000_000), 35 | Rational: Rational(0.3), 36 | AsJson: AsJson.new 37 | } 38 | 39 | before do 40 | @engine = Kiev::JSON.engine 41 | end 42 | 43 | after do 44 | Kiev::JSON.engine = @engine 45 | end 46 | 47 | subject { Kiev::JSON.generate(data) } 48 | let(:data) { TEST_DATA } 49 | 50 | it "accept all fancy stuff with Oj" do 51 | skip unless defined?(Oj) 52 | Kiev::JSON.engine = :oj 53 | expect(subject.frozen?).to be(false) 54 | # Obviously it's not Sidekiq itself, but env setup specific to Sidekiq 55 | if defined?(::Sidekiq) 56 | expect(subject).to eq( 57 | "{\"Regexp\":\"(?-mix:test)\",\"StringChinese\":\"二胡\"," \ 58 | "\"StringSpecial\":\"\u2028\u2029><&\",\"StringSpecial2\":\"/\",\"StringSpecial3\":\"\\\\\\b\\f\\n\\r\\t\"," \ 59 | "\"Time\":\"2012-01-05 23:58:07 +0900\",\"Date\":\"2012-01-05\",\"DateTime\":\"2012-01-05T23:58:07+00:00\"," \ 60 | "\"BigDecimal\":\"0.333333333333333333e0\",\"BigDecimalInfinity\":\"Infinity\",\"Float\":0.3333333333333333," \ 61 | "\"FloatInfinity\":null,\"Range\":\"1..10\",\"Complex\":\"0.3-0.5i\",\"Exception\":\"Exception\"," \ 62 | "\"OpenStruct\":\"#\"," \ 63 | "\"Rational\":\"5404319552844595/18014398509481984\",\"AsJson\":{\"a\":1}}" 64 | ) 65 | elsif defined?(ActiveSupport::JSON) 66 | expect(subject).to eq( 67 | "{\"Regexp\":\"(?-mix:test)\",\"StringChinese\":\"二胡\"," \ 68 | "\"StringSpecial\":\"\\u2028\\u2029\\u003e\\u003c\\u0026\",\"StringSpecial2\":\"/\"," \ 69 | "\"StringSpecial3\":\"\\\\\\b\\f\\n\\r\\t\",\"Time\":\"2012-01-05T23:58:07.999+09:00\"," \ 70 | "\"Date\":\"2012-01-05\",\"DateTime\":\"2012-01-05T23:58:07.999+00:00\"," \ 71 | "\"BigDecimal\":\"0.333333333333333333\",\"BigDecimalInfinity\":null,\"Float\":0.3333333333333333," \ 72 | "\"FloatInfinity\":null,\"Range\":\"1..10\",\"Complex\":\"0.3-0.5i\",\"Exception\":\"Exception\"," \ 73 | "\"OpenStruct\":{\"table\":{\"country\":\"Australia\",\"population\":20000000}}," \ 74 | "\"Rational\":\"5404319552844595/18014398509481984\",\"AsJson\":{\"a\":1}}" 75 | ) 76 | else 77 | puts 78 | expect(subject).to eq( 79 | "{\"Regexp\":\"(?-mix:test)\",\"StringChinese\":\"二胡\"," \ 80 | "\"StringSpecial\":\"\\u2028\\u2029\\u003e\\u003c\\u0026\",\"StringSpecial2\":\"/\"," \ 81 | "\"StringSpecial3\":\"\\\\\\b\\f\\n\\r\\t\",\"Time\":\"2012-01-05 23:58:07 +0900\",\"Date\":\"2012-01-05\"," \ 82 | "\"DateTime\":\"2012-01-05T23:58:07+00:00\",\"BigDecimal\":\"0.333333333333333333e0\"," \ 83 | "\"BigDecimalInfinity\":null,\"Float\":0.3333333333333333,\"FloatInfinity\":null,\"Range\":\"1..10\"," \ 84 | "\"Complex\":\"0.3-0.5i\",\"Exception\":\"Exception\"," \ 85 | "\"OpenStruct\":\"#\\u003cOpenStruct country=\\\"Australia\\\", population=20000000\\u003e\"," \ 86 | "\"Rational\":\"5404319552844595/18014398509481984\",\"AsJson\":{\"a\":1}}" 87 | ) 88 | end 89 | end 90 | 91 | it "accept all fancy staff with ActiveSupport" do 92 | skip unless defined?(ActiveSupport::JSON) 93 | Kiev::JSON.engine = :activesupport 94 | expect(subject.frozen?).to be(false) 95 | # Obviously it's not Sidekiq itself, but env setup specific to Sidekiq 96 | if !defined?(::Sidekiq) 97 | expect(subject).to eq( 98 | "{\"Regexp\":\"(?-mix:test)\",\"StringChinese\":\"二胡\"," \ 99 | "\"StringSpecial\":\"\\u2028\\u2029\\u003e\\u003c\\u0026\",\"StringSpecial2\":\"/\"," \ 100 | "\"StringSpecial3\":\"\\\\\\b\\f\\n\\r\\t\",\"Time\":\"2012-01-05T23:58:07.999+09:00\"," \ 101 | "\"Date\":\"2012-01-05\",\"DateTime\":\"2012-01-05T23:58:07.999+00:00\"," \ 102 | "\"BigDecimal\":\"0.333333333333333333\",\"BigDecimalInfinity\":null,\"Float\":0.3333333333333333," \ 103 | "\"FloatInfinity\":null,\"Range\":\"1..10\",\"Complex\":\"0.3-0.5i\",\"Exception\":\"Exception\"," \ 104 | "\"OpenStruct\":{\"table\":{\"country\":\"Australia\",\"population\":20000000}}," \ 105 | "\"Rational\":\"5404319552844595/18014398509481984\",\"AsJson\":{\"a\":1}}" 106 | ) 107 | else 108 | expect(subject).to eq( 109 | "{\"Regexp\":\"(?-mix:test)\",\"StringChinese\":\"二胡\"," \ 110 | "\"StringSpecial\":\"\\u2028\\u2029\\u003e\\u003c\\u0026\",\"StringSpecial2\":\"/\"," \ 111 | "\"StringSpecial3\":\"\\\\\\b\\f\\n\\r\\t\",\"Time\":\"2012-01-05T23:58:07.999+09:00\"," \ 112 | "\"Date\":\"2012-01-05\",\"DateTime\":\"2012-01-05T23:58:07.999+00:00\"," \ 113 | "\"BigDecimal\":\"0.333333333333333333\",\"BigDecimalInfinity\":null,\"Float\":0.3333333333333333," \ 114 | "\"FloatInfinity\":null,\"Range\":\"1..10\",\"Complex\":\"0.3-0.5i\",\"Exception\":\"Exception\"," \ 115 | "\"OpenStruct\":{\"table\":{\"country\":\"Australia\",\"population\":20000000}}," \ 116 | "\"Rational\":\"5404319552844595/18014398509481984\",\"AsJson\":{\"a\":1}}" 117 | ) 118 | end 119 | end 120 | 121 | it "does accept some fancy staff without ActiveSupport and Oj" do 122 | skip unless defined?(::JSON) 123 | Kiev::JSON.engine = :json 124 | data = TEST_DATA.dup 125 | data.delete(:FloatInfinity) 126 | expect(subject.frozen?).to be(false) 127 | if defined?(ActiveSupport::JSON) 128 | expect(Kiev::JSON.generate(data)).to eq( 129 | "{\"Regexp\":\"(?-mix:test)\",\"StringChinese\":\"二胡\"," \ 130 | "\"StringSpecial\":\"\u2028\u2029><&\",\"StringSpecial2\":\"/\",\"StringSpecial3\":\"\\\\\\b\\f\\n\\r\\t\"," \ 131 | "\"Time\":\"2012-01-05 23:58:07 +0900\",\"Date\":\"2012-01-05\",\"DateTime\":\"2012-01-05T23:58:07+00:00\"," \ 132 | "\"BigDecimal\":\"0.333333333333333333\",\"BigDecimalInfinity\":\"Infinity\",\"Float\":0.3333333333333333," \ 133 | "\"Range\":\"1..10\",\"Complex\":\"0.3-0.5i\",\"Exception\":\"Exception\"," \ 134 | "\"OpenStruct\":\"#\"," \ 135 | "\"Rational\":\"5404319552844595/18014398509481984\",\"AsJson\":{\"a\":1}}" 136 | ) 137 | else 138 | expect(Kiev::JSON.generate(data)).to eq( 139 | "{\"Regexp\":\"(?-mix:test)\",\"StringChinese\":\"二胡\"," \ 140 | "\"StringSpecial\":\"\u2028\u2029><&\",\"StringSpecial2\":\"/\",\"StringSpecial3\":\"\\\\\\b\\f\\n\\r\\t\"," \ 141 | "\"Time\":\"2012-01-05 23:58:07 +0900\",\"Date\":\"2012-01-05\",\"DateTime\":\"2012-01-05T23:58:07+00:00\"," \ 142 | "\"BigDecimal\":\"0.333333333333333333e0\",\"BigDecimalInfinity\":\"Infinity\",\"Float\":0.3333333333333333," \ 143 | "\"Range\":\"1..10\",\"Complex\":\"0.3-0.5i\",\"Exception\":\"Exception\"," \ 144 | "\"OpenStruct\":\"#\"," \ 145 | "\"Rational\":\"5404319552844595/18014398509481984\",\"AsJson\":{\"a\":1}}" 146 | ) 147 | end 148 | end 149 | 150 | it "does not accept float infinity without ActiveSupport and Oj" do 151 | skip unless defined?(::JSON) 152 | Kiev::JSON.engine = :json 153 | expect(subject).to eq("{\"error_json\":\"failed to generate json\"}") 154 | expect(subject.frozen?).to be(false) 155 | end 156 | 157 | it "does not accept binary encoding" do 158 | # Obviously it's not Sidekiq itself, but env setup specific to Sidekiq 159 | skip if defined?(::Sidekiq) 160 | data = { body: Zlib::Deflate.deflate("some text") } 161 | expect(Kiev::JSON.generate(data)).to eq("{\"error_json\":\"failed to generate json\"}") 162 | expect(subject.frozen?).to be(false) 163 | end 164 | 165 | context :logstash do 166 | subject { Kiev::JSON.logstash(data) } 167 | 168 | context "binary encoding" do 169 | let(:data) { { body: Zlib::Deflate.deflate("some text") } } 170 | 171 | it "accepts binary encoding" do 172 | expect(subject).to eq("{\"body\":\"x?+??MU(I?(\\u0001\\u0000\\u0011?\\u0003?\"}\n") 173 | end 174 | end 175 | 176 | context "float infinity" do 177 | let(:data) { { FloatInfinity: 0.5 / 0, NegativeFloatInfinity: -0.5 / 0 } } 178 | 179 | it "accepts float infinity" do 180 | Kiev::JSON.engine = :json 181 | expect(subject).to eq("{\"FloatInfinity\":null,\"NegativeFloatInfinity\":null}\n") 182 | end 183 | end 184 | end 185 | end 186 | -------------------------------------------------------------------------------- /spec/lib/kiev/kafka/context_extractor_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | if defined?(::Kafka) 6 | describe Kiev::Kafka::ContextExtractor do 7 | after do 8 | ::RequestStore.store[:kiev] = {} 9 | ::RequestStore.store.delete(:subrequest_count) 10 | end 11 | context 'when message has context' do 12 | let(:tracking_id) { SecureRandom.uuid } 13 | 14 | subject { described_class.new.call(message) } 15 | let(:headers) do 16 | { 17 | "X-Tracking-Id" => tracking_id, 18 | "X-Tree-Path" =>"KAFKA", 19 | "X-Request-Depth" => "4" 20 | } 21 | end 22 | let(:message) do 23 | Kafka::FetchedMessage.new(message: Kafka::Protocol::Record.new(key: "msg_key", value: "", headers: headers), topic: "", partition: 0) 24 | end 25 | it "extracts basic fields" do 26 | subject 27 | expect(Kiev.request_id).to eq(tracking_id) 28 | expect(Kiev::RequestStore.store[:tree_path]).to eq("KAFKA") 29 | expect(Kiev::RequestStore.store[:request_depth]).to eq(5) 30 | expect(Kiev::RequestStore.store.dig(:payload, :message_key)).to eq("msg_key") 31 | end 32 | 33 | context "for headers in plain format (no X- and uppercase)" do 34 | let(:headers) do 35 | { 36 | "tracking_id" => tracking_id, 37 | "tree_path" =>"KAFkA", 38 | "request_depth" => "3" 39 | } 40 | end 41 | 42 | it "extracts them as well" do 43 | subject 44 | expect(Kiev.request_id).to eq(tracking_id) 45 | expect(Kiev::RequestStore.store[:tree_path]).to eq("KAFkA") 46 | expect(Kiev::RequestStore.store[:request_depth]).to eq(4) 47 | expect(Kiev::RequestStore.store.dig(:payload, :message_key)).to eq("msg_key") 48 | end 49 | end 50 | 51 | context "extra fields if job-propagated are also stored in payload context" do 52 | let(:headers) do 53 | { 54 | "other_uuid" => tracking_id, 55 | "skip_me" =>"not tracked", 56 | "X-Accept-This" => "3" 57 | } 58 | end 59 | 60 | it "extracts them as well" do 61 | allow(Kiev::Config.instance).to receive(:jobs_propagated_fields).and_return(%i(other_uuid accept_this)) 62 | subject 63 | payload_context = Kiev::RequestStore.store[:payload] 64 | expect(payload_context[:other_uuid]).to eq(tracking_id) 65 | expect(payload_context[:accept_this]).to eq("3") 66 | expect(payload_context[:skip_me]).to be_nil 67 | end 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /spec/lib/kiev/kafka/context_injector_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | if defined?(::Kafka) 6 | describe Kiev::Kafka::ContextInjector do 7 | before do 8 | ::RequestStore.store[:subrequest_count] = 3 9 | end 10 | after do 11 | ::RequestStore.store[:kiev] = {} 12 | ::RequestStore.store.delete(:subrequest_count) 13 | end 14 | 15 | let(:kiev_store) { Kiev::RequestStore.store } 16 | 17 | subject { described_class.new.call } 18 | let(:tracking_id) { SecureRandom.uuid } 19 | before do 20 | kiev_store[:tracking_id] = tracking_id 21 | end 22 | 23 | it "injects payload context into hash argument" do 24 | expect(subject).to eq( 25 | "X-Tracking-Id" => tracking_id, 26 | "X-Tree-Path" => "B" 27 | ) 28 | end 29 | 30 | context "when more context present" do 31 | before do 32 | kiev_store[:tree_path] = "FAKA" 33 | kiev_store[:request_id] = tracking_id 34 | kiev_store[:request_depth] = 3 35 | end 36 | 37 | it "returns more in headers" do 38 | expect(subject).to eq( 39 | "X-Tracking-Id" => tracking_id, 40 | "X-Request-Id" => tracking_id, 41 | "X-Request-Depth" => 3, 42 | "X-Tree-Path" => "FAKAB" 43 | ) 44 | end 45 | end 46 | 47 | context "when jobs_propagated_fields setup" do 48 | before do 49 | default_setup = Kiev::Config.instance.all_jobs_propagated_fields 50 | allow(Kiev::Config.instance).to receive(:all_jobs_propagated_fields).and_return( 51 | default_setup + %i(some_other_field and_another) 52 | ) 53 | Kiev[:some_other_field] = "foo" 54 | Kiev[:and_another] = "bar" 55 | Kiev[:skip_me] = "bar" 56 | end 57 | 58 | it "passes them" do 59 | expect(subject).to eq( 60 | "X-Tracking-Id" => tracking_id, 61 | "X-Tree-Path" => "B", 62 | "some_other_field" => "foo", 63 | "and_another" => "bar" 64 | ) 65 | end 66 | end 67 | 68 | it "enhances argument headers variable as well" do 69 | some_headers = {foo: 42} 70 | described_class.new.call(some_headers) 71 | expect(some_headers).to eq( 72 | "X-Tracking-Id" => tracking_id, 73 | "X-Tree-Path" => "B", 74 | foo: 42 75 | ) 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /spec/lib/kiev/logger_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "json" 5 | 6 | describe Kiev::Logger do 7 | describe "FORMATTER" do 8 | before do 9 | Kiev.configure do |c| 10 | c.app = "test_app" 11 | c.persistent_log_fields = [:client] 12 | end 13 | ::RequestStore.clear! 14 | end 15 | 16 | after do 17 | Kiev.configure do |c| 18 | c.persistent_log_fields = [] 19 | end 20 | end 21 | 22 | def format(opts = {}) 23 | time = Time.new(2000, 1, 1, 0, 0, 0, "+00:00") 24 | event = opts.is_a?(Hash) && opts.delete(:event) || :test_event 25 | described_class::FORMATTER.call( 26 | ::Logger::Severity::INFO, time, event, opts 27 | ) 28 | end 29 | 30 | def subject(opts = {}) 31 | JSON.parse(format(opts)) 32 | end 33 | 34 | it "sets application" do 35 | expect(subject["application"]).to eq("test_app") 36 | end 37 | 38 | it "sets event" do 39 | expect(subject["log_name"]).to eq("test_event") 40 | end 41 | 42 | it "sets level" do 43 | expect(subject["level"]).to eq(1) 44 | end 45 | 46 | it "sets timestamp" do 47 | expect(subject["timestamp"]).to eq("2000-01-01T00:00:00.000Z") 48 | end 49 | 50 | it "sets request_id" do 51 | Kiev::RequestStore.store[:request_id] = "test_id" 52 | expect(subject["request_id"]).to eq("test_id") 53 | end 54 | 55 | it "sets tracking_id" do 56 | Kiev::RequestStore.store[:tracking_id] = "test_id" 57 | expect(subject["tracking_id"]).to eq("test_id") 58 | end 59 | 60 | it "sets web path and verb" do 61 | Kiev::RequestStore.store[:web] = true 62 | Kiev::RequestStore.store[:request_verb] = "GET" 63 | Kiev::RequestStore.store[:request_path] = "/test_path" 64 | expect(subject["verb"]).to eq("GET") 65 | expect(subject["path"]).to eq("/test_path") 66 | end 67 | 68 | it "sets job name" do 69 | Kiev::RequestStore.store[:background_job] = true 70 | Kiev::RequestStore.store[:job_name] = "TestJob" 71 | expect(subject["job_name"]).to eq("TestJob") 72 | end 73 | 74 | it "sets payload for request_finished" do 75 | Kiev[:test_payload] = true 76 | expect(subject["test_payload"]).to eq(nil) 77 | expect(subject(event: :request_finished)["test_payload"]).to eq(true) 78 | end 79 | 80 | it "sets persistent_log_fields for non request_finished" do 81 | Kiev[:client] = "test client" 82 | expect(subject["client"]).to eq("test client") 83 | end 84 | 85 | it "accepts string as message" do 86 | expect(subject("test_message")["message"]).to eq("test_message") 87 | end 88 | 89 | it "accepts hash as message" do 90 | expect(subject(data: 123)["data"]).to eq(123) 91 | end 92 | 93 | it "ends with new line" do 94 | expect(format).to match(/\n$/) 95 | end 96 | 97 | it "doesn't fail for 2 args" do 98 | time = Time.new(2000, 1, 1, 0, 0, 0, "+00:00") 99 | subj = described_class::FORMATTER.call(::Logger::Severity::INFO, time) 100 | 101 | expected = "{\"application\":\"test_app\",\"log_name\":\"log\",\"level\":1,"\ 102 | "\"timestamp\":\"2000-01-01T00:00:00.000Z\",\"message\":\"log\"}\n" 103 | expect(subj).to eq(expected) 104 | end 105 | 106 | it "calls #iso8601 on Time objects" do 107 | subj = subject(some_time_object: Time.new(2015, 1, 1, 12, 13, 14.123, "+00:00")) 108 | expect(subj["some_time_object"]).to eq("2015-01-01T12:13:14.122+00:00") 109 | end 110 | 111 | it "calls #iso8601 on DateTime objects" do 112 | skip unless defined?(DateTime) 113 | 114 | subj = subject(some_datetime_object: DateTime.new(2015, 1, 1, 12, 13, 14, "+00:00")) 115 | expect(subj["some_datetime_object"]).to eq("2015-01-01T12:13:14.000+00:00") 116 | end 117 | 118 | it "calls #iso8601 on ActiveSupport::TimeWithZone objects" do 119 | skip unless defined?(ActiveSupport::TimeWithZone) 120 | require "active_support/core_ext/time" 121 | 122 | subj = subject(some_timezone_object: Time.new(2015, 1, 1, 12, 13, 14, "+00:00").in_time_zone("UTC")) 123 | expect(subj["some_timezone_object"]).to eq("2015-01-01T12:13:14.000Z") 124 | end 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /spec/lib/kiev/param_filter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | describe Kiev::ParamFilter do 6 | describe "filter" do 7 | let(:filtered) { Kiev::Config.instance.filtered_params } 8 | let(:ignored) { Kiev::Config.instance.ignored_params } 9 | 10 | it "filters param" do 11 | expect(described_class.filter({ "password" => "password" }, filtered, ignored)).to eq("password" => "[FILTERED]") 12 | end 13 | 14 | it "filters nested param" do 15 | expected = { "u" => { "password" => "[FILTERED]" } } 16 | expect(described_class.filter({ "u" => { "password" => "password" } }, filtered, ignored)).to eq(expected) 17 | end 18 | 19 | it "filters only leafs" do 20 | input = { "token" => { "token" => "token", "type" => "type" } } 21 | expected = { "token" => { "token" => "[FILTERED]", "type" => "type" } } 22 | expect(described_class.filter(input, filtered, ignored)).to eq(expected) 23 | end 24 | 25 | it "filters symbol param" do 26 | expect(described_class.filter({ "password": "password" }, filtered, ignored)).to eq("password": "[FILTERED]") 27 | end 28 | 29 | it "filters mixed param" do 30 | expect(described_class.filter({ "password": "password", "password" => "password" }, filtered, ignored)) 31 | .to eq("password": "[FILTERED]", "password" => "[FILTERED]") 32 | end 33 | 34 | it "ignores param" do 35 | expect(described_class.filter({ "utf8" => "utf8" }, filtered, ignored)).to eq({}) 36 | end 37 | 38 | it "ignores nested param" do 39 | expect(described_class.filter({ "form" => { "action" => "submit" } }, filtered, ignored)).to eq("form" => {}) 40 | end 41 | 42 | it "ignores symbol param" do 43 | expect(described_class.filter({ "utf8": "utf8" }, filtered, ignored)).to eq({}) 44 | end 45 | 46 | it "ignores mixed params" do 47 | expect(described_class.filter({ "utf8": "utf8", "utf8" => "utf8" }, filtered, ignored)).to eq({}) 48 | end 49 | 50 | context "when configuration params specified as strings and symbols at the same time" do 51 | context "when filtered" do 52 | let(:filtered) { [:password, "type"] } 53 | 54 | it "filters both" do 55 | expect(described_class.filter({ type: "type", "password" => "password"}, filtered, ignored)) 56 | .to eq(type: "[FILTERED]", "password" => "[FILTERED]") 57 | end 58 | end 59 | 60 | context "when ignored" do 61 | let(:ignored) { [:password, "type"] } 62 | 63 | it "ignores both" do 64 | expect(described_class.filter({ type: "type", "password" => "password"}, filtered, ignored)).to eq({}) 65 | end 66 | end 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /spec/lib/kiev/que/job_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | if defined?(::Que::Job) 6 | describe Kiev::Que::Job do 7 | around do |spec| 8 | Que.adapter = QUE_ADAPTERS[:pg] 9 | Que.worker_count = 0 10 | Que.mode = :async 11 | Que.wake_interval = nil 12 | 13 | spec.run 14 | 15 | Que.mode = :off 16 | DB[:que_jobs].delete 17 | # A bit of lint: make sure that no advisory locks are left open. 18 | unless DB[:pg_locks].where(locktype: "advisory").empty? 19 | stdout.info("Advisory lock left open: #{desc} @ #{line}") 20 | end 21 | end 22 | 23 | class GlobalStore 24 | class << self 25 | attr_accessor :passed_args 26 | end 27 | end 28 | 29 | class TestJob < Kiev::Que::Job 30 | def run(*argument) 31 | GlobalStore.passed_args = argument 32 | end 33 | end 34 | 35 | class ErrorJob < Kiev::Que::Job 36 | def run(*argument) 37 | GlobalStore.passed_args = argument 38 | GlobalStore.undefined_method 39 | end 40 | end 41 | 42 | include KievHelper 43 | 44 | before do 45 | enable_log_tracking 46 | reset_logs 47 | Kiev::RequestStore.store.clear 48 | Kiev::RequestStore.store[:request_id] = "test" 49 | Kiev::RequestStore.store[:request_depth] = 0 50 | Kiev::RequestStore.store[:tree_path] = "Q" 51 | GlobalStore.passed_args = nil 52 | end 53 | 54 | after do 55 | disable_log_tracking 56 | end 57 | 58 | it "works with sync run" do 59 | TestJob.run("Hello world!") 60 | expect(GlobalStore.passed_args).to eq(["Hello world!"]) 61 | expect(log_first).to eq(nil) 62 | end 63 | 64 | it "works with sync enqueue" do 65 | ::Que.mode = :sync 66 | TestJob.enqueue("Hello world!") 67 | expect(GlobalStore.passed_args).to eq(["Hello world!"]) 68 | expect(log_first).to eq(nil) 69 | end 70 | 71 | it "works with async enqueue" do 72 | TestJob.enqueue("Hello world!") 73 | Kiev::RequestStore.store.clear 74 | Que::Job.work 75 | expect(GlobalStore.passed_args).to eq(["Hello world!"]) 76 | expect(log_first["application"]).to eq("test_app") 77 | expect(log_first["log_name"]).to eq("job_finished") 78 | expect(log_first["job_name"]).to eq("TestJob") 79 | expect(log_first["level"]).to eq("INFO") 80 | expect(log_first["params"]).to eq("[\"Hello world!\"]") 81 | expect(log_first["request_depth"]).to eq(1) 82 | expect(log_first["request_id"]).to eq("test") 83 | expect(log_first["tree_path"]).to eq("QB") 84 | expect(log_first["tree_leaf"]).to eq(true) 85 | expect(log_first["request_duration"]).to be 86 | expect(log_first["timestamp"]).to be 87 | expect(log_first["error_class"]).to be_nil 88 | expect(log_first["error_message"]).to be_nil 89 | expect(log_first["error_backtrace"]).to be_nil 90 | expect(Kiev::RequestStore.store[:request_id]).to be_nil 91 | end 92 | 93 | it "logs error for async enqueue" do 94 | ErrorJob.enqueue("Hello world!") 95 | Kiev::RequestStore.store.clear 96 | Que::Job.work 97 | expect(GlobalStore.passed_args).to eq(["Hello world!"]) 98 | expect(log_first).to be 99 | expect(log_first["application"]).to eq("test_app") 100 | expect(log_first["log_name"]).to eq("job_finished") 101 | expect(log_first["job_name"]).to eq("ErrorJob") 102 | expect(log_first["level"]).to eq("ERROR") 103 | expect(log_first["params"]).to eq("[\"Hello world!\"]") 104 | expect(log_first["request_depth"]).to eq(1) 105 | expect(log_first["request_id"]).to eq("test") 106 | expect(log_first["tree_path"]).to eq("QB") 107 | expect(log_first["tree_leaf"]).to eq(true) 108 | expect(log_first["request_duration"]).to be 109 | expect(log_first["timestamp"]).to be 110 | expect(log_first["error_class"]).to eq("NoMethodError") 111 | expect(log_first["error_message"]).to start_with("undefined method `undefined_method' for GlobalStore:Class") 112 | expect(log_first["error_backtrace"]).to be 113 | expect(Kiev::RequestStore.store[:request_id]).to be_nil 114 | end 115 | 116 | it "does not log error for sync enqueue" do 117 | ::Que.mode = :sync 118 | expect { ErrorJob.enqueue("Hello world!") }.to raise_error(NoMethodError) 119 | expect(GlobalStore.passed_args).to eq(["Hello world!"]) 120 | expect(log_first).to eq(nil) 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /spec/lib/kiev/rack/request_id_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | if defined?(Rack) 6 | describe Kiev::Rack::RequestId do 7 | include Rack::Test::Methods 8 | let(:app) { described_class.new(proc { [200, {}, ["Hello, world."]] }) } 9 | before { ::RequestStore.clear! } 10 | 11 | describe "X-Request-Id" do 12 | subject { get("/") } 13 | 14 | context "when there is no X-Request-Id" do 15 | before { allow(SecureRandom).to receive(:uuid).and_return("qwerty") } 16 | it do 17 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 18 | expect(subject.headers["X-Request-Id"]).to eq("qwerty") 19 | expect(subject.headers["X-Tracking-Id"]).to eq("qwerty") 20 | expect(Kiev::RequestStore.store[:request_id]).to eq("qwerty") 21 | expect(Kiev::RequestStore.store[:tracking_id]).to eq("qwerty") 22 | end 23 | end 24 | 25 | context "when there is X-Request-Id" do 26 | before { header("X-Request-Id", "test") } 27 | it do 28 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 29 | expect(subject.headers["X-Request-Id"]).to eq("test") 30 | expect(subject.headers["X-Tracking-Id"]).to eq("test") 31 | expect(Kiev::RequestStore.store[:request_id]).to eq("test") 32 | expect(Kiev::RequestStore.store[:tracking_id]).to eq("test") 33 | end 34 | end 35 | 36 | context "when there are both X-Request-Id and X-Tracking-Id with different values" do 37 | before do 38 | header("X-Request-Id", "req-test") 39 | header("X-Tracking-Id", "track-test") 40 | end 41 | it "returns and logs tracking_id as it has precedence" do 42 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 43 | expect(Kiev::RequestStore.store[:tracking_id]).to eq(nil) 44 | expect(subject.headers["X-Request-Id"]).to eq("track-test") 45 | expect(subject.headers["X-Tracking-Id"]).to eq("track-test") 46 | expect(Kiev::RequestStore.store[:request_id]).to eq("track-test") 47 | expect(Kiev::RequestStore.store[:tracking_id]).to eq("track-test") 48 | end 49 | end 50 | 51 | context "when there is big X-Request-Id" do 52 | before { header("X-Request-Id", "a" * 300) } 53 | it do 54 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 55 | expect(subject.headers["X-Request-Id"]).to eq("a" * 255) 56 | expect(subject.headers["X-Tracking-Id"]).to eq("a" * 255) 57 | expect(Kiev::RequestStore.store[:request_id]).to eq("a" * 255) 58 | expect(Kiev::RequestStore.store[:tracking_id]).to eq("a" * 255) 59 | end 60 | end 61 | 62 | context "when there is a X-Request-Id set to an empty string" do 63 | before { header("X-Request-Id", "") } 64 | it do 65 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 66 | expect(subject.headers["X-Request-Id"]).not_to eq("") 67 | expect(subject.headers["X-Tracking-Id"]).not_to eq("") 68 | expect(Kiev::RequestStore.store[:request_id]).not_to eq("") 69 | expect(Kiev::RequestStore.store[:tracking_id]).not_to eq("") 70 | end 71 | end 72 | end 73 | 74 | describe "X-Tracking-Id" do 75 | subject { get("/") } 76 | 77 | context "when there is no X-Tracking-Id" do 78 | before { allow(SecureRandom).to receive(:uuid).and_return("qwerty") } 79 | it do 80 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 81 | expect(Kiev::RequestStore.store[:tracking_id]).to eq(nil) 82 | expect(subject.headers["X-Request-Id"]).to eq("qwerty") 83 | expect(subject.headers["X-Tracking-Id"]).to eq("qwerty") 84 | expect(Kiev::RequestStore.store[:request_id]).to eq("qwerty") 85 | expect(Kiev::RequestStore.store[:tracking_id]).to eq("qwerty") 86 | end 87 | end 88 | 89 | context "when there is X-Tracking-Id" do 90 | before { header("X-Tracking-Id", "test") } 91 | it do 92 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 93 | expect(Kiev::RequestStore.store[:tracking_id]).to eq(nil) 94 | expect(subject.headers["X-Request-Id"]).to eq("test") 95 | expect(subject.headers["X-Tracking-Id"]).to eq("test") 96 | expect(Kiev::RequestStore.store[:tracking_id]).to eq("test") 97 | expect(Kiev::RequestStore.store[:request_id]).to eq("test") 98 | end 99 | end 100 | 101 | context "when there is big X-Tracking-Id" do 102 | before { header("X-Tracking-Id", "a" * 300) } 103 | it do 104 | expect(Kiev::RequestStore.store[:request_id]).to eq(nil) 105 | expect(subject.headers["X-Tracking-Id"]).to eq("a" * 255) 106 | expect(Kiev::RequestStore.store[:request_id]).to eq("a" * 255) 107 | end 108 | end 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /spec/lib/kiev/rack/request_logger_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "zlib" 5 | 6 | if defined?(Rack) 7 | describe Kiev::Rack::RequestLogger do 8 | include Rack::Test::Methods 9 | before do 10 | allow(Kiev).to receive(:event) 11 | allow(logger).to receive(:log) 12 | allow(Time).to receive(:now).and_return(Time.new(2000)) 13 | end 14 | 15 | let(:rack_app) { proc { [200, {}, ["body"]] } } 16 | let(:app) { described_class.new(rack_app) } 17 | let(:logger) { Kiev::Config.instance.logger } 18 | subject do 19 | get("/") 20 | Kiev 21 | end 22 | 23 | def request_finished(options = {}) 24 | [:request_finished, { 25 | host: "example.org", 26 | params: nil, 27 | ip: "127.0.0.1", 28 | user_agent: nil, 29 | status: 200, 30 | request_duration: 0.0, 31 | route: nil 32 | }.merge(options)] 33 | end 34 | 35 | context "200 response" do 36 | it "logs request" do 37 | expect(subject).to have_received(:event).with(*request_finished) 38 | end 39 | 40 | it "sets user_agent" do 41 | header("User-Agent", "Mozilla") 42 | expect(subject).to have_received(:event).with(*request_finished(user_agent: "Mozilla")) 43 | end 44 | 45 | context "other requests" do 46 | subject { Kiev } 47 | 48 | it "sets params" do 49 | get("/", test: "123") 50 | expect(subject).to have_received(:event).with(*request_finished(params: { "test" => "123" })) 51 | end 52 | 53 | it "filters params" do 54 | allow(Kiev).to receive(:event).and_call_original 55 | get("/", password: "secret") 56 | 57 | expect(logger).to have_received(:log) 58 | .with( 59 | 1, 60 | request_finished(params: { "password" => "[FILTERED]" }).last, 61 | :request_finished 62 | ) 63 | end 64 | 65 | it "ignores params" do 66 | allow(Kiev).to receive(:event).and_call_original 67 | get("/", utf8: "1") 68 | 69 | expect(logger).to have_received(:log).with(1, request_finished(params: {}).last, :request_finished) 70 | end 71 | 72 | it "ignores request body" do 73 | post("/", "{\"password\":\"secret\"}", "CONTENT_TYPE" => "application/json") 74 | expect(subject).to have_received(:event).with(*request_finished) 75 | end 76 | end 77 | end 78 | 79 | context "401 response" do 80 | context "html" do 81 | let(:rack_app) { proc { [401, { "Content-Type" => "text/html" }, "body"] } } 82 | it "does not log body" do 83 | expect(subject).to have_received(:event).with(*request_finished(status: 401, level: "ERROR")) 84 | end 85 | end 86 | 87 | context "json" do 88 | let(:rack_app) { proc { [401, { "Content-Type" => "application/json" }, ["{\"secret\":\"not filtered\"}"]] } } 89 | it "logs body" do 90 | expect(subject).to have_received(:event) 91 | .with(*request_finished(status: 401, body: "{\"secret\":\"not filtered\"}", level: "ERROR")) 92 | end 93 | end 94 | 95 | context "xml" do 96 | let(:rack_app) do 97 | proc { [401, { "Content-Type" => "text/xml" }, ["not filtered"]] } 98 | end 99 | it "logs body" do 100 | expect(subject).to have_received(:event) 101 | .with(*request_finished(status: 401, body: "not filtered", level: "ERROR")) 102 | end 103 | end 104 | end 105 | 106 | context "404 response" do 107 | context "html" do 108 | let(:rack_app) { proc { [404, { "Content-Type": "text/html" }, "body"] } } 109 | it "does not log body" do 110 | expect(subject).to have_received(:event) 111 | .with(*request_finished(status: 404, level: "ERROR")) 112 | end 113 | end 114 | 115 | context "json" do 116 | let(:payload) { %({ "a": 1 }) } 117 | let(:rack_app) { proc { [404, { "Content-Type" => "application/json" }, [payload]] } } 118 | it "logs body" do 119 | expect(subject).to have_received(:event) 120 | .with(*request_finished(status: 404, body: payload, level: "ERROR")) 121 | end 122 | end 123 | end 124 | 125 | context "gzip" do 126 | let(:raw_payload) { %({ "a": 1 }) } 127 | context "correctly encoded" do 128 | let(:payload) { 129 | sio = StringIO.new 130 | gz = Zlib::GzipWriter.new(sio) 131 | gz.write(raw_payload) 132 | gz.close 133 | sio.string 134 | } 135 | let(:rack_app) { 136 | proc { 137 | [404, { "Content-Type" => "application/json", "Content-Encoding" => "gzip" }, [payload]] 138 | } 139 | } 140 | it "logs body" do 141 | expect(subject).to have_received(:event) 142 | .with(*request_finished(status: 404, body: raw_payload, level: "ERROR")) 143 | end 144 | end 145 | 146 | context "improperly encoded" do 147 | let(:payload) { raw_payload } 148 | let(:rack_app) { 149 | proc { 150 | [404, { "Content-Type" => "application/json", "Content-Encoding" => "gzip" }, [payload]] 151 | } 152 | } 153 | it "logs gzip pars error" do 154 | expect(subject).to have_received(:event) 155 | .with(*request_finished(status: 404, body: raw_payload, gzip_parse_error: "not in gzip format", level: "ERROR")) 156 | end 157 | end 158 | end 159 | end 160 | end 161 | -------------------------------------------------------------------------------- /spec/lib/kiev/rack/store_request_details_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | if defined?(Rack) 6 | describe Kiev::Rack::StoreRequestDetails do 7 | include Rack::Test::Methods 8 | let(:app) { described_class.new(proc { [200, { "HTTP_X_REQUEST_ID" => "id" }, ["Hello, world."]] }) } 9 | before { ::RequestStore.clear! } 10 | 11 | describe "RequestStore" do 12 | subject { get("/") } 13 | 14 | it "stores request_path" do 15 | expect(Kiev::RequestStore.store[:request_path]).to eq(nil) 16 | subject 17 | expect(Kiev::RequestStore.store[:request_path]).to eq("/") 18 | end 19 | 20 | it "stores request_verb" do 21 | expect(Kiev::RequestStore.store[:request_verb]).to eq(nil) 22 | subject 23 | expect(Kiev::RequestStore.store[:request_verb]).to eq("GET") 24 | end 25 | 26 | it "stores web" do 27 | expect(Kiev::RequestStore.store[:web]).to eq(nil) 28 | subject 29 | expect(Kiev::RequestStore.store[:web]).to eq(true) 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /spec/lib/kiev/request_body_filter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "rack" 5 | 6 | describe Kiev::RequestBodyFilter do 7 | describe ".filter" do 8 | let(:request) do 9 | Rack::Request.new( 10 | Rack::MockRequest.env_for( 11 | "/", 12 | "REQUEST_METHOD" => "POST", 13 | "CONTENT_TYPE" => "application/json", 14 | :input => "{\"password\":12345}" 15 | ) 16 | ) 17 | end 18 | 19 | it "doesn't filter" do 20 | initial = Kiev.config.disable_filter_for_log_levels 21 | Kiev.config.disable_filter_for_log_levels = [::Logger::INFO] 22 | result = described_class.filter( 23 | "application/json", 24 | request.body, 25 | ["password"], 26 | [] 27 | ) 28 | expect(result).to eq("{\"password\":12345}") 29 | Kiev.config.disable_filter_for_log_levels = initial 30 | end 31 | 32 | it "filters by default" do 33 | result = described_class.filter( 34 | "application/json", 35 | request.body, 36 | ["password"], 37 | [] 38 | ) 39 | expect(result).to eq("password" => "[FILTERED]") 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /spec/lib/kiev/shoryuken_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "kiev/test" 5 | 6 | if defined?(::Shoryuken) 7 | RSpec.describe Kiev::Shoryuken do 8 | # { 9 | # message_id: "437eb14c-7a98-457c-a431-01671564237e", 10 | # body: "test", 11 | # message_attributes: message_attributes, 12 | # delete: nil 13 | # } 14 | # { 15 | # "request_id" => "acc6acfe-525e-49a6-a12b-ce3f07564620", 16 | # "request_depth" => 1, 17 | # "tree_path" => "B" 18 | # } 19 | 20 | before(:all) do 21 | # Shoryuken skips configure_server blocks unless it's running in a worker 22 | # process. "worker process"ness is defined only by existence of this module. 23 | # In Shoryuken itself it's only defined in `bin` and only gets created when 24 | # using `shoryuken` executable. It's not (readily) available for requiring. 25 | module Shoryuken::CLI 26 | end 27 | 28 | Shoryuken.logger.level = Logger::FATAL 29 | Kiev::Shoryuken.enable 30 | Kiev::Test::Log.configure 31 | end 32 | 33 | describe "client middleware when sending a message" do 34 | include Kiev::RequestStore::Mixin 35 | 36 | let(:credentials) { Aws::Credentials.new("access_key_id", "secret_access_key") } 37 | let(:sqs) { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) } 38 | let(:queue_name) { "default" } 39 | let(:queue_url) { "https://sqs.eu-west-1.amazonaws.com:6059/0123456789/#{queue_name}" } 40 | 41 | let(:queue) { Shoryuken::Queue.new(sqs, queue_name) } 42 | before do 43 | allow(queue).to receive(:url).and_return(queue_url) 44 | allow(sqs).to receive(:send_message) 45 | end 46 | 47 | around(:each) do |example| 48 | wrap_request_store { example.run } 49 | end 50 | 51 | let(:message_attributes) { {} } 52 | let(:formatted_attributes) do 53 | next unless message_attributes 54 | message_attributes.each_with_object({}) do |(k, v), attrs| 55 | attrs[k.to_s] = { data_type: "String", string_value: v } 56 | end 57 | end 58 | 59 | def send_message 60 | message = { message_body: '{"a": 42}' } 61 | if message_attributes 62 | message[:message_attributes] = formatted_attributes 63 | end 64 | queue.send_message(message) 65 | end 66 | 67 | context "without anything in Kiev store" do 68 | before { send_message } 69 | 70 | it "tags it with tree_path B (first request, asynchronous)" do 71 | expect(sqs) 72 | .to have_received(:send_message) 73 | .with(hash_including(message_attributes: { "tree_path" => { data_type: "String", string_value: "B" } })) 74 | end 75 | end 76 | 77 | context "with a specified request_id in Kiev store" do 78 | let(:request_id) { "acc6acfe-525e-49a6-a12b-ce3f07564620" } 79 | 80 | before do 81 | Kiev::RequestStore.store[:request_id] = request_id 82 | Kiev::RequestStore.store[:tracking_id] = request_id 83 | end 84 | 85 | it "preserves request_id" do 86 | expect(sqs) 87 | .to receive(:send_message) 88 | .with( 89 | hash_including( 90 | message_attributes: hash_including( 91 | "tracking_id" => { data_type: "String", string_value: request_id }, 92 | "request_id" => { data_type: "String", string_value: request_id } 93 | ) 94 | ) 95 | ) 96 | send_message 97 | end 98 | end 99 | 100 | context "with a specified tree_path in Kiev store" do 101 | let(:source_tree_path) { "DAaaAAaAAAAAaaAAaaA" } 102 | 103 | before do 104 | Kiev::RequestStore.store[:tree_path] = source_tree_path 105 | end 106 | 107 | it "appends B to tree_path" do 108 | expect(sqs) 109 | .to receive(:send_message) 110 | .with( 111 | hash_including( 112 | message_attributes: hash_including( 113 | "tree_path" => { data_type: "String", string_value: (source_tree_path + "B") } 114 | ) 115 | ) 116 | ) 117 | send_message 118 | end 119 | end 120 | 121 | context "with a specified request depth in Kiev store" do 122 | let(:request_depth) { 5 } 123 | 124 | before do 125 | Kiev::RequestStore.store[:request_depth] = request_depth 126 | end 127 | 128 | it "preserves it as a string" do 129 | expect(sqs) 130 | .to receive(:send_message) 131 | .with( 132 | hash_including( 133 | message_attributes: hash_including( 134 | "request_depth" => { data_type: "String", string_value: request_depth.to_s } 135 | ) 136 | ) 137 | ) 138 | send_message 139 | end 140 | end 141 | end 142 | 143 | describe "server middleware" do 144 | context "when receiving a message" do 145 | let(:queue) { "default" } 146 | let(:sqs_queue) { double(Shoryuken::Queue, visibility_timeout: 30) } 147 | 148 | let(:processor) { Shoryuken::Processor.new(queue, sqs_msg) } 149 | 150 | let(:message_body) { "test" } 151 | let(:message_attributes) { {} } 152 | 153 | let(:formatted_attributes) do 154 | message_attributes.each_with_object({}) do |(k, v), attrs| 155 | attrs[k.to_s] = double( 156 | Aws::SQS::Types::MessageAttributeValue, 157 | data_type: "String", 158 | string_value: v 159 | ) 160 | end 161 | end 162 | 163 | def sqs_msg_from(**attrs) 164 | double( 165 | Shoryuken::Message, 166 | message_id: SecureRandom.uuid, 167 | receipt_handle: SecureRandom.uuid, 168 | **attrs 169 | ) 170 | end 171 | 172 | let(:sqs_msg_attrs) do 173 | { 174 | queue_url: queue, 175 | body: message_body, 176 | message_attributes: formatted_attributes 177 | } 178 | end 179 | 180 | let(:sqs_msg) { sqs_msg_from(sqs_msg_attrs) } 181 | 182 | before do 183 | class TestWorker 184 | include Shoryuken::Worker 185 | shoryuken_options queue: "default" 186 | 187 | def perform(_sqs_msg, _body) 188 | true 189 | end 190 | end 191 | 192 | allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue) 193 | end 194 | 195 | before { Kiev::Test::Log.clear } 196 | 197 | context "without message attributes" do 198 | it { expect { processor.process }.to_not raise_error } 199 | 200 | describe "logged entry" do 201 | subject { Kiev::Test::Log.entries.first } 202 | 203 | before { processor.process } 204 | 205 | it { is_expected.to be_a(Hash) } 206 | 207 | it "has fields of a successful job" do 208 | is_expected.to include( 209 | "log_name" => "job_finished", 210 | "level" => "INFO", 211 | "body" => message_body, 212 | "tree_leaf" => true, 213 | "tree_path" => "B", 214 | "request_depth" => 0, 215 | "timestamp" => a_string_matching(/.+/), 216 | "request_id" => a_string_matching(/.+/), 217 | "tracking_id" => a_string_matching(/.+/), 218 | "request_duration" => (a_value > 0) 219 | ) 220 | end 221 | 222 | it "has no error fields" do 223 | is_expected.to_not include( 224 | "error_class", 225 | "error_message", 226 | "error_backtrace" 227 | ) 228 | end 229 | 230 | it "does not populate the store with a new request_id" do 231 | expect(Kiev::RequestStore.store).to_not have_key(:request_id) 232 | end 233 | 234 | context "when two messages are sent" do 235 | let(:other_sqs_msg) { sqs_msg_from(sqs_msg_attrs) } 236 | let(:other_processor) { Shoryuken::Processor.new(queue, other_sqs_msg) } 237 | before { other_processor.process } 238 | 239 | it "generates different request_ids" do 240 | first, second = Kiev::Test::Log.entries 241 | expect(first["request_id"]).to_not eq(second["request_id"]) 242 | expect(first["tracking_id"]).to_not eq(second["tracking_id"]) 243 | end 244 | end 245 | end 246 | end 247 | 248 | context "with message attributes" do 249 | let(:request_id) { "acc6acfe-525e-49a6-a12b-ce3f07564620" } 250 | let(:tree_path) { "AWYeAH" } 251 | let(:request_depth) { tree_path.length } 252 | 253 | let(:message_attributes) do 254 | { 255 | request_id: request_id, 256 | tracking_id: request_id, 257 | tree_path: tree_path, 258 | request_depth: request_depth 259 | } 260 | end 261 | 262 | describe "logged_entry" do 263 | before { processor.process } 264 | subject { Kiev::Test::Log.entries.first } 265 | it "processes tracing fields properly" do 266 | is_expected.to include( 267 | "request_id" => request_id, 268 | "tracking_id" => request_id, 269 | "tree_path" => tree_path, 270 | "request_depth" => (request_depth + 1) 271 | ) 272 | end 273 | end 274 | end 275 | 276 | describe "failing worker" do 277 | let(:queue) { "error" } 278 | 279 | before do 280 | class UnexpectedFailure < StandardError; end 281 | 282 | class FailingWorker 283 | include Shoryuken::Worker 284 | shoryuken_options queue: "error" 285 | 286 | def perform(_sqs_msg, _body) 287 | raise UnexpectedFailure, "error message" 288 | end 289 | end 290 | 291 | allow(Shoryuken::Client) 292 | .to receive(:queues) 293 | .with(queue) 294 | .and_return(sqs_queue) 295 | end 296 | 297 | it "doesn't rescue the exception" do 298 | expect { processor.process }.to raise_error(UnexpectedFailure) 299 | end 300 | 301 | describe "logged entry" do 302 | subject { Kiev::Test::Log.entries.first } 303 | 304 | before do 305 | begin 306 | processor.process 307 | rescue UnexpectedFailure 308 | end 309 | end 310 | 311 | it "describes the error" do 312 | is_expected.to include( 313 | "error_class" => UnexpectedFailure.to_s, 314 | "error_message" => "error message", 315 | "error_backtrace" => an_instance_of(String) 316 | ) 317 | end 318 | end 319 | end 320 | 321 | context "when tree_path suffixing is configured explicitly" do 322 | let(:queue) { "suffixed" } 323 | let(:tree_path) { "ABD" } 324 | let(:suffix) { "K" } 325 | let(:message_attributes) do 326 | { 327 | request_id: SecureRandom.uuid, 328 | request_depth: tree_path.length, 329 | tree_path: tree_path 330 | } 331 | end 332 | 333 | before do 334 | Shoryuken.configure_server do |config| 335 | Kiev::Shoryuken.suffix_tree_path(config, suffix) 336 | end 337 | class SuffixedWorker 338 | include Shoryuken::Worker 339 | shoryuken_options queue: "suffixed" 340 | 341 | def perform(_sqs_msg, _body) 342 | true 343 | end 344 | end 345 | processor.process 346 | end 347 | 348 | after do 349 | Shoryuken.configure_server do |config| 350 | config.server_middleware.remove(Kiev::Shoryuken::Middleware::TreePathSuffix) 351 | end 352 | end 353 | 354 | describe "logged entry" do 355 | subject { Kiev::Test::Log.entries.first } 356 | it "adds a configured suffix to it" do 357 | is_expected.to include("tree_path" => tree_path + suffix) 358 | end 359 | end 360 | end 361 | 362 | context "when sensitive data" do 363 | let(:message_body) { { "password" => "secret" } } 364 | 365 | subject { Kiev::Test::Log.entries.first } 366 | 367 | before { processor.process } 368 | 369 | it "filters logging data" do 370 | is_expected.to include("body" => "{\"password\":\"[FILTERED]\"}") 371 | end 372 | end 373 | end 374 | end 375 | end 376 | end 377 | -------------------------------------------------------------------------------- /spec/lib/kiev/subrequest_helper_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | describe Kiev::SubrequestHelper do 6 | before do 7 | Kiev::RequestStore.store.clear 8 | Kiev::RequestStore.store[:request_depth] = 0 9 | Kiev::RequestStore.store[:request_id] = "asdf" 10 | Kiev::RequestStore.store[:tree_path] = "A" 11 | end 12 | describe "root_path" do 13 | it { expect(described_class.root_path(synchronous: true)).to eq("A") } 14 | it { expect(described_class.root_path(synchronous: false)).to eq("B") } 15 | end 16 | describe "subrequest_path" do 17 | it { expect(described_class.subrequest_path(synchronous: true)).to eq("AA") } 18 | it { expect(described_class.subrequest_path(synchronous: false)).to eq("AB") } 19 | end 20 | context "consequent pathes are lexicographically sortable" do 21 | it do 22 | a = described_class.subrequest_path(synchronous: true) 23 | b = described_class.subrequest_path(synchronous: true) 24 | expect(a < b).to be(true) 25 | end 26 | it do 27 | a = described_class.subrequest_path(synchronous: true) 28 | b = described_class.subrequest_path(synchronous: false) 29 | expect(a < b).to be(true) 30 | end 31 | it do 32 | a = described_class.subrequest_path(synchronous: false) 33 | b = described_class.subrequest_path(synchronous: true) 34 | expect(a < b).to be(true) 35 | end 36 | it do 37 | a = described_class.subrequest_path(synchronous: false) 38 | b = described_class.subrequest_path(synchronous: false) 39 | expect(a < b).to be(true) 40 | end 41 | end 42 | describe "headers" do 43 | it do 44 | expect(described_class.headers).to eq( 45 | "X-Tree-Path" => "AA", 46 | "X-Request-Depth" => "0", 47 | "X-Request-Id" => "asdf" 48 | ) 49 | expect(described_class.headers).to eq( 50 | "X-Tree-Path" => "AC", 51 | "X-Request-Depth" => "0", 52 | "X-Request-Id" => "asdf" 53 | ) 54 | end 55 | it "supports metadata" do 56 | expect(described_class.headers(metadata: true)).to eq( 57 | request_depth: "0", 58 | request_id: "asdf", 59 | tree_path: "AA" 60 | ) 61 | expect(described_class.headers(metadata: true)).to eq( 62 | request_depth: "0", 63 | request_id: "asdf", 64 | tree_path: "AC" 65 | ) 66 | end 67 | end 68 | describe "payload" do 69 | it do 70 | expect(described_class.payload).to eq( 71 | "tree_path" => "AB", 72 | "request_depth" => 0, 73 | "request_id" => "asdf" 74 | ) 75 | expect(described_class.payload).to eq( 76 | "tree_path" => "AD", 77 | "request_depth" => 0, 78 | "request_id" => "asdf" 79 | ) 80 | end 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /spec/lib/kiev_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | describe Kiev do 6 | include KievHelper 7 | 8 | describe "logger" do 9 | it "always returns same instance" do 10 | log_1 = described_class.logger 11 | described_class.configure do |c| 12 | c.log_path = "/dev/null" 13 | end 14 | log_2 = described_class.logger 15 | expect(log_1).to eq(log_2) 16 | end 17 | end 18 | 19 | describe "payload" do 20 | it "expects Hash as argument" do 21 | expect { described_class.payload("abc") }.to raise_error(ArgumentError) 22 | end 23 | it "stores all data in RequestStore" do 24 | described_class.payload(a: 1) 25 | described_class.payload(b: 2) 26 | expect(Kiev::RequestStore.store[:payload]).to eq(a: 1, b: 2) 27 | end 28 | end 29 | 30 | describe "event" do 31 | before do 32 | enable_log_tracking 33 | reset_logs 34 | end 35 | after do 36 | disable_log_tracking 37 | end 38 | 39 | it "accepts one argument" do 40 | Kiev.event(:test_one) 41 | expect(log_first["log_name"]).to eq("test_one") 42 | end 43 | 44 | it "accepts two arguments" do 45 | Kiev.event(:test_one, data: "hello") 46 | expect(log_first["data"]).to eq("hello") 47 | end 48 | 49 | it "accepts data as string" do 50 | Kiev.event(:test_one, "hello") 51 | expect(log_first["message"]).to eq("hello") 52 | end 53 | 54 | context "when sensitive data" do 55 | let(:data) { { data: "hello" } } 56 | 57 | before { allow(Kiev::ParamFilter).to receive(:filter) } 58 | 59 | it "filters logging data" do 60 | Kiev.event(:test_one, data) 61 | expect(Kiev::ParamFilter).to have_received(:filter) 62 | .with(data, Kiev::Config.instance.filtered_params, Kiev::Config.instance.ignored_params) 63 | end 64 | end 65 | 66 | describe "event with predefined severity" do 67 | shared_examples "with log severity" do |severity| 68 | it "has log severity" do 69 | initial = described_class.log_level 70 | described_class.configure do |c| 71 | c.log_level = ::Logger::DEBUG 72 | end 73 | 74 | Kiev.public_send(severity, :test_one) 75 | expect(log_first["level"]).to eq(severity.to_s.upcase) 76 | 77 | described_class.configure do |c| 78 | c.log_level = initial 79 | end 80 | end 81 | end 82 | 83 | context "with debug severity" do 84 | include_examples "with log severity", :debug 85 | end 86 | 87 | context "with info severity" do 88 | include_examples "with log severity", :info 89 | end 90 | 91 | context "with warn severity" do 92 | include_examples "with log severity", :warn 93 | end 94 | 95 | context "with error severity" do 96 | include_examples "with log severity", :error 97 | end 98 | 99 | context "with fatal severity" do 100 | include_examples "with log severity", :fatal 101 | end 102 | end 103 | 104 | describe "log data filtering" do 105 | it "filters params by default" do 106 | Kiev.event(:test_one, { credit_card_number: "123" }) 107 | expect(log_first["credit_card_number"]).to eq("[FILTERED]") 108 | end 109 | 110 | context "when diabled for particular log level" do 111 | it "doesn't filters params" do 112 | initial = described_class.disable_filter_for_log_levels 113 | described_class.configure do |c| 114 | c.disable_filter_for_log_levels = [1] 115 | end 116 | 117 | Kiev.event(:test_one, { credit_card_number: "123" }) 118 | expect(log_first["credit_card_number"]).to eq("123") 119 | 120 | described_class.configure do |c| 121 | c.disable_filter_for_log_levels = initial 122 | end 123 | end 124 | end 125 | end 126 | end 127 | end 128 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler" 4 | require "delegate" 5 | 6 | Bundler.require :default, :development 7 | require "rack/test" if defined?(Rack) 8 | 9 | require "helpers/log_helper" 10 | 11 | if defined?(::Que::Job) 12 | require "helpers/que_helper" 13 | end 14 | -------------------------------------------------------------------------------- /test/data/test.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis euismod, neque eget lobortis pulvinar, leo odio pharetra metus, ac egestas odio elit sit amet quam. Nullam tempus mauris ut placerat pulvinar. Nunc id pellentesque turpis. In egestas turpis eget turpis accumsan dapibus. Ut quis pretium arcu. Morbi suscipit porttitor urna, ut elementum nulla hendrerit faucibus. Aliquam sodales euismod suscipit. Proin pulvinar nunc nisl, ac fermentum lorem venenatis quis. Quisque placerat ligula lorem, in ornare sem ullamcorper vel. Nam ut libero vel felis rutrum tempor. Sed a lacus iaculis purus tempor ultrices. In porttitor hendrerit volutpat. Duis dignissim tristique ligula quis tristique. Etiam pharetra dolor auctor justo fermentum, sed ultricies enim pellentesque. Sed sed mattis lorem, id commodo enim. 2 | 3 | Aliquam dignissim leo vitae purus sodales iaculis. Suspendisse diam metus, varius id quam eget, sagittis accumsan risus. Sed ac magna eu massa pretium auctor quis at enim. Nunc eu justo vel sapien hendrerit ultrices nec sit amet neque. Cras molestie semper sapien, vitae venenatis justo aliquet vel. Donec lacinia nisl felis, sed elementum velit accumsan id. Maecenas ornare facilisis euismod. Proin laoreet, lectus non cursus aliquam, nibh lacus tincidunt nunc, vitae sagittis purus lectus eu dui. Aliquam laoreet ullamcorper vehicula. Ut vel porta nulla, id suscipit mi. 4 | 5 | Maecenas vulputate magna libero, eget finibus massa pellentesque non. Aenean volutpat sem id nunc pharetra, non mattis ex commodo. Duis sed ligula tortor. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Vestibulum urna ligula, ullamcorper et lectus eget, luctus porta elit. Nulla vitae ipsum cursus, suscipit nisl consectetur, luctus est. In tincidunt justo at eros venenatis maximus. Nam sem libero, tristique ut erat non, molestie aliquam justo. In consequat ante ut purus volutpat, ut molestie eros tincidunt. 6 | 7 | Cras lobortis ligula vitae turpis maximus, vitae consequat mauris malesuada. Quisque sagittis tempus magna, quis euismod nisl feugiat in. Donec quis leo in felis accumsan condimentum nec id ipsum. Duis malesuada quis purus ut consectetur. Aliquam erat volutpat. Cras et lacus sollicitudin, mattis nulla eu, vestibulum risus. Nulla eu mi a erat maximus ultrices sit amet fringilla risus. Maecenas ac varius ligula, quis fringilla diam. Etiam malesuada aliquet eros, sed blandit lorem tempor nec. Nullam ac porttitor urna. Ut eleifend nisl ac vehicula molestie. Vestibulum felis enim, ultrices a pellentesque eget, fringilla a est. Duis mi nisl, rutrum sollicitudin lectus vitae, luctus venenatis sapien. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. 8 | 9 | Nulla massa enim, dignissim in vehicula id, fringilla quis metus. Nam eu lorem risus. Mauris eu orci et urna varius tincidunt. Aliquam eu justo at mauris sagittis consequat a vel sem. Nam finibus magna a elit tempor aliquam eget vel augue. Curabitur consectetur malesuada eros, nec pretium sem sagittis eget. Nam euismod faucibus mattis. Nullam vel purus et velit fringilla interdum. Vivamus semper ipsum velit, id molestie libero iaculis in. Sed dapibus libero sit amet interdum tincidunt. Morbi fringilla ornare vehicula. Donec cursus lacus id eros efficitur molestie. 10 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler" 4 | require "delegate" 5 | 6 | Bundler.require :default, :development 7 | 8 | require "rack/test" 9 | require "minitest" 10 | require "minitest/autorun" 11 | require "minitest/reporters" 12 | 13 | reporter_options = { color: true } 14 | Minitest::Reporters.use!([Minitest::Reporters::DefaultReporter.new(reporter_options)]) 15 | 16 | LOG_IO = StringIO.new 17 | ROOT_FOLDER = File.expand_path(File.dirname(__FILE__)).to_s.freeze 18 | DATA_FOLDER = "#{ROOT_FOLDER}/data" 19 | 20 | require "json" 21 | module LogHelper 22 | def setup 23 | reset_logs 24 | super 25 | end 26 | 27 | def reset_logs 28 | LOG_IO.rewind 29 | LOG_IO.truncate(0) 30 | @logs = nil 31 | end 32 | 33 | def logs 34 | return @logs if @logs 35 | LOG_IO.rewind 36 | raw_logs = LOG_IO.read 37 | begin 38 | @logs = raw_logs.split("\n").map(&JSON.method(:parse)) 39 | rescue 40 | puts raw_logs 41 | raise 42 | end 43 | end 44 | 45 | def log_first 46 | logs.first 47 | end 48 | 49 | def log_last 50 | logs.last 51 | end 52 | end 53 | 54 | class KievIgnoredException < StandardError; end 55 | 56 | Kiev.configure do |c| 57 | c.log_path = LOG_IO 58 | c.log_request_body_condition = proc do |request, _response| 59 | !!(request.content_type =~ /(application|text)\/(xml|json)/) 60 | end 61 | c.propagated_fields = { 62 | special_field: "Special-Field" 63 | } 64 | c.ignored_rack_exceptions << "KievIgnoredException" 65 | end 66 | 67 | if defined?(Combustion) 68 | require "rails/test_help" 69 | # Rails.env = "production" 70 | Combustion.path = "test/rails_app" 71 | Combustion.initialize!(:action_controller, :active_record) do 72 | config.action_dispatch.show_exceptions = false 73 | config.consider_all_requests_local = false 74 | config.active_support.test_order = :random 75 | if ActionDispatch.const_defined?(:ParamsParser) 76 | # middleware to parse XML request body 77 | config.middleware.swap( 78 | ActionDispatch::ParamsParser, ActionDispatch::ParamsParser, 79 | Mime::XML => proc do |raw_post| 80 | Hash.from_xml(raw_post) 81 | end 82 | ) 83 | else 84 | original_parsers = ActionDispatch::Request.parameter_parsers 85 | xml_parser = -> (raw_post) { Hash.from_xml(raw_post) || {} } 86 | new_parsers = original_parsers.merge(xml: xml_parser) 87 | ActionDispatch::Request.parameter_parsers = new_parsers 88 | end 89 | end 90 | end 91 | 92 | if defined?(Sinatra) 93 | require "sinatra_app/test_app" 94 | end 95 | 96 | if defined?(Sidekiq) 97 | $TESTING = true 98 | require "sidekiq/processor" 99 | 100 | Sidekiq.logger.level = Logger::ERROR 101 | 102 | REDIS_URL = ENV["REDIS_URL"] || "redis://localhost/15" 103 | REDIS = Sidekiq::RedisConnection.create(url: REDIS_URL) 104 | 105 | Kiev::Sidekiq.enable 106 | Sidekiq.configure_client do |config| 107 | config.redis = { url: REDIS_URL } 108 | # this is required because we do not run server in the tests 109 | # so we configure server_middleware for **client** 110 | Kiev::Sidekiq.enable_server_middleware(config) 111 | end 112 | end 113 | 114 | begin 115 | require "faraday" 116 | rescue LoadError 117 | puts "No Faraday" 118 | end 119 | -------------------------------------------------------------------------------- /test/her_ext_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "helper" 4 | 5 | class HerExtTest < MiniTest::Test 6 | if defined?(Faraday) 7 | def conn 8 | Faraday::Connection.new("http://example.net/") do |builder| 9 | builder.use Kiev::HerExt::ClientRequestId 10 | builder.adapter(:test) do |stub| 11 | stub.get("/") do |env| 12 | [200, {}, env[:request_headers]["X-Request-Id"]] 13 | end 14 | end 15 | end 16 | end 17 | 18 | def test_middleware_adds_request_id 19 | Kiev::RequestStore.store[:request_id] = "test" 20 | response = conn.get("/") 21 | assert_equal("test", response.body) 22 | end 23 | end 24 | 25 | if defined?(Her) 26 | Her::API.setup(url: "https://api.example.com") do |builder| 27 | # Request 28 | builder.use Kiev::HerExt::ClientRequestId 29 | # Response 30 | builder.use Her::Middleware::DefaultParseJSON 31 | # Stub connection 32 | builder.adapter(:test) do |stub| 33 | stub.get("/users/1") do |env| 34 | [200, {}, { name: env[:request_headers]["X-Request-Id"] }.to_json] 35 | end 36 | end 37 | end 38 | 39 | class ::User 40 | include ::Her::Model 41 | end 42 | 43 | def test_her_passes_request_id 44 | Kiev::RequestStore.store[:request_id] = "test1" 45 | assert_equal("test1", User.find(1).name) 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /test/rails_app/app/controllers/admin/root_controller.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Admin 4 | class RootController < ActionController::Base 5 | def get_by_id 6 | respond_to do |format| 7 | format.html { render html: "body" } 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /test/rails_app/app/controllers/root_controller.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class RootController < ActionController::Base 4 | def show 5 | respond_to do |format| 6 | format.html { render html: "body" } 7 | format.json { render json: "{\"body\":true}" } 8 | format.xml { render xml: "true" } 9 | end 10 | end 11 | 12 | def post_file 13 | respond_to do |format| 14 | format.html do 15 | params[:file].read 16 | params[:file].close 17 | render text: "body" 18 | end 19 | end 20 | end 21 | 22 | def log_in_action 23 | respond_to do |format| 24 | Rails.logger.info("test") 25 | format.html { render html: "body" } 26 | end 27 | end 28 | 29 | def request_data 30 | respond_to do |format| 31 | Kiev.payload( 32 | a: 0.0 / 0, 33 | b: BigDecimal("1"), 34 | c: "test", 35 | "c" => "c", 36 | d: User.new(id: 100, name: "Joe"), 37 | e: -3.14, 38 | f: true, 39 | j: false 40 | ) 41 | format.html { render html: "body" } 42 | end 43 | end 44 | 45 | def raise_exception 46 | raise RuntimeError, "Error" 47 | end 48 | 49 | def record_not_found 50 | raise ActiveRecord::RecordNotFound if defined?(ActiveRecord) 51 | end 52 | 53 | def get_by_id 54 | respond_to do |format| 55 | format.html { render html: "body" } 56 | end 57 | end 58 | 59 | def test_event 60 | Kiev.event(:test_event, some_data: User.new(id: 1000, name: "Jane", money: BigDecimal("1") / 3)) 61 | respond_to do |format| 62 | format.html { render html: "body" } 63 | end 64 | end 65 | 66 | def exception_as_control_flow 67 | raise KievIgnoredException, "exception message" 68 | end 69 | 70 | # You should be careful about rescue_from 71 | # Order matters. More generic errors should go first 72 | # This handler also will catch ActiveRecord::RecordNotFound and others 73 | def error_generic(exception) 74 | # if you are using generic error handler you must pass error to Kiev explicitly 75 | Kiev.error = exception 76 | # using this to show propper error code for ActiveRecord::RecordNotFound 77 | # but text in case of ActiveRecord::RecordNotFound will be wrong 78 | render( 79 | status: ::ActionDispatch::ExceptionWrapper.status_code_for_exception(exception.class.name), 80 | plain: "Internal server error" 81 | ) 82 | end 83 | rescue_from StandardError, with: :error_generic 84 | 85 | # https://apidock.com/rails/ActiveSupport/Rescuable/ClassMethods/rescue_from 86 | rescue_from("KievIgnoredException") do |exception| 87 | render plain: exception.message, status: 403 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /test/rails_app/app/models/user.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class User < ActiveRecord::Base 4 | end 5 | -------------------------------------------------------------------------------- /test/rails_app/config/database.yml: -------------------------------------------------------------------------------- 1 | test: 2 | adapter: sqlite3 3 | database: db/combustion_test.sqlite 4 | 5 | production: 6 | adapter: sqlite3 7 | database: db/combustion_production.sqlite 8 | -------------------------------------------------------------------------------- /test/rails_app/config/routes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Rails.application.routes.draw do 4 | namespace(:admin) do 5 | get("get_by_id/:id" => "root#get_by_id") 6 | post("get_by_id/:id" => "root#get_by_id") 7 | end 8 | 9 | get("/" => "root#show") 10 | post("/" => "root#show") 11 | post("post_file" => "root#post_file") 12 | get("log_in_action" => "root#log_in_action") 13 | get("request_data" => "root#request_data") 14 | get("raise_exception" => "root#raise_exception") 15 | get("record_not_found" => "root#record_not_found") 16 | get("get_by_id/:id" => "root#get_by_id") 17 | post("get_by_id/:id" => "root#get_by_id") 18 | get("test_event" => "root#test_event") 19 | get("exception_as_control_flow" => "root#exception_as_control_flow") 20 | end 21 | -------------------------------------------------------------------------------- /test/rails_app/db/.gitignore: -------------------------------------------------------------------------------- 1 | *.sqlite 2 | -------------------------------------------------------------------------------- /test/rails_app/db/schema.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | ActiveRecord::Schema.define do 4 | create_table :users do |t| 5 | t.string :name 6 | t.decimal :money 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /test/rails_app/log/.gitignore: -------------------------------------------------------------------------------- 1 | *.log -------------------------------------------------------------------------------- /test/rails_app/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blacklane/kiev/edda94115bdacd1cfc06df1720fce0575d534ff7/test/rails_app/public/favicon.ico -------------------------------------------------------------------------------- /test/rails_integration_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "helper" 4 | 5 | if defined?(Rails) 6 | class RailsIntegrationTest < ActionDispatch::IntegrationTest 7 | include LogHelper 8 | 9 | def test_simple_get 10 | get("/") 11 | assert_equal("GET", log_first["verb"]) 12 | assert_equal("/", log_first["path"]) 13 | assert_equal(200, log_first["status"]) 14 | assert_equal("www.example.com", log_first["host"]) 15 | assert_equal("request_finished", log_first["log_name"]) 16 | assert_equal("INFO", log_first["level"]) 17 | assert_equal("127.0.0.1", log_first["ip"]) 18 | assert_equal("root#show", log_first["route"]) 19 | assert_equal(true, log_first["tree_leaf"]) 20 | assert_equal("A", log_first["tree_path"]) 21 | refute_empty(log_first["timestamp"]) 22 | refute_empty(log_first["request_id"]) 23 | refute_nil(log_first["request_duration"]) 24 | end 25 | 26 | def test_x_request_id 27 | get( 28 | "/", 29 | params: {}, 30 | headers: { 31 | "X-Request-Id" => "external-uu-rid", 32 | "X-Tracking-Id" => "external-uu-rid", 33 | "X-Request-Depth" => "0", 34 | "X-Tree-Path" => "AA" 35 | } 36 | ) 37 | assert_equal("external-uu-rid", log_first["request_id"]) 38 | assert_equal("external-uu-rid", log_first["tracking_id"]) 39 | assert_equal("AA", log_first["tree_path"]) 40 | assert_equal(1, log_first["request_depth"]) 41 | end 42 | 43 | def test_special_field 44 | post("/", params: "", headers: { "Special-Field" => "special" }) 45 | assert_equal("special", log_first["special_field"]) 46 | end 47 | 48 | def test_get_with_params 49 | get("/", params: { some_data: "abc", password: "secret", utf8: "1" }) 50 | assert_equal("{\"some_data\":\"abc\",\"password\":\"[FILTERED]\"}", log_first["params"]) 51 | end 52 | 53 | def test_post_with_params 54 | upload = Rack::Test::UploadedFile.new("#{DATA_FOLDER}/test.txt", "image/jpeg") 55 | post("/post_file", params: { some_data: "abc", "file" => upload }) 56 | assert_not_nil(log_first) 57 | assert_equal( 58 | "{\"some_data\":\"abc\",\"file\":{\"original_filename\":\"test.txt\",\"content_type\":\"image/jpeg\","\ 59 | "\"headers\":\"content-disposition: form-data; name=\\\"file\\\"; filename=\\\"test.txt\\\"\\r\\n"\ 60 | "content-type: image/jpeg\\r\\ncontent-length: 3308\\r\\n\"}}", 61 | log_first["params"] 62 | ) 63 | end 64 | 65 | def test_log 66 | get("/log_in_action") 67 | assert_equal("log", log_first["log_name"]) 68 | assert_equal("INFO", log_first["level"]) 69 | refute_empty(log_first["request_id"]) 70 | assert_nil(log_first["route"]) 71 | assert_not_nil(log_last) 72 | assert_equal("root#log_in_action", log_last["route"]) 73 | end 74 | 75 | def test_data 76 | get("/request_data") 77 | assert_nil(log_first["a"]) 78 | assert_equal("1.0", log_first["b"]) 79 | assert_equal("c", log_first["c"]) 80 | assert_equal("{\"id\":100,\"name\":\"Joe\",\"money\":null}", log_first["d"]) 81 | assert_equal(-3.14, log_first["e"]) 82 | assert_equal(true, log_first["f"]) 83 | assert_equal(false, log_first["j"]) 84 | end 85 | 86 | def test_exception 87 | begin 88 | get("/raise_exception") 89 | assert_equal(500, status) 90 | rescue 91 | # in case of action_dispatch.show_exceptions = false 92 | end 93 | assert_equal("root#raise_exception", log_first["route"]) 94 | assert_match(/RuntimeError/, log_first["error_class"]) 95 | assert_match(/Error/, log_first["error_message"]) 96 | refute_empty(log_first["error_backtrace"]) 97 | refute_empty(log_first["request_id"]) 98 | assert_equal("request_finished", log_first["log_name"]) 99 | assert_equal("ERROR", log_first["level"]) 100 | assert_equal(500, log_first["status"]) 101 | end 102 | 103 | if defined?(ActiveRecord) 104 | def test_record_not_found 105 | begin 106 | get("/record_not_found") 107 | assert_equal(404, status) 108 | rescue 109 | # in case of action_dispatch.show_exceptions = false 110 | end 111 | assert_equal("root#record_not_found", log_first["route"]) 112 | assert_nil(log_first["error_class"]) 113 | refute_empty(log_first["request_id"]) 114 | assert_equal("request_finished", log_first["log_name"]) 115 | assert_equal("ERROR", log_first["level"]) 116 | assert_equal(404, log_first["status"]) 117 | end 118 | end 119 | 120 | def test_cexception_as_control_flow 121 | status = get("/exception_as_control_flow") 122 | assert_equal(403, status) 123 | assert_nil(log_first["error_class"]) 124 | assert_nil(log_first["error_message"]) 125 | assert_nil(log_first["error_backtrace"]) 126 | refute_empty(log_first["request_id"]) 127 | assert_equal("request_finished", log_first["log_name"]) 128 | assert_equal("ERROR", log_first["level"]) 129 | end 130 | 131 | def test_json_post 132 | json = "{\"some_data\": \"abc\", \"password\": \"secret\", \"utf8\": \"1\"}" 133 | post("/", params: json, headers: { "CONTENT_TYPE" => "application/json" }) 134 | assert_equal("{\"some_data\":\"abc\",\"password\":\"[FILTERED]\"}", log_first["params"]) 135 | end 136 | 137 | def test_malformed_json_post 138 | begin 139 | malformed_json = "{\"some_data\": \"abc\", \"password\": \"secret\", \"utf8\": " 140 | status = post("/", params: malformed_json, headers: { "CONTENT_TYPE" => "application/json" }) 141 | assert_equal(400, status) 142 | rescue 143 | # in case of action_dispatch.show_exceptions = false 144 | end 145 | assert_nil(log_first["params"]) 146 | assert_equal(400, log_first["status"]) 147 | end 148 | 149 | def test_xml_post 150 | post("/", params: "bc1", headers: { "CONTENT_TYPE" => "application/xml" }) 151 | assert_equal("{\"xml\":{\"a\":\"b\",\"password\":\"[FILTERED]\"}}", log_first["params"]) 152 | end 153 | 154 | def test_malformed_xml_post 155 | begin 156 | status = post("/", params: "not xml", headers: { "CONTENT_TYPE" => "application/xml" }) 157 | assert_equal(400, status) 158 | rescue 159 | # in case of action_dispatch.show_exceptions = false 160 | end 161 | assert_nil(log_first["params"]) 162 | assert_equal(400, log_first["status"]) 163 | end 164 | 165 | def test_route 166 | get("/get_by_id/123") 167 | assert_equal("root#get_by_id", log_first["route"]) 168 | assert_equal("/get_by_id/123", log_first["path"]) 169 | reset_logs 170 | get("/get_by_id/234") 171 | assert_equal("root#get_by_id", log_last["route"]) 172 | assert_equal("/get_by_id/234", log_last["path"]) 173 | reset_logs 174 | post("/get_by_id/345") 175 | assert_equal("root#get_by_id", log_last["route"]) 176 | assert_equal("/get_by_id/345", log_last["path"]) 177 | reset_logs 178 | get("/admin/get_by_id/123") 179 | assert_equal("admin/root#get_by_id", log_first["route"]) 180 | assert_equal("/admin/get_by_id/123", log_first["path"]) 181 | reset_logs 182 | get("/admin/get_by_id/234") 183 | assert_equal("admin/root#get_by_id", log_last["route"]) 184 | assert_equal("/admin/get_by_id/234", log_last["path"]) 185 | reset_logs 186 | post("/admin/get_by_id/345") 187 | assert_equal("admin/root#get_by_id", log_last["route"]) 188 | assert_equal("/admin/get_by_id/345", log_last["path"]) 189 | end 190 | 191 | def test_event 192 | get("/test_event") 193 | assert_equal("{\"id\":1000,\"name\":\"Jane\",\"money\":\"0.333333333333333333\"}", log_first["some_data"]) 194 | assert_equal("test_event", log_first["log_name"]) 195 | end 196 | end 197 | end 198 | -------------------------------------------------------------------------------- /test/sidekiq_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "helper" 4 | 5 | if defined?(Sidekiq) 6 | class SidekiqTest < MiniTest::Spec 7 | include LogHelper 8 | 9 | class CustomWorker 10 | include Sidekiq::Worker 11 | def perform(_args) 12 | 1 13 | end 14 | end 15 | 16 | class SubWorker 17 | include Sidekiq::Worker 18 | def perform(_args) 19 | CustomWorker.perform_async("test") 20 | 2 21 | end 22 | end 23 | 24 | class ErrorWorker 25 | include Sidekiq::Worker 26 | def perform(_args) 27 | CustomWorker.undefined_method 28 | end 29 | end 30 | 31 | before do 32 | Sidekiq.redis = REDIS 33 | Sidekiq.redis(&:flushdb) 34 | Kiev::RequestStore.store.clear 35 | end 36 | 37 | def run_sidekiq(opts = {}) 38 | msg = Sidekiq.dump_json({ "class" => CustomWorker.to_s, "args" => ["test"] }.merge(opts)) 39 | boss = Minitest::Mock.new 40 | boss.expect(:options, { queues: ["default"] }, []) 41 | boss.expect(:options, { queues: ["default"] }, []) 42 | processor = Sidekiq::Processor.new(boss) 43 | boss.expect(:processor_done, nil, [processor]) 44 | processor.process(Sidekiq::BasicFetch::UnitOfWork.new("queue:default", msg)) 45 | rescue NoMethodError 46 | # do nothing, for CustomWorker.undefined_method 47 | end 48 | 49 | it "server middleware logs successful job" do 50 | run_sidekiq 51 | assert_equal("SidekiqTest::CustomWorker", log_first["job_name"]) 52 | assert_equal("job_finished", log_first["log_name"]) 53 | assert_equal("INFO", log_first["level"]) 54 | assert_equal("[\"test\"]", log_first["params"]) 55 | assert_equal(true, log_first["tree_leaf"]) 56 | assert_equal("B", log_first["tree_path"]) 57 | refute_empty(log_first["timestamp"]) 58 | refute_empty(log_first["request_id"]) 59 | refute_empty(log_first["tracking_id"]) 60 | refute_nil(log_first["request_duration"]) 61 | assert_equal(0, log_first["request_depth"]) 62 | assert_nil(log_first["error_class"]) 63 | assert_nil(log_first["error_message"]) 64 | assert_nil(log_first["error_backtrace"]) 65 | assert_nil(Kiev::RequestStore.store[:request_id]) 66 | assert_nil(Kiev::RequestStore.store[:tracking_id]) 67 | end 68 | 69 | it "server middleware logs error job" do 70 | run_sidekiq("class" => ErrorWorker.to_s) 71 | assert_equal("SidekiqTest::ErrorWorker", log_first["job_name"]) 72 | assert_equal("job_finished", log_first["log_name"]) 73 | assert_equal("ERROR", log_first["level"]) 74 | assert_equal("[\"test\"]", log_first["params"]) 75 | refute_empty(log_first["timestamp"]) 76 | refute_empty(log_first["request_id"]) 77 | refute_empty(log_first["tracking_id"]) 78 | refute_nil(log_first["request_duration"]) 79 | assert_equal(0, log_first["request_depth"]) 80 | assert_equal("NoMethodError", log_first["error_class"]) 81 | assert_equal( 82 | "undefined method `undefined_method' for SidekiqTest::CustomWorker:Class", 83 | log_first["error_message"].lines.first.chomp 84 | ) 85 | refute_nil(log_first["error_backtrace"]) 86 | assert_nil(Kiev::RequestStore.store[:request_id]) 87 | end 88 | 89 | it "server middleware preserves existing request_id" do 90 | run_sidekiq("request_id" => "test") 91 | assert_equal("test", log_first["request_id"]) 92 | assert_equal("test", log_first["tracking_id"]) 93 | assert_nil(Kiev::RequestStore.store[:request_id]) 94 | assert_nil(Kiev::RequestStore.store[:tracking_id]) 95 | end 96 | 97 | it "server middleware provides a value if existing request_id is an empty string" do 98 | run_sidekiq("request_id" => "") 99 | refute_empty(log_first["request_id"]) 100 | refute_empty(log_first["tracking_id"]) 101 | assert_nil(Kiev::RequestStore.store[:request_id]) 102 | assert_nil(Kiev::RequestStore.store[:tracking_id]) 103 | end 104 | 105 | it "server middleware generates new request_id each time" do 106 | run_sidekiq 107 | run_sidekiq 108 | refute_equal(log_last["request_id"], log_first["request_id"]) 109 | refute_equal(log_last["tracking_id"], log_first["tracking_id"]) 110 | end 111 | 112 | it "server job propagates request_id to underlying job" do 113 | queue = Sidekiq.redis { |r| r.lrange("queue:default", 0, -1) }.map(&JSON.method(:parse)) 114 | assert_equal(queue.length, 0) 115 | run_sidekiq("class" => SubWorker.to_s) 116 | queue = Sidekiq.redis { |r| r.lrange("queue:default", 0, -1) }.map(&JSON.method(:parse)) 117 | assert_equal(queue.length, 1) 118 | assert_equal(queue.first["class"], "SidekiqTest::CustomWorker") 119 | assert_equal(queue.first["request_id"], log_first["request_id"]) 120 | assert_equal(queue.first["tracking_id"], log_first["tracking_id"]) 121 | end 122 | 123 | it "client middleware stores request_id in job" do 124 | Kiev::RequestStore.store[:request_id] = "test" 125 | mw = Kiev::Sidekiq::ClientRequestId.new 126 | 127 | msg = {} 128 | mw.call(nil, msg, nil, nil) {} 129 | assert_equal("test", msg["request_id"]) 130 | 131 | msg = { "request_id" => "not_test" } 132 | mw.call(nil, msg, nil, nil) {} 133 | assert_equal("test", msg["request_id"]) 134 | end 135 | 136 | it "client middleware stores tree_path in job" do 137 | Kiev::RequestStore.store[:tree_path] = "Q" 138 | mw = Kiev::Sidekiq::ClientRequestId.new 139 | 140 | msg = {} 141 | mw.call(nil, msg, nil, nil) {} 142 | assert_equal("QB", msg["tree_path"]) 143 | 144 | msg = { "tree_path" => "not_test" } 145 | mw.call(nil, msg, nil, nil) {} 146 | assert_equal("QD", msg["tree_path"]) 147 | end 148 | end 149 | end 150 | -------------------------------------------------------------------------------- /test/sinatra_app/test_app.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "xmlsimple" 4 | require "sinatra/namespace" 5 | 6 | class TestApp < Sinatra::Base 7 | include Kiev::Rack 8 | 9 | # Enable error pages that show backtrace and environment information when an unhandled exception occurs. 10 | # Enabled in development environments by default. 11 | disable :show_exceptions 12 | # log exception backtraces to STDERR 13 | disable :dump_errors 14 | # allow exceptions to propagate outside of the app 15 | disable :raise_errors 16 | 17 | use Rack::Parser, parsers: { 18 | "application/json" => proc { |data| JSON.parse(data) }, 19 | "application/xml" => proc { |data| XmlSimple.xml_in(data) } 20 | } 21 | 22 | get("/") { "Hello World" } 23 | post("/") { "Hello World" } 24 | 25 | post("/post_file") do 26 | params[:file][:tempfile].read 27 | params[:file][:tempfile].close 28 | "Hello World" 29 | end 30 | 31 | get("/log_in_action") do 32 | Kiev.logger.info("test") 33 | "body" 34 | end 35 | 36 | get("/request_data") do 37 | Kiev.payload(a: 0.0 / 0, b: BigDecimal("1"), c: "test", "c" => "c") 38 | end 39 | 40 | get("/raise_exception_handled") { raise RuntimeError, "Error" } 41 | 42 | # you should not rescue from Exception here, 43 | # unless you are using Puma in cluster mode 44 | # otherwise you will shadow posix signals 45 | error(StandardError) do |_exception| 46 | status(500) 47 | "

Internal\ Server\ Error<\/h1>" 48 | end 49 | 50 | error(RuntimeError) do |_exception| 51 | status(502) 52 | "internal server error" 53 | end 54 | 55 | get("/raise_exception_unhandled") { raise StandardError, "Error" } 56 | 57 | get("/exception_as_control_flow") { raise KievIgnoredException, "exception message" } 58 | 59 | error(KievIgnoredException) do |exception| 60 | status(403) 61 | exception.message 62 | end 63 | 64 | UUID_PARAM = "(?([a-z0-9]){8}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){12})" 65 | 66 | register Sinatra::Namespace 67 | namespace("/admin") do 68 | get(%r{/resource/#{UUID_PARAM}/test}) { "body" } 69 | end 70 | 71 | get(%r{/resource/#{UUID_PARAM}/test}) { "body" } 72 | 73 | get("/test_halt") { halt(400, { "Content-Type" => "plain/text" }, "halt response") } 74 | 75 | get("/test_custom_halt") { halt(999) } 76 | 77 | error(999) do 78 | status(401) 79 | "custom halt" 80 | end 81 | end 82 | 83 | class TestApp2 < Sinatra::Base 84 | include Kiev::Rack 85 | 86 | # Enable error pages that show backtrace and environment information when an unhandled exception occurs. 87 | # Enabled in development environments by default. 88 | enable :show_exceptions 89 | # log exception backtraces to STDERR 90 | enable :dump_errors 91 | # allow exceptions to propagate outside of the app 92 | enable :raise_errors 93 | 94 | get("/raise_exception_handled") { raise RuntimeError, "Error" } 95 | 96 | error(RuntimeError) do |_exception| 97 | status(502) 98 | "internal server error" 99 | end 100 | 101 | get("/raise_exception_unhandled") { raise StandardError, "Error" } 102 | end 103 | -------------------------------------------------------------------------------- /test/sinatra_integration_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "helper" 4 | 5 | if defined?(Sinatra) 6 | class SinatraIntegrationTest < MiniTest::Test 7 | include Rack::Test::Methods 8 | include LogHelper 9 | 10 | def app 11 | TestApp 12 | end 13 | 14 | def test_simple_get 15 | get("/") 16 | assert_equal("GET", log_first["verb"]) 17 | assert_equal("/", log_first["path"]) 18 | assert_equal(200, log_first["status"]) 19 | assert_equal("example.org", log_first["host"]) 20 | assert_equal("request_finished", log_first["log_name"]) 21 | assert_equal("INFO", log_first["level"]) 22 | assert_equal("127.0.0.1", log_first["ip"]) 23 | assert_equal("GET /", log_first["route"]) 24 | assert_equal(true, log_first["tree_leaf"]) 25 | assert_equal("A", log_first["tree_path"]) 26 | refute_empty(log_first["timestamp"]) 27 | refute_empty(log_first["request_id"]) 28 | refute_nil(log_first["request_duration"]) 29 | end 30 | 31 | def test_x_request_id 32 | post( 33 | "/", 34 | "", 35 | "HTTP_X_REQUEST_ID" => "external-uu-rid", 36 | "HTTP_X_REQUEST_DEPTH" => "0", 37 | "HTTP_X_TREE_PATH" => "AA" 38 | ) 39 | assert_equal("external-uu-rid", log_first["request_id"]) 40 | assert_equal("AA", log_first["tree_path"]) 41 | assert_equal(1, log_first["request_depth"]) 42 | end 43 | 44 | def test_special_field 45 | post("/", "", "HTTP_SPECIAL_FIELD" => "special") 46 | assert_equal("special", log_first["special_field"]) 47 | end 48 | 49 | def test_get_with_params 50 | get("/", some_data: "abc", password: "secret", utf8: "1") 51 | assert_equal("{\"some_data\":\"abc\",\"password\":\"[FILTERED]\"}", log_first["params"]) 52 | end 53 | 54 | def test_post_with_params 55 | file = Rack::Test::UploadedFile.new("#{DATA_FOLDER}/test.txt", "image/jpeg") 56 | post("/post_file", some_data: "abc", "file" => file) 57 | assert_equal( 58 | "{\"some_data\":\"abc\",\"file\":{\"filename\":\"test.txt\",\"type\":\"image/jpeg\",\"name\":\"file\"," \ 59 | "\"head\":\"content-disposition: form-data; name=\\\"file\\\"; filename=\\\"test.txt\\\"\\r\\n" \ 60 | "content-type: image/jpeg\\r\\ncontent-length: 3308\\r\\n\"}}", 61 | log_first["params"] 62 | ) 63 | end 64 | 65 | def test_log 66 | get("/log_in_action") 67 | assert_equal("log", log_first["log_name"]) 68 | assert_equal("INFO", log_first["level"]) 69 | refute_empty(log_first["request_id"]) 70 | end 71 | 72 | def test_data 73 | get("/request_data") 74 | assert_nil(log_first["a"]) 75 | assert_equal("0.1e1", log_first["b"]) # WTF? 76 | assert_equal("c", log_first["c"]) 77 | end 78 | 79 | def test_exception_handled 80 | response = get("/raise_exception_handled") 81 | assert_match("internal server error", response.body) 82 | assert_equal(response.headers["X-Request-Id"], log_first["request_id"]) 83 | assert_match(/RuntimeError/, log_first["error_class"]) 84 | assert_match(/Error/, log_first["error_message"]) 85 | refute_empty(log_first["error_backtrace"]) 86 | refute_empty(log_first["request_id"]) 87 | assert_equal("request_finished", log_first["log_name"]) 88 | assert_equal("ERROR", log_first["level"]) 89 | assert_equal(502, log_first["status"]) 90 | end 91 | 92 | def test_exception_unhandled 93 | response = get("/raise_exception_unhandled") 94 | assert_match("

Internal Server Error

", response.body) 95 | assert_equal(response.headers["X-Request-Id"], log_first["request_id"]) 96 | assert_match(/StandardError/, log_first["error_class"]) 97 | assert_match(/Error/, log_first["error_message"]) 98 | refute_empty(log_first["error_backtrace"]) 99 | refute_empty(log_first["request_id"]) 100 | assert_equal("request_finished", log_first["log_name"]) 101 | assert_equal("ERROR", log_first["level"]) 102 | assert_equal(500, log_first["status"]) 103 | end 104 | 105 | def test_cexception_as_control_flow 106 | response = get("/exception_as_control_flow") 107 | assert_match("exception message", response.body) 108 | assert_equal(response.headers["X-Request-Id"], log_first["request_id"]) 109 | assert_nil(log_first["error_class"]) 110 | assert_nil(log_first["error_message"]) 111 | assert_nil(log_first["error_backtrace"]) 112 | refute_empty(log_first["request_id"]) 113 | assert_equal("request_finished", log_first["log_name"]) 114 | assert_equal("ERROR", log_first["level"]) 115 | assert_equal(403, log_first["status"]) 116 | end 117 | 118 | def test_json_post 119 | data = "{\"some_data\": \"abc\", \"password\": \"secret\", \"utf8\": \"1\"}" 120 | post("/", data, "CONTENT_TYPE" => "application/json") 121 | assert_equal("{\"some_data\":\"abc\",\"password\":\"[FILTERED]\"}", log_first["params"]) 122 | end 123 | 124 | def test_json_post_body 125 | data = "{\"some_data\": \"abc\", \"password\": \"secret\", \"utf8\": \"1\"}" 126 | post("/", data, "CONTENT_TYPE" => "text/json") 127 | assert_equal("{\"some_data\":\"abc\",\"password\":\"[FILTERED]\"}", log_first["request_body"]) 128 | end 129 | 130 | def test_malformed_json_post 131 | data = "{\"some_data\": \"abc\", \"password\": \"secret\", \"utf8\": \"1}" 132 | post("/", data, "CONTENT_TYPE" => "application/json") 133 | assert_nil(log_first["params"]) 134 | assert_equal(400, last_response.status) 135 | end 136 | 137 | def test_malformed_json_post_log 138 | post("/", "{\"some_data\": \"abc\", \"password\": \"secret\", \"utf8\": \"1}", "CONTENT_TYPE" => "text/json") 139 | assert_nil(log_first["params"]) 140 | assert_equal("{\"some_data\": \"abc\", \"password\": \"secret\", \"utf8\": \"1}", log_first["request_body"]) 141 | end 142 | 143 | def test_xml_post 144 | post("/", "bc1", "CONTENT_TYPE" => "application/xml") 145 | assert_equal("{\"a\":[\"b\"],\"password\":\"[FILTERED]\"}", log_first["params"]) 146 | end 147 | 148 | def test_malformed_xml_post_body 149 | post("/", "bc1", "CONTENT_TYPE" => "application/xml") 150 | assert_nil(log_first["params"]) 151 | assert_equal(400, last_response.status) 152 | end 153 | 154 | def test_malformed_xml_post_content_type 155 | post("/", "b", "CONTENT_TYPE" => nil) 156 | # This happens due to wrong CONTENT_TYPE 157 | # This case is not handled by Kiev gem, instead it should be handled by application: 158 | # - reject all requests without CONTENT_TYPE 159 | # - treat all requests as CONTENT_TYPE=application/xml 160 | expected = "{\"\\u003cxml\\u003e\\u003ca d\":\"d\\u003eb\\u003c/a\\u003e\\u003c/xml\\u003e\"}" 161 | assert_equal(expected, log_first["params"]) 162 | assert_equal(200, last_response.status) 163 | end 164 | 165 | def test_xml_post_body_log 166 | post("/", "bc1", "CONTENT_TYPE" => "text/xml") 167 | assert_nil(log_first["params"]) 168 | assert_equal("b[FILTERED]1", log_first["request_body"]) 169 | end 170 | 171 | def test_route 172 | get("/resource/d5588e5e-8360-4214-8c81-1c60212e7e97/test") 173 | if Sinatra::VERSION == "2.0.0" 174 | assert_equal( 175 | "GET \\/resource\\/" \ 176 | "(?([a-z0-9]){8}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){12})\\/test", 177 | log_first["route"] 178 | ) 179 | else 180 | assert_equal( 181 | "GET (?-mix:\\/resource\\/" \ 182 | "(?([a-z0-9]){8}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){12})\\/test)", 183 | log_first["route"] 184 | ) 185 | end 186 | end 187 | 188 | def test_route_with_namespace 189 | get("/admin/resource/d5588e5e-8360-4214-8c81-1c60212e7e97/test") 190 | if Sinatra::VERSION == "2.0.0" 191 | assert_equal( 192 | "GET (sinatra:\"/admin\" + regular:\"\\\\/resource\\\\/" \ 193 | "(?([a-z0-9]){8}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){12})\\\\/test\")", 194 | log_first["route"] 195 | ) 196 | else 197 | assert_equal( 198 | "GET (?-mix:^(?-mix:(?-mix:\\/admin)(?-mix:\\/resource\\/" \ 199 | "(?([a-z0-9]){8}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){4}-([a-z0-9]){12})\\/test))$)", 200 | log_first["route"] 201 | ) 202 | end 203 | end 204 | 205 | def test_halt 206 | response = get("/test_halt") 207 | assert_equal(400, log_first["status"]) 208 | assert_equal(response.headers["X-Request-Id"], log_first["request_id"]) 209 | assert_equal("halt response", response.body) 210 | assert_nil(log_first["error_class"]) 211 | end 212 | 213 | def test_custom_halt 214 | response = get("/test_custom_halt") 215 | assert_equal(401, log_first["status"]) 216 | assert_equal(response.headers["X-Request-Id"], log_first["request_id"]) 217 | assert_equal("custom halt", response.body) 218 | assert_nil(log_first["error_class"]) 219 | end 220 | end 221 | 222 | class SinatraIntegrationTest2 < MiniTest::Test 223 | include Rack::Test::Methods 224 | include LogHelper 225 | 226 | def app 227 | TestApp2 228 | end 229 | 230 | def test_exception_handled 231 | response = get("/raise_exception_handled") 232 | assert_match("internal server error", response.body) 233 | assert_nil(response.headers["X-Request-Id"]) # because of enable :show_exceptions 234 | assert_match(/RuntimeError/, log_first["error_class"]) 235 | assert_match(/Error/, log_first["error_message"]) 236 | refute_empty(log_first["error_backtrace"]) 237 | refute_empty(log_first["request_id"]) 238 | assert_equal("request_finished", log_first["log_name"]) 239 | assert_equal("ERROR", log_first["level"]) 240 | assert_equal(500, log_first["status"]) # because of enable :show_exceptions 241 | end 242 | 243 | def test_exception_unhandled 244 | begin 245 | get("/raise_exception_unhandled") 246 | rescue 247 | # because of enable :raise_errors 248 | end 249 | assert_match(/StandardError/, log_first["error_class"]) 250 | assert_match(/Error/, log_first["error_message"]) 251 | refute_empty(log_first["error_backtrace"]) 252 | refute_empty(log_first["request_id"]) 253 | assert_equal("request_finished", log_first["log_name"]) 254 | assert_equal("ERROR", log_first["level"]) 255 | assert_equal(500, log_first["status"]) 256 | end 257 | end 258 | end 259 | --------------------------------------------------------------------------------