├── .rubocop.yml ├── README.md ├── lib └── sidekiq_profiling_middleware │ ├── memory_profiler.rb │ ├── s3.rb │ ├── stack_prof.rb │ └── util.rb └── sidekiq_profiling_middleware.gemspec /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | DisabledByDefault: true 3 | 4 | Bundler/DuplicatedGem: 5 | Enabled: true 6 | 7 | Bundler/OrderedGems: 8 | Enabled: true 9 | 10 | Layout/BlockAlignment: 11 | Enabled: true 12 | 13 | Layout/BlockEndNewline: 14 | Enabled: true 15 | 16 | Layout/ConditionPosition: 17 | Enabled: true 18 | 19 | Layout/DefEndAlignment: 20 | Enabled: true 21 | 22 | Layout/EndAlignment: 23 | Enabled: false 24 | 25 | Layout/EndOfLine: 26 | Enabled: true 27 | 28 | Layout/InitialIndentation: 29 | Enabled: true 30 | 31 | Layout/SpaceAfterColon: 32 | Enabled: true 33 | 34 | Layout/SpaceAfterComma: 35 | Enabled: true 36 | 37 | Layout/SpaceAfterMethodName: 38 | Enabled: true 39 | 40 | Layout/SpaceAfterNot: 41 | Enabled: true 42 | 43 | Layout/SpaceAfterSemicolon: 44 | Enabled: true 45 | 46 | Layout/SpaceAroundBlockParameters: 47 | Enabled: true 48 | 49 | Layout/SpaceAroundEqualsInParameterDefault: 50 | Enabled: true 51 | 52 | Layout/SpaceBeforeBlockBraces: 53 | Enabled: true 54 | 55 | Layout/SpaceInsideArrayLiteralBrackets: 56 | Enabled: true 57 | EnforcedStyle: no_space 58 | 59 | Layout/SpaceInsideArrayPercentLiteral: 60 | Enabled: true 61 | 62 | Layout/SpaceInsideBlockBraces: 63 | Enabled: true 64 | 65 | Layout/SpaceInsideParens: 66 | Enabled: true 67 | 68 | Layout/SpaceInsideRangeLiteral: 69 | Enabled: true 70 | 71 | Layout/SpaceInsideReferenceBrackets: 72 | Enabled: true 73 | 74 | Layout/Tab: 75 | Enabled: true 76 | 77 | Layout/TrailingBlankLines: 78 | Enabled: true 79 | 80 | Layout/TrailingWhitespace: 81 | Enabled: true 82 | 83 | Lint/CircularArgumentReference: 84 | Enabled: true 85 | 86 | Lint/Debugger: 87 | Enabled: true 88 | 89 | Lint/DeprecatedClassMethods: 90 | Enabled: true 91 | 92 | Lint/DuplicateMethods: 93 | Enabled: true 94 | 95 | Lint/DuplicatedKey: 96 | Enabled: true 97 | 98 | Lint/EachWithObjectArgument: 99 | Enabled: true 100 | 101 | Lint/ElseLayout: 102 | Enabled: true 103 | 104 | Lint/EmptyEnsure: 105 | Enabled: true 106 | 107 | Lint/EmptyInterpolation: 108 | Enabled: true 109 | 110 | Lint/EndInMethod: 111 | Enabled: true 112 | 113 | Lint/EnsureReturn: 114 | Enabled: true 115 | 116 | Lint/FloatOutOfRange: 117 | Enabled: true 118 | 119 | Lint/FormatParameterMismatch: 120 | Enabled: true 121 | 122 | Lint/LiteralAsCondition: 123 | Enabled: true 124 | 125 | Lint/LiteralInInterpolation: 126 | Enabled: true 127 | 128 | Lint/Loop: 129 | Enabled: true 130 | 131 | Lint/NextWithoutAccumulator: 132 | Enabled: true 133 | 134 | Lint/RandOne: 135 | Enabled: true 136 | 137 | Lint/RequireParentheses: 138 | Enabled: true 139 | 140 | Lint/RescueException: 141 | Enabled: true 142 | 143 | Lint/StringConversionInInterpolation: 144 | Enabled: true 145 | 146 | Lint/UnderscorePrefixedVariableName: 147 | Enabled: true 148 | 149 | Lint/UnneededCopDisableDirective: 150 | Enabled: true 151 | 152 | Lint/UnneededSplatExpansion: 153 | Enabled: true 154 | 155 | Lint/UnreachableCode: 156 | Enabled: true 157 | 158 | Lint/UselessComparison: 159 | Enabled: true 160 | 161 | Lint/UselessSetterCall: 162 | Enabled: true 163 | 164 | Lint/Void: 165 | Enabled: true 166 | 167 | Metrics/AbcSize: 168 | Enabled: false 169 | 170 | Metrics/BlockLength: 171 | Enabled: false 172 | 173 | Metrics/BlockNesting: 174 | Enabled: false 175 | 176 | Metrics/ClassLength: 177 | Enabled: false 178 | 179 | Metrics/CyclomaticComplexity: 180 | Enabled: false 181 | 182 | Metrics/LineLength: 183 | Enabled: false 184 | 185 | Metrics/MethodLength: 186 | Enabled: false 187 | 188 | Metrics/ModuleLength: 189 | Enabled: false 190 | 191 | Metrics/ParameterLists: 192 | Enabled: false 193 | 194 | Metrics/PerceivedComplexity: 195 | Enabled: false 196 | 197 | Naming/AsciiIdentifiers: 198 | Enabled: true 199 | 200 | Naming/ClassAndModuleCamelCase: 201 | Enabled: true 202 | 203 | Naming/FileName: 204 | Enabled: true 205 | 206 | Naming/MethodName: 207 | Enabled: true 208 | 209 | Performance/CaseWhenSplat: 210 | Enabled: false 211 | 212 | Performance/Count: 213 | Enabled: true 214 | 215 | Performance/Detect: 216 | Enabled: true 217 | 218 | Performance/DoubleStartEndWith: 219 | Enabled: true 220 | 221 | Performance/EndWith: 222 | Enabled: true 223 | 224 | Performance/FlatMap: 225 | Enabled: true 226 | 227 | Performance/LstripRstrip: 228 | Enabled: true 229 | 230 | Performance/RangeInclude: 231 | Enabled: false 232 | 233 | Performance/RedundantMatch: 234 | Enabled: false 235 | 236 | Performance/RedundantMerge: 237 | Enabled: true 238 | MaxKeyValuePairs: 1 239 | 240 | Performance/RedundantSortBy: 241 | Enabled: true 242 | 243 | Performance/ReverseEach: 244 | Enabled: true 245 | 246 | Performance/Sample: 247 | Enabled: true 248 | 249 | Performance/Size: 250 | Enabled: true 251 | 252 | Performance/StartWith: 253 | Enabled: true 254 | 255 | Security/Eval: 256 | Enabled: true 257 | 258 | Style/ArrayJoin: 259 | Enabled: true 260 | 261 | Style/BeginBlock: 262 | Enabled: true 263 | 264 | Style/BlockComments: 265 | Enabled: true 266 | 267 | Style/CaseEquality: 268 | Enabled: true 269 | 270 | Style/CharacterLiteral: 271 | Enabled: true 272 | 273 | Style/ClassMethods: 274 | Enabled: true 275 | 276 | Style/Copyright: 277 | Enabled: false 278 | 279 | Style/DefWithParentheses: 280 | Enabled: true 281 | 282 | Style/EndBlock: 283 | Enabled: true 284 | 285 | Style/FlipFlop: 286 | Enabled: true 287 | 288 | Style/For: 289 | Enabled: true 290 | 291 | Style/FrozenStringLiteralComment: 292 | Enabled: true 293 | 294 | Style/HashSyntax: 295 | Enabled: true 296 | EnforcedStyle: ruby19_no_mixed_keys 297 | 298 | Style/LambdaCall: 299 | Enabled: true 300 | 301 | Style/MethodCallWithoutArgsParentheses: 302 | Enabled: true 303 | 304 | Style/MethodDefParentheses: 305 | Enabled: true 306 | 307 | Style/MultilineIfThen: 308 | Enabled: true 309 | 310 | Style/NilComparison: 311 | Enabled: true 312 | 313 | Style/Not: 314 | Enabled: true 315 | 316 | Style/OneLineConditional: 317 | Enabled: true 318 | 319 | Style/StabbyLambdaParentheses: 320 | Enabled: true 321 | 322 | Style/StringLiterals: 323 | Enabled: true 324 | EnforcedStyle: double_quotes -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sidekiq_profiling_middleware 2 | 3 | Profile Sidekiq with StackProf & MemoryProfiler with optional support for S3 exports. 4 | 5 | ## Installation 6 | 7 | Two middleware classes are available: 8 | 9 | * `SidekiqProfilingMiddleware::StackProf`: requires [stackprof](https://github.com/tmm1/stackprof) 10 | * `SidekiqProfilingMiddleware::MemoryProfile`: requires [memory_profiler](https://github.com/SamSaffron/memory_profiler) 11 | 12 | (You should only use one at a time, otherwise that will be quite confusing) 13 | 14 | ```ruby 15 | require "sidekiq_profiling_middleware/stack_prof" 16 | 17 | Sidekiq.configure_server do |config| 18 | .... 19 | config.server_middleware do |chain| 20 | chain.add( 21 | SidekiqProfilingMiddleware::StackProf, 22 | only: [ThisReallySlowWorker].to_set, 23 | s3_bucket: "cj-profiling", 24 | output_prefix: "stackprof/#{ENV["GIT_SHA]}_", 25 | ) 26 | # OR 27 | chain.add SidekiqProfilingMiddleware::StackProf, output_prefix: "tmp/#{Rails.env}_#{ENV["GIT_SHA"]}_" 28 | end 29 | end 30 | ``` 31 | 32 | ## Concurrency (set it to 1) 33 | 34 | The hooks provided by Ruby that are used by StackProf & MemoryProfiler are not able to distinguish threads well, so you will need to set your Sidekiq `concurrency` to `1`. This will ensure that only one job operates at a time ensuring other jobs don't confuse your memory profile or stacktraces. 35 | 36 | With this in mind you should set this up on a separate Sidekiq instance so as not to drop the concurrency capability of your Sidekiq cluster. 37 | 38 | ## S3 exporting 39 | 40 | If you run Sidekiq in a Dockerized environment like ECS or Kubernetes you will probably not want profiling reports dumped to a filesystem path as you don't want to bloat the container size and track down which Docker host contains the profiles. To solve this you can have profiles uploaded to S3 using the option `s3_bucket`. 41 | 42 | The AWS SDK is used so you'll want to set your AWS config/credentials using environment variables or EC2 instance profiles (preferred when running inside AWS). See [Setup Config](https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html) for more info. 43 | 44 | Alternatively you can configure the client using `SidekiqProfilingMiddleware::S3.client = .....`. 45 | -------------------------------------------------------------------------------- /lib/sidekiq_profiling_middleware/memory_profiler.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require "sidekiq_profiling_middleware/util" 3 | require "memory_profiler" 4 | 5 | module SidekiqProfilingMiddleware 6 | class MemoryProfiler 7 | def initialize(output_prefix: nil, only: nil, s3_bucket: nil, memory_profiler_options: {}) 8 | @options = memory_profiler_options 9 | 10 | @output_prefix = output_prefix || self.class.default_output_prefix 11 | @only = only 12 | @s3_bucket = s3_bucket 13 | end 14 | 15 | def call(worker, msg, queue) 16 | # bail out if whitelist doesn't match 17 | if only && !only.include?(worker.class) 18 | return yield 19 | end 20 | 21 | report = ::MemoryProfiler.report(options) do 22 | yield 23 | end 24 | 25 | out = "#{output_prefix}#{Util.worker_names[worker.class]}_#{Util.current_epoch_ms}.txt" 26 | 27 | unless s3_bucket 28 | report.pretty_print(to_file: out) 29 | return 30 | end 31 | 32 | require "sidekiq_profiling_middleware/s3" 33 | 34 | out = S3::Object.new(bucket: s3_bucket, key: out) 35 | report.pretty_print(out) 36 | ensure 37 | out.upload if out && s3_bucket 38 | end 39 | 40 | def self.default_output_prefix 41 | @default_output_prefix ||= Util.default_output_prefix("memory_profiler") 42 | end 43 | 44 | private 45 | 46 | attr_reader( 47 | :only, 48 | :options, 49 | :output_prefix, 50 | :s3_bucket, 51 | ) 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/sidekiq_profiling_middleware/s3.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require "stringio" 3 | require "aws-sdk" 4 | 5 | module SidekiqProfilingMiddleware 6 | class S3 7 | class Object < StringIO 8 | def initialize(bucket:, key:) 9 | @bucket = bucket 10 | @key = key 11 | 12 | super() 13 | end 14 | 15 | def upload 16 | rewind 17 | 18 | S3.client.put_object(bucket: bucket, key: key, body: self) 19 | end 20 | 21 | private 22 | 23 | attr_reader :bucket, :key 24 | end 25 | 26 | def self.client=(client) 27 | @client = client 28 | end 29 | 30 | def self.client 31 | @client ||= Aws::S3::Client.new 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/sidekiq_profiling_middleware/stack_prof.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require "sidekiq_profiling_middleware/util" 3 | require "stackprof" 4 | 5 | module SidekiqProfilingMiddleware 6 | class StackProf 7 | def initialize(output_prefix: nil, only: nil, s3_bucket: nil, stack_prof_options: {}) 8 | stack_prof_options[:mode] ||= :cpu 9 | stack_prof_options[:interval] ||= 1000 10 | @options = stack_prof_options 11 | 12 | @output_prefix = output_prefix || self.class.default_output_prefix 13 | @only = only 14 | @s3_bucket = s3_bucket 15 | end 16 | 17 | def call(worker, msg, queue) 18 | # bail out if whitelist doesn't match 19 | if only && !only.include?(worker.class) 20 | return yield 21 | end 22 | 23 | out = "#{output_prefix}#{Util.worker_names[worker.class]}_#{Util.current_epoch_ms}.dump" 24 | 25 | unless s3_bucket 26 | ::StackProf.run(options.merge(out: out)) { yield } 27 | return 28 | end 29 | 30 | require "sidekiq_profiling_middleware/s3" 31 | 32 | out = S3::Object.new(bucket: s3_bucket, key: out) 33 | rep = ::StackProf.run(options) { yield } 34 | Marshal.dump(rep, out) 35 | ensure 36 | out.upload if out && s3_bucket 37 | end 38 | 39 | def self.default_output_prefix 40 | @default_output_prefix ||= Util.default_output_prefix("stackprof") 41 | end 42 | 43 | private 44 | 45 | attr_reader( 46 | :only, 47 | :options, 48 | :output_prefix, 49 | :s3_bucket, 50 | ) 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/sidekiq_profiling_middleware/util.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | module SidekiqProfilingMiddleware 3 | class Util 4 | BOOTED_AT_FORMAT = "%m-%d-%H-%M-%S" 5 | 6 | def self.default_output_prefix(name) 7 | "tmp/#{name}_bootedat#{Time.now.strftime(BOOTED_AT_FORMAT)}_" 8 | end 9 | 10 | def self.worker_names 11 | # allocate hash for quickly converting class names to 12 | # nice names a file system would like 13 | @worker_names ||= Hash.new do |hash, worker_name| 14 | hash[worker_name] = worker_name.to_s.gsub(/\W+/, "_").gsub(/(^_|_$)/, "") 15 | end 16 | end 17 | 18 | def self.current_epoch_ms 19 | (Time.now.utc.to_f * 1000).to_i 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /sidekiq_profiling_middleware.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | Gem::Specification.new do |s| 3 | s.name = "sidekiq_profiling_middleware" 4 | s.version = "0.0.3" 5 | s.date = "2010-04-28" 6 | s.summary = "StackProf and MemoryProfiler middleware for Sidekiq" 7 | s.authors = ["Callum Jones"] 8 | s.email = "contact@callumj.com" 9 | s.files = %w( 10 | lib/sidekiq_profiling_middleware/memory_profiler.rb 11 | lib/sidekiq_profiling_middleware/s3.rb 12 | lib/sidekiq_profiling_middleware/stack_prof.rb 13 | lib/sidekiq_profiling_middleware/util.rb 14 | ) 15 | s.homepage = "https://github.com/callumj/sidekiq_profiling_middleware" 16 | s.license = "MIT" 17 | 18 | s.add_development_dependency "rubocop", "0.54.0" 19 | end 20 | --------------------------------------------------------------------------------