├── .env ├── .gitignore ├── .rubocop-bundler.yml ├── .rubocop.yml ├── .rubocop_todo.yml ├── .ruby-version ├── .travis.yml ├── Gemfile ├── Gemfile.lock ├── Procfile ├── README.md ├── Rakefile ├── bin ├── production ├── rake ├── rspec └── staging ├── config.ru ├── config ├── follow_redirects.vcl └── newrelic.yml ├── db └── migrations │ ├── 01_rubygems_org_schema_dump.rb │ ├── 02_prune.rb │ ├── 03_add_ruby_and_rubygems_requirements.rb │ ├── 04_add_deps_md5_to_rubygems.rb │ ├── 05_create_checksums.rb │ ├── 06_add_created_at_to_versions.rb │ ├── 07_add_checksum_to_versions.rb │ ├── 08_add_info_checksum_to_versions.rb │ ├── 09_remove_deps_md5_from_rubygems.rb │ ├── 10_add_yanked_at_to_versions.rb │ └── 11_add_yanked_info_checksum_to_versions.rb ├── lib ├── bundler_api.rb ├── bundler_api │ ├── agent_reporting.rb │ ├── cache.rb │ ├── checksum.rb │ ├── env.rb │ ├── gem_helper.rb │ ├── gem_info.rb │ ├── metriks.rb │ ├── redis.rb │ ├── runtime_instrumentation.rb │ ├── strategy.rb │ ├── update │ │ ├── atomic_counter.rb │ │ ├── consumer_pool.rb │ │ ├── fix_dep_job.rb │ │ ├── gem_db_helper.rb │ │ ├── job.rb │ │ └── yank_job.rb │ └── web.rb └── puma │ └── instrumented_cli.rb ├── public └── robots.txt ├── puma.rb ├── script ├── console ├── integration │ ├── migrate.rb │ └── test.rb ├── migrate ├── setup └── web ├── spec ├── agent_reporting_spec.rb ├── cache_spec.rb ├── gem_helper_spec.rb ├── gem_info_spec.rb ├── spec_helper.rb ├── support │ ├── artifice_apps.rb │ ├── database.rb │ ├── etag.rb │ ├── gem_builder.rb │ ├── gemspec_helper.rb │ ├── latch.rb │ └── matchers.rb ├── update │ ├── atomic_counter_spec.rb │ ├── consumer_pool_spec.rb │ ├── fix_dep_job_spec.rb │ ├── gem_db_helper_spec.rb │ ├── job_spec.rb │ └── yank_job_spec.rb └── web_spec.rb └── versions.list /.env: -------------------------------------------------------------------------------- 1 | # Default configuration. Override any of these settings by creating a 2 | # .env.local file. 3 | 4 | RACK_ENV=development 5 | DATABASE_URL=postgres:///bundler-api 6 | FOLLOWER_DATABASE_URL=postgres:///bundler-api 7 | TEST_DATABASE_ADMIN_URL=postgres:///postgres 8 | TEST_DATABASE_URL=postgres:///bundler-api-test 9 | MIN_THREADS=0 10 | MAX_THREADS=16 11 | REDIS_ENV=REDIS_URL 12 | REDIS_URL=redis://127.0.0.1:6379 13 | MEMCACHE_SERVERS=127.0.0.1:11211 14 | RUBYGEMS_URL=https://rubygems.org 15 | DOWNLOAD_BASE=https://rubygems.global.ssl.fastly.net 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **.orig 2 | *.rbc 3 | *.sassc 4 | .rspec 5 | .sass-cache/ 6 | bin/ 7 | .bundle/ 8 | coverage/ 9 | db/*.sqlite3 10 | log/ 11 | public/system/ 12 | spec/tmp/ 13 | tmp/ 14 | vendor/bundle/ 15 | capybara-*.html 16 | pickle-email-*.html 17 | rerun.txt 18 | pickle-email-*.html 19 | .env.local 20 | tags 21 | -------------------------------------------------------------------------------- /.rubocop-bundler.yml: -------------------------------------------------------------------------------- 1 | inherit_from: 2 | - .rubocop_todo.yml 3 | 4 | AllCops: 5 | TargetRubyVersion: 1.9 6 | Exclude: 7 | - tmp/**/* 8 | - lib/bundler/vendor/**/* 9 | DisplayCopNames: true 10 | 11 | # Lint 12 | 13 | # They are idiomatic 14 | Lint/AssignmentInCondition: 15 | Enabled: false 16 | 17 | Lint/EndAlignment: 18 | AlignWith: variable 19 | AutoCorrect: true 20 | 21 | Lint/UnusedMethodArgument: 22 | Enabled: false 23 | 24 | # Style 25 | 26 | Style/AccessModifierIndentation: 27 | EnforcedStyle: outdent 28 | 29 | Style/Alias: 30 | EnforcedStyle: prefer_alias_method 31 | 32 | Style/AlignParameters: 33 | EnforcedStyle: with_fixed_indentation 34 | 35 | Style/FrozenStringLiteralComment: 36 | EnforcedStyle: always 37 | 38 | Style/MultilineBlockChain: 39 | Enabled: false 40 | 41 | Style/MultilineOperationIndentation: 42 | EnforcedStyle: indented 43 | 44 | Style/PerlBackrefs: 45 | Enabled: false 46 | 47 | Style/SingleLineBlockParams: 48 | Enabled: false 49 | 50 | Style/SpaceInsideBlockBraces: 51 | SpaceBeforeBlockParameters: false 52 | 53 | Style/TrivialAccessors: 54 | Enabled: false 55 | 56 | # We adopted raise instead of fail. 57 | Style/SignalException: 58 | EnforcedStyle: only_raise 59 | 60 | Style/StringLiterals: 61 | EnforcedStyle: double_quotes 62 | 63 | Style/StringLiteralsInInterpolation: 64 | EnforcedStyle: double_quotes 65 | 66 | # Having these make it easier to *not* forget to add one when adding a new 67 | # value and you can simply copy the previous line. 68 | Style/TrailingCommaInLiteral: 69 | EnforcedStyleForMultiline: comma 70 | 71 | Style/TrailingUnderscoreVariable: 72 | Enabled: false 73 | 74 | # `String.new` is preferred style with enabled frozen string literal 75 | Style/EmptyLiteral: 76 | Enabled: false 77 | 78 | # 1.8.7 support 79 | 80 | Style/HashSyntax: 81 | EnforcedStyle: hash_rockets 82 | 83 | Style/Lambda: 84 | Enabled: false 85 | 86 | Style/DotPosition: 87 | EnforcedStyle: trailing 88 | 89 | Style/EachWithObject: 90 | Enabled: false 91 | 92 | Style/SpecialGlobalVars: 93 | Enabled: false 94 | 95 | Style/TrailingCommaInArguments: 96 | Enabled: false 97 | 98 | Performance/FlatMap: 99 | Enabled: false 100 | 101 | # Metrics 102 | 103 | # We've chosen to use Rubocop only for style, and not for complexity or quality checks. 104 | Metrics/ClassLength: 105 | Enabled: false 106 | 107 | Metrics/ModuleLength: 108 | Enabled: false 109 | 110 | Metrics/MethodLength: 111 | Enabled: false 112 | 113 | Metrics/BlockNesting: 114 | Enabled: false 115 | 116 | Metrics/AbcSize: 117 | Enabled: false 118 | 119 | Metrics/CyclomaticComplexity: 120 | Enabled: false 121 | 122 | Metrics/PerceivedComplexity: 123 | Enabled: false 124 | 125 | Metrics/ParameterLists: 126 | Enabled: false 127 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: .rubocop-bundler.yml 2 | 3 | AllCops: 4 | Exclude: 5 | - bin/**/* 6 | - vendor/**/* 7 | DisplayCopNames: true 8 | TargetRubyVersion: 2.3 9 | -------------------------------------------------------------------------------- /.rubocop_todo.yml: -------------------------------------------------------------------------------- 1 | # This configuration was generated by 2 | # `rubocop --auto-gen-config` 3 | # on 2016-02-25 17:43:22 -0600 using RuboCop version 0.37.2. 4 | # The point is for the user to remove these configuration records 5 | # one by one as the offenses are removed from the code base. 6 | # Note that changes in the inspected code, or installation of new 7 | # versions of RuboCop, may require this file to be generated again. 8 | 9 | # Offense count: 1 10 | Lint/AmbiguousOperator: 11 | Exclude: 12 | - 'script/migrate' 13 | 14 | # Offense count: 2 15 | Lint/Eval: 16 | Exclude: 17 | - 'spec/gem_helper_spec.rb' 18 | - 'spec/support/gemspec_helper.rb' 19 | 20 | # Offense count: 1 21 | Lint/HandleExceptions: 22 | Exclude: 23 | - 'Rakefile' 24 | 25 | # Offense count: 1 26 | Lint/LiteralInCondition: 27 | Exclude: 28 | - 'lib/bundler_api/runtime_instrumentation.rb' 29 | 30 | # Offense count: 7 31 | # Cop supports --auto-correct. 32 | # Configuration parameters: IgnoreEmptyBlocks. 33 | Lint/UnusedBlockArgument: 34 | Exclude: 35 | - 'Rakefile' 36 | - 'spec/support/matchers.rb' 37 | 38 | # Offense count: 21 39 | Lint/UselessAssignment: 40 | Exclude: 41 | - 'Rakefile' 42 | - 'lib/bundler_api/update/gem_db_helper.rb' 43 | - 'lib/bundler_api/update/job.rb' 44 | - 'script/integration/migrate.rb' 45 | - 'spec/gem_info_spec.rb' 46 | - 'spec/support/artifice_apps.rb' 47 | - 'spec/update/gem_db_helper_spec.rb' 48 | - 'spec/web_spec.rb' 49 | 50 | # Offense count: 95 51 | # Configuration parameters: AllowHeredoc, AllowURI, URISchemes. 52 | # URISchemes: http, https 53 | Metrics/LineLength: 54 | Max: 122 55 | 56 | # Offense count: 2 57 | Metrics/PerceivedComplexity: 58 | Max: 9 59 | 60 | # Offense count: 5 61 | Performance/TimesMap: 62 | Exclude: 63 | - 'spec/gem_helper_spec.rb' 64 | - 'spec/update/job_spec.rb' 65 | - 'spec/web_spec.rb' 66 | 67 | # Offense count: 5 68 | # Cop supports --auto-correct. 69 | # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth. 70 | # SupportedStyles: outdent, indent 71 | Style/AccessModifierIndentation: 72 | Enabled: false 73 | 74 | # Offense count: 5 75 | Style/AccessorMethodName: 76 | Exclude: 77 | - 'Rakefile' 78 | - 'lib/bundler_api/web.rb' 79 | - 'script/integration/migrate.rb' 80 | 81 | # Offense count: 4 82 | # Cop supports --auto-correct. 83 | Style/AlignArray: 84 | Exclude: 85 | - 'lib/bundler_api/agent_reporting.rb' 86 | 87 | # Offense count: 3 88 | # Cop supports --auto-correct. 89 | # Configuration parameters: EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle, SupportedLastArgumentHashStyles. 90 | # SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit 91 | Style/AlignHash: 92 | Exclude: 93 | - 'lib/bundler_api/web.rb' 94 | - 'spec/web_spec.rb' 95 | 96 | # Offense count: 2 97 | # Cop supports --auto-correct. 98 | # Configuration parameters: EnforcedStyle, SupportedStyles. 99 | # SupportedStyles: with_first_parameter, with_fixed_indentation 100 | Style/AlignParameters: 101 | Exclude: 102 | - 'lib/bundler_api/web.rb' 103 | 104 | # Offense count: 1 105 | # Cop supports --auto-correct. 106 | # Configuration parameters: EnforcedStyle, SupportedStyles. 107 | # SupportedStyles: always, conditionals 108 | Style/AndOr: 109 | Exclude: 110 | - 'lib/bundler_api/env.rb' 111 | 112 | # Offense count: 8 113 | # Cop supports --auto-correct. 114 | # Configuration parameters: EnforcedStyle, SupportedStyles, ProceduralMethods, FunctionalMethods, IgnoredMethods. 115 | # SupportedStyles: line_count_based, semantic, braces_for_chaining 116 | # ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object 117 | # FunctionalMethods: let, let!, subject, watch 118 | # IgnoredMethods: lambda, proc, it 119 | Style/BlockDelimiters: 120 | Exclude: 121 | - 'lib/bundler_api/update/consumer_pool.rb' 122 | - 'spec/gem_helper_spec.rb' 123 | - 'spec/update/gem_db_helper_spec.rb' 124 | - 'spec/update/yank_job_spec.rb' 125 | - 'spec/web_spec.rb' 126 | 127 | # Offense count: 2 128 | # Cop supports --auto-correct. 129 | Style/BlockEndNewline: 130 | Exclude: 131 | - 'spec/web_spec.rb' 132 | 133 | # Offense count: 4 134 | # Cop supports --auto-correct. 135 | # Configuration parameters: EnforcedStyle, SupportedStyles. 136 | # SupportedStyles: braces, no_braces, context_dependent 137 | Style/BracesAroundHashParameters: 138 | Exclude: 139 | - 'lib/bundler_api/cache.rb' 140 | - 'lib/bundler_api/web.rb' 141 | - 'spec/gem_info_spec.rb' 142 | - 'spec/update/yank_job_spec.rb' 143 | 144 | # Offense count: 13 145 | # Configuration parameters: EnforcedStyle, SupportedStyles. 146 | # SupportedStyles: nested, compact 147 | Style/ClassAndModuleChildren: 148 | Exclude: 149 | - 'lib/bundler_api/agent_reporting.rb' 150 | - 'lib/bundler_api/checksum.rb' 151 | - 'lib/bundler_api/gem_helper.rb' 152 | - 'lib/bundler_api/gem_info.rb' 153 | - 'lib/bundler_api/runtime_instrumentation.rb' 154 | - 'lib/bundler_api/update/fix_dep_job.rb' 155 | - 'lib/bundler_api/update/gem_db_helper.rb' 156 | - 'lib/bundler_api/update/job.rb' 157 | - 'lib/bundler_api/update/yank_job.rb' 158 | - 'lib/bundler_api/web.rb' 159 | - 'lib/puma/instrumented_cli.rb' 160 | 161 | # Offense count: 8 162 | Style/ClassVars: 163 | Exclude: 164 | - 'lib/bundler_api/update/job.rb' 165 | - 'spec/support/artifice_apps.rb' 166 | - 'spec/update/consumer_pool_spec.rb' 167 | 168 | # Offense count: 20 169 | Style/Documentation: 170 | Enabled: false 171 | 172 | # Offense count: 19 173 | # Cop supports --auto-correct. 174 | # Configuration parameters: EnforcedStyle, SupportedStyles. 175 | # SupportedStyles: leading, trailing 176 | Style/DotPosition: 177 | Enabled: false 178 | 179 | # Offense count: 2 180 | # Cop supports --auto-correct. 181 | # Configuration parameters: AllowAdjacentOneLineDefs. 182 | Style/EmptyLineBetweenDefs: 183 | Exclude: 184 | - 'lib/bundler_api/cache.rb' 185 | - 'spec/agent_reporting_spec.rb' 186 | 187 | # Offense count: 1 188 | # Cop supports --auto-correct. 189 | Style/EmptyLines: 190 | Exclude: 191 | - 'spec/web_spec.rb' 192 | 193 | # Offense count: 4 194 | # Cop supports --auto-correct. 195 | Style/EmptyLinesAroundAccessModifier: 196 | Exclude: 197 | - 'lib/bundler_api/update/consumer_pool.rb' 198 | - 'lib/bundler_api/update/gem_db_helper.rb' 199 | - 'lib/bundler_api/update/job.rb' 200 | - 'spec/support/gemspec_helper.rb' 201 | 202 | # Offense count: 1 203 | # Cop supports --auto-correct. 204 | # Configuration parameters: EnforcedStyle, SupportedStyles. 205 | # SupportedStyles: empty_lines, no_empty_lines 206 | Style/EmptyLinesAroundBlockBody: 207 | Exclude: 208 | - 'Rakefile' 209 | 210 | # Offense count: 7 211 | # Cop supports --auto-correct. 212 | # Configuration parameters: EnforcedStyle, SupportedStyles. 213 | # SupportedStyles: empty_lines, no_empty_lines 214 | Style/EmptyLinesAroundClassBody: 215 | Exclude: 216 | - 'lib/bundler_api/cache.rb' 217 | - 'lib/bundler_api/checksum.rb' 218 | - 'lib/bundler_api/strategy.rb' 219 | - 'lib/bundler_api/update/atomic_counter.rb' 220 | - 'lib/bundler_api/update/gem_db_helper.rb' 221 | - 'lib/bundler_api/web.rb' 222 | 223 | # Offense count: 59 224 | # Cop supports --auto-correct. 225 | # Configuration parameters: EnforcedStyle, SupportedStyles. 226 | # SupportedStyles: when_needed, always 227 | Style/FrozenStringLiteralComment: 228 | Enabled: false 229 | 230 | # Offense count: 19 231 | # Configuration parameters: AllowedVariables. 232 | Style/GlobalVars: 233 | Exclude: 234 | - 'lib/bundler_api/appsignal.rb' 235 | - 'spec/gem_info_spec.rb' 236 | - 'spec/support/database.rb' 237 | - 'spec/update/fix_dep_job_spec.rb' 238 | - 'spec/update/gem_db_helper_spec.rb' 239 | - 'spec/update/job_spec.rb' 240 | - 'spec/web_spec.rb' 241 | 242 | # Offense count: 3 243 | # Configuration parameters: MinBodyLength. 244 | Style/GuardClause: 245 | Exclude: 246 | - 'lib/bundler_api/checksum.rb' 247 | - 'lib/bundler_api/update/gem_db_helper.rb' 248 | 249 | # Offense count: 273 250 | # Cop supports --auto-correct. 251 | # Configuration parameters: EnforcedStyle, SupportedStyles, UseHashRocketsWithSymbolValues. 252 | # SupportedStyles: ruby19, ruby19_no_mixed_keys, hash_rockets 253 | Style/HashSyntax: 254 | Enabled: false 255 | 256 | # Offense count: 4 257 | # Cop supports --auto-correct. 258 | # Configuration parameters: MaxLineLength. 259 | Style/IfUnlessModifier: 260 | Exclude: 261 | - 'lib/bundler_api/agent_reporting.rb' 262 | - 'lib/bundler_api/update/gem_db_helper.rb' 263 | 264 | # Offense count: 2 265 | # Cop supports --auto-correct. 266 | # Configuration parameters: SupportedStyles, IndentationWidth. 267 | # SupportedStyles: special_inside_parentheses, consistent, align_brackets 268 | Style/IndentArray: 269 | EnforcedStyle: consistent 270 | 271 | # Offense count: 2 272 | # Cop supports --auto-correct. 273 | # Configuration parameters: Width. 274 | Style/IndentationWidth: 275 | Exclude: 276 | - 'lib/bundler_api/web.rb' 277 | - 'spec/support/gemspec_helper.rb' 278 | 279 | # Offense count: 1 280 | # Cop supports --auto-correct. 281 | Style/InfiniteLoop: 282 | Exclude: 283 | - 'lib/bundler_api/runtime_instrumentation.rb' 284 | 285 | # Offense count: 1 286 | # Cop supports --auto-correct. 287 | Style/LeadingCommentSpace: 288 | Exclude: 289 | - 'script/integration/test.rb' 290 | 291 | # Offense count: 8 292 | # Cop supports --auto-correct. 293 | Style/LineEndConcatenation: 294 | Exclude: 295 | - 'lib/bundler_api/update/job.rb' 296 | - 'spec/web_spec.rb' 297 | 298 | # Offense count: 1 299 | # Cop supports --auto-correct. 300 | Style/MethodCallParentheses: 301 | Exclude: 302 | - 'spec/update/job_spec.rb' 303 | 304 | # Offense count: 2 305 | # Cop supports --auto-correct. 306 | Style/MultilineBlockLayout: 307 | Exclude: 308 | - 'spec/web_spec.rb' 309 | 310 | # Offense count: 12 311 | # Cop supports --auto-correct. 312 | # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth. 313 | # SupportedStyles: aligned, indented 314 | Style/MultilineMethodCallIndentation: 315 | Enabled: false 316 | 317 | # Offense count: 1 318 | # Cop supports --auto-correct. 319 | # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth. 320 | # SupportedStyles: aligned, indented 321 | Style/MultilineOperationIndentation: 322 | Enabled: false 323 | 324 | # Offense count: 1 325 | # Cop supports --auto-correct. 326 | Style/MutableConstant: 327 | Exclude: 328 | - 'lib/bundler_api/gem_info.rb' 329 | 330 | # Offense count: 3 331 | # Cop supports --auto-correct. 332 | # Configuration parameters: EnforcedStyle, MinBodyLength, SupportedStyles. 333 | # SupportedStyles: skip_modifier_ifs, always 334 | Style/Next: 335 | Exclude: 336 | - 'lib/bundler_api/update/gem_db_helper.rb' 337 | - 'script/integration/migrate.rb' 338 | - 'script/integration/test.rb' 339 | 340 | # Offense count: 2 341 | # Cop supports --auto-correct. 342 | Style/NumericLiterals: 343 | MinDigits: 9 344 | 345 | # Offense count: 3 346 | # Cop supports --auto-correct. 347 | # Configuration parameters: PreferredDelimiters. 348 | Style/PercentLiteralDelimiters: 349 | Exclude: 350 | - 'script/migrate' 351 | - 'script/setup' 352 | 353 | # Offense count: 1 354 | # Cop supports --auto-correct. 355 | # Configuration parameters: AllowAsExpressionSeparator. 356 | Style/Semicolon: 357 | Exclude: 358 | - 'spec/agent_reporting_spec.rb' 359 | 360 | # Offense count: 1 361 | # Cop supports --auto-correct. 362 | # Configuration parameters: EnforcedStyle, SupportedStyles. 363 | # SupportedStyles: only_raise, only_fail, semantic 364 | Style/SignalException: 365 | Exclude: 366 | - 'spec/support/database.rb' 367 | 368 | # Offense count: 2 369 | # Cop supports --auto-correct. 370 | # Configuration parameters: AllowIfMethodIsEmpty. 371 | Style/SingleLineMethods: 372 | Exclude: 373 | - 'spec/agent_reporting_spec.rb' 374 | 375 | # Offense count: 10 376 | # Cop supports --auto-correct. 377 | Style/SpaceAfterComma: 378 | Exclude: 379 | - 'lib/bundler_api/gem_info.rb' 380 | - 'script/integration/migrate.rb' 381 | - 'spec/gem_info_spec.rb' 382 | - 'spec/update/gem_db_helper_spec.rb' 383 | - 'spec/web_spec.rb' 384 | 385 | # Offense count: 64 386 | # Cop supports --auto-correct. 387 | # Configuration parameters: AllowForAlignment. 388 | Style/SpaceAroundOperators: 389 | Exclude: 390 | - 'Rakefile' 391 | - 'db/migrations/01_rubygems_org_schema_dump.rb' 392 | - 'db/migrations/02_prune.rb' 393 | - 'db/migrations/03_add_ruby_and_rubygems_requirements.rb' 394 | - 'lib/bundler_api/gem_helper.rb' 395 | - 'spec/gem_info_spec.rb' 396 | - 'spec/web_spec.rb' 397 | 398 | # Offense count: 3 399 | # Cop supports --auto-correct. 400 | # Configuration parameters: EnforcedStyle, SupportedStyles. 401 | # SupportedStyles: space, no_space 402 | Style/SpaceBeforeBlockBraces: 403 | Enabled: false 404 | 405 | # Offense count: 11 406 | # Cop supports --auto-correct. 407 | # Configuration parameters: EnforcedStyle, SupportedStyles, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters. 408 | # SupportedStyles: space, no_space 409 | Style/SpaceInsideBlockBraces: 410 | Enabled: false 411 | 412 | # Offense count: 5 413 | # Cop supports --auto-correct. 414 | Style/SpaceInsideBrackets: 415 | Exclude: 416 | - 'lib/bundler_api/agent_reporting.rb' 417 | - 'spec/agent_reporting_spec.rb' 418 | 419 | # Offense count: 10 420 | # Cop supports --auto-correct. 421 | # Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SupportedStyles. 422 | # SupportedStyles: space, no_space 423 | Style/SpaceInsideHashLiteralBraces: 424 | Enabled: false 425 | 426 | # Offense count: 41 427 | # Cop supports --auto-correct. 428 | Style/SpaceInsideParens: 429 | Exclude: 430 | - 'spec/agent_reporting_spec.rb' 431 | - 'spec/web_spec.rb' 432 | 433 | # Offense count: 12 434 | # Cop supports --auto-correct. 435 | # Configuration parameters: EnforcedStyle, SupportedStyles. 436 | # SupportedStyles: space, no_space 437 | Style/SpaceInsideStringInterpolation: 438 | Exclude: 439 | - 'lib/bundler_api/agent_reporting.rb' 440 | - 'spec/support/matchers.rb' 441 | - 'spec/web_spec.rb' 442 | 443 | # Offense count: 643 444 | # Cop supports --auto-correct. 445 | # Configuration parameters: EnforcedStyle, SupportedStyles, ConsistentQuotesInMultiline. 446 | # SupportedStyles: single_quotes, double_quotes 447 | Style/StringLiterals: 448 | Enabled: false 449 | 450 | # Offense count: 5 451 | # Cop supports --auto-correct. 452 | # Configuration parameters: EnforcedStyle, SupportedStyles. 453 | # SupportedStyles: single_quotes, double_quotes 454 | Style/StringLiteralsInInterpolation: 455 | Enabled: false 456 | 457 | # Offense count: 1 458 | Style/StructInheritance: 459 | Exclude: 460 | - 'lib/bundler_api/gem_helper.rb' 461 | 462 | # Offense count: 2 463 | # Cop supports --auto-correct. 464 | # Configuration parameters: IgnoredMethods. 465 | # IgnoredMethods: respond_to 466 | Style/SymbolProc: 467 | Exclude: 468 | - 'lib/bundler_api/update/consumer_pool.rb' 469 | - 'script/setup' 470 | 471 | # Offense count: 3 472 | # Cop supports --auto-correct. 473 | # Configuration parameters: EnforcedStyle, SupportedStyles. 474 | # SupportedStyles: final_newline, final_blank_line 475 | Style/TrailingBlankLines: 476 | Exclude: 477 | - 'lib/bundler_api/metriks.rb' 478 | - 'lib/bundler_api/redis.rb' 479 | - 'script/console' 480 | 481 | # Offense count: 14 482 | # Cop supports --auto-correct. 483 | # Configuration parameters: EnforcedStyleForMultiline, SupportedStyles. 484 | # SupportedStyles: comma, consistent_comma, no_comma 485 | Style/TrailingCommaInLiteral: 486 | Exclude: 487 | - 'lib/bundler_api/cache.rb' 488 | - 'lib/bundler_api/gem_info.rb' 489 | - 'lib/bundler_api/web.rb' 490 | - 'spec/gem_info_spec.rb' 491 | - 'spec/support/gem_builder.rb' 492 | - 'spec/update/yank_job_spec.rb' 493 | - 'spec/web_spec.rb' 494 | 495 | # Offense count: 7 496 | # Cop supports --auto-correct. 497 | Style/TrailingWhitespace: 498 | Exclude: 499 | - 'db/migrations/01_rubygems_org_schema_dump.rb' 500 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.3.1 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | 3 | services: memcached 4 | addons: 5 | postgresql: "9.3" 6 | 7 | rvm: 2.3.1 8 | 9 | bundler_args: --binstubs --deployment 10 | 11 | sudo: false 12 | 13 | cache: bundler 14 | 15 | before_script: 16 | - psql -c 'create database "bundler-api";' -U postgres 17 | - ./script/setup --verbose 18 | 19 | script: 20 | - bundle exec rake spec 21 | 22 | notifications: 23 | slack: 24 | secure: iIhKCxayvPZbgASh9iQsPgUo3T/bMGDC8gJd4EO+LsIrzCVHHRV8/lx+0BisW4RaxYKFo1m2QOzJhSKYv/s4XpfZo7iyM8igRS6Ex+K7ZZtSSAmdq9Da7+45NE/q1eTILMcSeNmbfiuj8tX1nrU+G+w/+N6VcL6MoncxCJi+EAs= 25 | 26 | deploy: 27 | provider: heroku 28 | api_key: 29 | secure: CqqIlZu3vkiQoVgrvwwYQPJt/od8CeuicY8BzWF3fN2dIb7hAdvJTr/0neeve3FFoboWPezrylrzMlp8bXxKYI7zM67Q2RitoU72tmrKJNV27Si1L7hVh1eTLVCdYOJCFcxE1sntEBkiXhz6qxK2C/XS/1TwfT5NEsMywqXWl+8= 30 | app: bundler-api-staging 31 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | # source "https://bundler-api-staging.herokuapp.com" 3 | 4 | ruby File.read(File.expand_path('../.ruby-version', __FILE__)).strip 5 | 6 | git_source(:github) do |repo| 7 | repo = "#{repo}/#{repo}" unless repo.include?("/") 8 | "https://github.com/#{repo}.git" 9 | end 10 | 11 | gem 'librato-metrics' 12 | gem 'metriks-librato_metrics', github: 'indirect/metriks-librato_metrics' 13 | gem 'metriks-middleware' 14 | gem 'pg' 15 | gem 'puma' 16 | gem 'puma_worker_killer' 17 | gem 'rack-timeout', '0.4.0.beta.1', github: 'indirect/rack-timeout' 18 | gem 'rake' 19 | gem 'dalli' 20 | gem 'redis' 21 | gem 'sequel' 22 | gem 'sequel_pg', require: false 23 | gem 'sinatra' 24 | gem 'json' 25 | gem 'compact_index' 26 | gem 'newrelic_rpm' 27 | 28 | group :development do 29 | gem 'pry-byebug' 30 | end 31 | 32 | group :test do 33 | gem 'artifice' 34 | gem 'rack-test' 35 | gem 'rspec-core' 36 | gem 'rspec-expectations' 37 | gem 'rspec-mocks' 38 | end 39 | 40 | group :development, :test do 41 | gem 'foreman' 42 | gem 'dotenv', require: false 43 | gem 'rubocop', require: false 44 | end 45 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GIT 2 | remote: https://github.com/indirect/metriks-librato_metrics.git 3 | revision: 59d2aad7e15b577f7313288a29da5307160cc1be 4 | specs: 5 | metriks-librato_metrics (1.0.3) 6 | metriks (>= 0.9.9.6) 7 | 8 | GIT 9 | remote: https://github.com/indirect/rack-timeout.git 10 | revision: 96c78a76bb6f1b45990e40ccd26df652e6202db8 11 | specs: 12 | rack-timeout (0.4.0.beta.1) 13 | 14 | GEM 15 | remote: https://rubygems.org/ 16 | specs: 17 | aggregate (0.2.2) 18 | artifice (0.6) 19 | rack-test 20 | ast (2.2.0) 21 | atomic (1.1.99) 22 | avl_tree (1.2.1) 23 | atomic (~> 1.1) 24 | byebug (8.2.1) 25 | coderay (1.1.0) 26 | compact_index (0.10.0) 27 | dalli (2.7.6) 28 | diff-lcs (1.2.5) 29 | dotenv (2.0.2) 30 | faraday (0.9.2) 31 | multipart-post (>= 1.2, < 3) 32 | foreman (0.78.0) 33 | thor (~> 0.19.1) 34 | get_process_mem (0.2.0) 35 | hitimes (1.2.3) 36 | json (1.8.3) 37 | librato-metrics (1.6.0) 38 | aggregate (~> 0.2.2) 39 | faraday (~> 0.7) 40 | multi_json 41 | method_source (0.8.2) 42 | metriks (0.9.9.7) 43 | atomic (~> 1.0) 44 | avl_tree (~> 1.2.0) 45 | hitimes (~> 1.1) 46 | metriks-middleware (2.1.1) 47 | metriks (~> 0.9.9) 48 | multi_json (1.11.2) 49 | multipart-post (2.0.0) 50 | newrelic_rpm (3.15.0.314) 51 | parser (2.3.0.6) 52 | ast (~> 2.2) 53 | pg (0.18.4) 54 | powerpack (0.1.1) 55 | pry (0.10.3) 56 | coderay (~> 1.1.0) 57 | method_source (~> 0.8.1) 58 | slop (~> 3.4) 59 | pry-byebug (3.3.0) 60 | byebug (~> 8.0) 61 | pry (~> 0.10) 62 | puma (3.0.0) 63 | puma_worker_killer (0.0.5) 64 | get_process_mem (~> 0.2) 65 | puma (>= 2.7, < 4) 66 | rack (1.6.4) 67 | rack-protection (1.5.3) 68 | rack 69 | rack-test (0.6.3) 70 | rack (>= 1.0) 71 | rainbow (2.1.0) 72 | rake (10.4.2) 73 | redis (3.2.2) 74 | rspec-core (3.4.3) 75 | rspec-support (~> 3.4.0) 76 | rspec-expectations (3.4.0) 77 | diff-lcs (>= 1.2.0, < 2.0) 78 | rspec-support (~> 3.4.0) 79 | rspec-mocks (3.4.1) 80 | diff-lcs (>= 1.2.0, < 2.0) 81 | rspec-support (~> 3.4.0) 82 | rspec-support (3.4.1) 83 | rubocop (0.37.2) 84 | parser (>= 2.3.0.4, < 3.0) 85 | powerpack (~> 0.1) 86 | rainbow (>= 1.99.1, < 3.0) 87 | ruby-progressbar (~> 1.7) 88 | unicode-display_width (~> 0.3) 89 | ruby-progressbar (1.7.5) 90 | sequel (4.30.0) 91 | sequel_pg (1.6.13) 92 | pg (>= 0.8.0) 93 | sequel (>= 3.39.0) 94 | sinatra (1.4.6) 95 | rack (~> 1.4) 96 | rack-protection (~> 1.4) 97 | tilt (>= 1.3, < 3) 98 | slop (3.6.0) 99 | thor (0.19.1) 100 | tilt (2.0.2) 101 | unicode-display_width (0.3.1) 102 | 103 | PLATFORMS 104 | ruby 105 | 106 | DEPENDENCIES 107 | artifice 108 | compact_index 109 | dalli 110 | dotenv 111 | foreman 112 | json 113 | librato-metrics 114 | metriks-librato_metrics! 115 | metriks-middleware 116 | newrelic_rpm 117 | pg 118 | pry-byebug 119 | puma 120 | puma_worker_killer 121 | rack-test 122 | rack-timeout (= 0.4.0.beta.1)! 123 | rake 124 | redis 125 | rspec-core 126 | rspec-expectations 127 | rspec-mocks 128 | rubocop 129 | sequel 130 | sequel_pg 131 | sinatra 132 | 133 | RUBY VERSION 134 | ruby 2.3.1p112 135 | 136 | BUNDLED WITH 137 | 1.13.2 138 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: script/web 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # :warning: Deprecated 2 | 3 | This repository, and the web app it powered, are deprecated and no longer in use by either Bundler or RubyGems. https://github.com/rubygems/rubygems.org now powers all of the endpoints this app previously served. 4 | 5 | [![Code Climate](https://codeclimate.com/github/bundler/bundler-api/badges/gpa.svg)](https://codeclimate.com/github/bundler/bundler-api) 6 | [![Build Status](https://travis-ci.org/bundler/bundler-api.svg?branch=master)](https://travis-ci.org/bundler/bundler-api) 7 | 8 | # bundler-api 9 | 10 | ## Getting Started 11 | 12 | Run `script/setup` to create and migrate the database specified in the 13 | `$DATABASE_URL` environment variable. 14 | 15 | ## Environment 16 | 17 | The default environment is stored in `.env`. Override any of the settings 18 | found there by creating a `.env.local` file. 19 | 20 | ## Production Databases 21 | 22 | - `AMBER`: The primary database, set to `DATABASE_URL`. Writes from `web` and `update` processes go here. It is also the `FOLLOW_DATABASE_URL`, so reads come from it as well. 23 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | $: << 'lib' 2 | 3 | require 'bundler/setup' 4 | require 'bundler_api/env' 5 | 6 | require 'sequel' 7 | require 'open-uri' 8 | require 'zlib' 9 | require 'tmpdir' 10 | require 'net/http' 11 | require 'time' 12 | require 'compact_index' 13 | 14 | require 'bundler_api/cache' 15 | require 'bundler_api/update/consumer_pool' 16 | require 'bundler_api/update/job' 17 | require 'bundler_api/update/yank_job' 18 | require 'bundler_api/update/fix_dep_job' 19 | require 'bundler_api/update/atomic_counter' 20 | require 'bundler_api/gem_helper' 21 | require 'bundler_api/gem_info' 22 | 23 | $stdout.sync = true 24 | Thread.abort_on_exception = true 25 | 26 | begin 27 | require 'rspec/core/rake_task' 28 | require 'rubocop/rake_task' 29 | 30 | RuboCop::RakeTask.new 31 | 32 | desc "Run specs" 33 | RSpec::Core::RakeTask.new(:spec) do |t| 34 | t.rspec_opts = %w(--color) 35 | end 36 | task :spec => :rubocop 37 | task :default => :spec 38 | rescue LoadError => e 39 | # rspec won't exist on production 40 | end 41 | 42 | def download_index(url) 43 | uri = URI(url) 44 | 45 | Tempfile.create(uri.path) do |file| 46 | file.write(open(uri.to_s).read) 47 | file.rewind 48 | 49 | Zlib::GzipReader.open(file) do |gz| 50 | Marshal.load(gz) 51 | end 52 | end 53 | end 54 | 55 | def get_specs 56 | rubygems_host = ENV.fetch("RUBYGEMS_URL", "https://rubygems.org") 57 | specs_uri = File.join(rubygems_host, "specs.4.8.gz") 58 | prerelease_specs_uri = File.join(rubygems_host, "prerelease_specs.4.8.gz") 59 | specs_threads = [] 60 | 61 | specs_threads << Thread.new { download_index(specs_uri) } 62 | specs_threads << Thread.new { [:prerelease] } 63 | specs_threads << Thread.new { download_index(prerelease_specs_uri) } 64 | specs = specs_threads.inject([]) {|sum, t| sum + t.value } 65 | print "# of specs from indexes: #{specs.size - 1}\n" 66 | 67 | specs 68 | end 69 | 70 | def get_local_gems(db) 71 | dataset = db[<<-SQL] 72 | SELECT rubygems.name, versions.number, versions.platform, versions.id 73 | FROM rubygems, versions 74 | WHERE rubygems.id = versions.rubygem_id 75 | AND indexed = true 76 | SQL 77 | 78 | local_gems = {} 79 | dataset.all.each do |h| 80 | gem_helper = BundlerApi::GemHelper.new(h[:name], h[:number], h[:platform]) 81 | local_gems[gem_helper.full_name] = h[:id] 82 | end 83 | print "# of non yanked local gem versions: #{local_gems.size}\n" 84 | 85 | local_gems 86 | end 87 | 88 | def update(db, thread_count) 89 | specs = get_specs 90 | return 60 unless specs 91 | 92 | add_gem_count = BundlerApi::AtomicCounter.new 93 | mutex = Mutex.new 94 | yank_mutex = Mutex.new 95 | local_gems = get_local_gems(db) 96 | prerelease = false 97 | pool = BundlerApi::ConsumerPool.new(thread_count) 98 | cache = BundlerApi::CacheInvalidator.new 99 | 100 | if (specs.size - 1) == local_gems.size 101 | puts "Gem counts match, seems like we're already up to date!" 102 | return 103 | end 104 | 105 | pool.start 106 | specs.each do |spec| 107 | if spec == :prerelease 108 | prerelease = true 109 | next 110 | end 111 | 112 | name, version, platform = spec 113 | payload = BundlerApi::GemHelper.new(name, version, platform, prerelease) 114 | pool.enq(BundlerApi::YankJob.new(local_gems, payload, yank_mutex)) 115 | pool.enq(BundlerApi::Job.new(db, payload, mutex, add_gem_count, cache: cache)) 116 | end 117 | 118 | print "Finished Enqueuing Jobs!\n" 119 | 120 | pool.poison 121 | pool.join 122 | 123 | cache = BundlerApi::CacheInvalidator.new 124 | 125 | unless local_gems.empty? 126 | print "Yanking #{local_gems.size} gems\n" 127 | local_gems.keys.each {|name| print "Yanking: #{name}\n" } 128 | 129 | db[:versions] 130 | .where(:id => local_gems.values) 131 | .update(indexed: false, yanked_at: Time.now) 132 | 133 | versions = db[:versions] 134 | .join(:rubygems, id: :rubygem_id) 135 | .where(versions__id: local_gems.values) 136 | versions.each do |version| 137 | gem_helper = BundlerApi::GemHelper.new(version[:name], version[:number], version[:platform]) 138 | cache.purge_gem(gem_helper) 139 | end 140 | end 141 | 142 | cache.purge_specs if !local_gems.empty? || add_gem_count.count > 0 143 | 144 | print "# of gem versions added: #{add_gem_count.count}\n" 145 | print "# of gem versions yanked: #{local_gems.size}\n" 146 | end 147 | 148 | def fix_deps(db, thread_count) 149 | specs = get_specs 150 | return 60 unless specs 151 | counter = BundlerApi::AtomicCounter.new 152 | mutex = Mutex.new 153 | prerelease = false 154 | pool = BundlerApi::ConsumerPool.new(thread_count) 155 | 156 | pool.start 157 | 158 | prerelease = false 159 | specs.each do |spec| 160 | if spec == :prerelease 161 | prerelease = true 162 | next 163 | end 164 | 165 | name, version, platform = spec 166 | payload = BundlerApi::GemHelper.new(name, version, platform, prerelease) 167 | pool.enq(BundlerApi::FixDepJob.new(db, payload, counter, mutex)) 168 | end 169 | 170 | print "Finished Enqueuing Jobs!\n" 171 | 172 | pool.poison 173 | pool.join 174 | 175 | print "# of gem deps fixed: #{counter.count}\n" 176 | end 177 | 178 | def database_connection(limit = 1) 179 | limit = [limit, 50].min 180 | print "Connecting to database with up to #{limit} connections... " 181 | Sequel.connect(ENV['DATABASE_URL'], max_connections: limit) do |db| 182 | print "connected!\n" 183 | yield db 184 | end 185 | end 186 | 187 | desc "update database" 188 | task :update, :thread_count do |t, args| 189 | thread_count = (args[:thread_count] || 1).to_i 190 | database_connection(thread_count) do |db| 191 | db["select count(*) from rubygems"].count 192 | update(db, thread_count) 193 | end 194 | end 195 | 196 | desc "fixing existing dependencies" 197 | task :fix_deps, :thread_count do |t, args| 198 | thread_count = (args[:thread_count] || 1).to_i 199 | database_connection(thread_count) do |db| 200 | fix_deps(db, thread_count) 201 | end 202 | end 203 | 204 | desc "Add a specific single gem version to the database" 205 | task :add_spec, :name, :version, :platform, :prerelease do |t, args| 206 | cache = BundlerApi::CacheInvalidator.new 207 | args.with_defaults(:platform => 'ruby', :prerelease => false) 208 | payload = BundlerApi::GemHelper.new(args[:name], Gem::Version.new(args[:version]), args[:platform], args[:prerelease]) 209 | database_connection do |db| 210 | BundlerApi::Job.new(db, payload, cache: cache).run 211 | end 212 | end 213 | 214 | desc "Generate/update the versions.list file" 215 | task :versions do |t, args| 216 | database_connection do |db| 217 | file_path = BundlerApi::GemInfo::VERSIONS_FILE_PATH 218 | versions_file = CompactIndex::VersionsFile.new(file_path) 219 | gem_info = BundlerApi::GemInfo.new(db) 220 | 221 | last_update = versions_file.updated_at 222 | gems = gem_info.versions(last_update) 223 | versions_file.update_with(gems) 224 | end 225 | end 226 | 227 | desc "Yank a specific single gem from the database" 228 | task :yank_spec, :name, :version, :platform do |t, args| 229 | args.with_defaults(:platform => 'ruby') 230 | database_connection do |db| 231 | gem_id = db[:rubygems].where(name: args[:name]).first[:id] 232 | version = db[:versions].where( 233 | rubygem_id: gem_id, 234 | number: args[:version], 235 | platform: args[:platform] 236 | ).first 237 | version.update(indexed: false, yanked_at: Time.now) 238 | 239 | puts "Yanked #{version}!" 240 | end 241 | end 242 | 243 | desc "Purge new index from Fastly cache" 244 | task :purge_cdn do 245 | cdn = BundlerApi::CacheInvalidator.new.cdn_client 246 | cdn.purge_path("/names") 247 | cdn.purge_path("/versions") 248 | cdn.purge_key("info/*") 249 | end 250 | -------------------------------------------------------------------------------- /bin/production: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | HEROKU_APP=bundler-api HKAPP=bundler-api exec "${HEROKU_COMMAND:-heroku}" "$@" 3 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # This file was generated by Bundler. 4 | # 5 | # The application 'rake' is installed as part of a gem, and 6 | # this file is here to facilitate running it. 7 | # 8 | 9 | require 'pathname' 10 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", 11 | Pathname.new(__FILE__).realpath) 12 | 13 | require 'rubygems' 14 | require 'bundler/setup' 15 | 16 | load Gem.bin_path('rake', 'rake') 17 | -------------------------------------------------------------------------------- /bin/rspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # This file was generated by Bundler. 4 | # 5 | # The application 'rspec' is installed as part of a gem, and 6 | # this file is here to facilitate running it. 7 | # 8 | 9 | require 'pathname' 10 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", 11 | Pathname.new(__FILE__).realpath) 12 | 13 | require 'rubygems' 14 | require 'bundler/setup' 15 | 16 | load Gem.bin_path('rspec-core', 'rspec') 17 | -------------------------------------------------------------------------------- /bin/staging: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | HEROKU_APP=bundler-api-staging HKAPP=bundler-api-staging exec "${HEROKU_COMMAND:-heroku}" "$@" 3 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | $stdout.sync = true 2 | 3 | lib = File.expand_path('../lib', __FILE__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | 6 | require 'bundler_api/web' 7 | require 'rack-timeout' 8 | require 'newrelic_rpm' 9 | 10 | use Rack::Timeout, service_timeout: ENV.fetch('RACK_TIMEOUT', 5).to_i 11 | use Rack::Deflater 12 | run BundlerApi::Web.new 13 | -------------------------------------------------------------------------------- /config/follow_redirects.vcl: -------------------------------------------------------------------------------- 1 | sub vcl_fetch { 2 | #FASTLY fetch 3 | 4 | if (beresp.status == 302 && beresp.http.Location ~ "s3.amazonaws.com") { 5 | set req.url = regsub(beresp.http.Location,"^https://[^/]+(/.*)","\1"); 6 | restart; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /config/newrelic.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This file configures the New Relic Agent. New Relic monitors Ruby, Java, 3 | # .NET, PHP, Python and Node applications with deep visibility and low 4 | # overhead. For more information, visit www.newrelic.com. 5 | # 6 | # Generated March 22, 2016 7 | # 8 | # This configuration file is custom generated for rubygems 9 | # 10 | # For full documentation of agent configuration options, please refer to 11 | # https://docs.newrelic.com/docs/agents/ruby-agent/installation-configuration/ruby-agent-configuration 12 | 13 | common: &default_settings 14 | # Required license key associated with your New Relic account. 15 | license_key: <%= ENV['NEW_RELIC_LICENSE_KEY'] %> 16 | 17 | # Your application name. Renaming here affects where data displays in New 18 | # Relic. For more details, see https://docs.newrelic.com/docs/apm/new-relic-apm/maintenance/renaming-applications 19 | app_name: bundler-api 20 | 21 | # To disable the agent regardless of other settings, uncomment the following: 22 | # agent_enabled: false 23 | 24 | # Logging level for log/newrelic_agent.log 25 | log_level: info 26 | 27 | 28 | # Environment-specific settings are in this section. 29 | # RAILS_ENV or RACK_ENV (as appropriate) is used to determine the environment. 30 | # If your application has other named environments, configure them here. 31 | development: 32 | <<: *default_settings 33 | app_name: bundler-api (Development) 34 | 35 | # NOTE: There is substantial overhead when running in developer mode. 36 | # Do not use for production or load testing. 37 | developer_mode: true 38 | 39 | test: 40 | <<: *default_settings 41 | # It doesn't make sense to report to New Relic from automated test runs. 42 | monitor_mode: false 43 | 44 | staging: 45 | <<: *default_settings 46 | app_name: bundler-api (Staging) 47 | 48 | production: 49 | <<: *default_settings 50 | -------------------------------------------------------------------------------- /db/migrations/01_rubygems_org_schema_dump.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | create_table(:dependencies, :ignore_index_errors=>true) do 4 | primary_key :id 5 | String :requirements, :size=>255 6 | DateTime :created_at 7 | DateTime :updated_at 8 | Integer :rubygem_id 9 | Integer :version_id 10 | String :scope, :size=>255 11 | String :unresolved_name, :size=>255 12 | 13 | index [:rubygem_id], :name=>:index_dependencies_on_rubygem_id 14 | index [:unresolved_name], :name=>:index_dependencies_on_unresolved_name 15 | index [:version_id], :name=>:index_dependencies_on_version_id 16 | end 17 | 18 | create_table(:linksets, :ignore_index_errors=>true) do 19 | primary_key :id 20 | Integer :rubygem_id 21 | String :home, :size=>255 22 | String :wiki, :size=>255 23 | String :docs, :size=>255 24 | String :mail, :size=>255 25 | String :code, :size=>255 26 | String :bugs, :size=>255 27 | DateTime :created_at 28 | DateTime :updated_at 29 | 30 | index [:rubygem_id], :name=>:index_linksets_on_rubygem_id 31 | end 32 | 33 | create_table(:rubygems, :ignore_index_errors=>true) do 34 | primary_key :id 35 | String :name, :size=>255 36 | DateTime :created_at 37 | DateTime :updated_at 38 | Integer :downloads, :default=>0 39 | String :slug, :size=>255 40 | 41 | index [:name], :name=>:index_rubygems_on_name, :unique=>true 42 | end 43 | 44 | create_table(:versions, :ignore_index_errors=>true) do 45 | primary_key :id 46 | String :authors, :text=>true 47 | String :description, :text=>true 48 | String :number, :size=>255 49 | Integer :rubygem_id 50 | DateTime :built_at 51 | DateTime :updated_at 52 | String :rubyforge_project, :size=>255 53 | String :summary, :text=>true 54 | String :platform, :size=>255 55 | DateTime :created_at 56 | TrueClass :indexed, :default=>true 57 | TrueClass :prerelease 58 | Integer :position 59 | TrueClass :latest 60 | String :full_name, :size=>255 61 | 62 | index [:built_at], :name=>:index_versions_on_built_at 63 | index [:created_at], :name=>:index_versions_on_created_at 64 | index [:full_name], :name=>:index_versions_on_full_name 65 | index [:indexed], :name=>:index_versions_on_indexed 66 | index [:number], :name=>:index_versions_on_number 67 | index [:position], :name=>:index_versions_on_position 68 | index [:prerelease], :name=>:index_versions_on_prerelease 69 | index [:rubygem_id], :name=>:index_versions_on_rubygem_id 70 | index [:rubygem_id, :number, :platform], :name=>:index_versions_on_rubygem_id_and_number_and_platform, :unique=>true 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /db/migrations/02_prune.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | up do 3 | drop_table :linksets 4 | 5 | alter_table :dependencies do 6 | drop_column :created_at 7 | drop_column :updated_at 8 | drop_column :unresolved_name 9 | end 10 | 11 | alter_table :rubygems do 12 | drop_column :created_at 13 | drop_column :updated_at 14 | drop_column :downloads 15 | drop_column :slug 16 | end 17 | 18 | alter_table :versions do 19 | drop_column :created_at 20 | drop_column :updated_at 21 | drop_column :authors 22 | drop_column :description 23 | drop_column :built_at 24 | drop_column :rubyforge_project 25 | drop_column :position 26 | drop_column :latest 27 | end 28 | end 29 | 30 | down do 31 | create_table(:linksets, :ignore_index_errors=>true) do 32 | primary_key :id 33 | Integer :rubygem_id 34 | String :home, :size=>255 35 | String :wiki, :size=>255 36 | String :docs, :size=>255 37 | String :mail, :size=>255 38 | String :code, :size=>255 39 | String :bugs, :size=>255 40 | DateTime :created_at 41 | DateTime :updated_at 42 | 43 | index [:rubygem_id], :name=>:index_linksets_on_rubygem_id 44 | end 45 | 46 | alter_table :dependencies do 47 | add_column :created_at, DateTime 48 | add_column :updated_at, DateTime 49 | add_column :unresolved_name, String, :size=>255 50 | add_index [:unresolved_name], :name=>:index_dependencies_on_unresolved_name 51 | end 52 | 53 | alter_table :rubygems do 54 | add_column :created_at, DateTime 55 | add_column :updated_at, DateTime 56 | add_column :downloads, Integer, :default=>0 57 | add_column :slug, String, :size=>255 58 | end 59 | 60 | alter_table :versions do 61 | add_column :created_at, DateTime 62 | add_column :updated_at, DateTime 63 | add_column :authors, String, :text=>true 64 | add_column :description, String, :text=>true 65 | add_column :built_at, DateTime 66 | add_column :rubyforge_project, String, :size=>255 67 | add_column :position, Integer 68 | add_column :latest, TrueClass 69 | add_index [:created_at], :name=>:index_versions_on_created_at 70 | add_index [:built_at], :name=>:index_versions_on_built_at 71 | add_index [:position], :name=>:index_versions_on_position 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /db/migrations/03_add_ruby_and_rubygems_requirements.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | up do 3 | alter_table :versions do 4 | add_column :rubygems_version, String, :size=>255 5 | add_column :required_ruby_version, String, :size=>255 6 | end 7 | end 8 | 9 | down do 10 | alter_table :versions do 11 | drop_column :rubygems_version 12 | drop_column :required_ruby_version 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrations/04_add_deps_md5_to_rubygems.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :rubygems do 4 | add_column :deps_md5, String 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrations/05_create_checksums.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | create_table :checksums do 4 | primary_key :id 5 | String :name 6 | String :md5 7 | 8 | index [:name], name: :index_checksums_on_name, unqiue: true 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrations/06_add_created_at_to_versions.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :versions do 4 | add_column :created_at, DateTime 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrations/07_add_checksum_to_versions.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :versions do 4 | add_column :checksum, String, :size => 255 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrations/08_add_info_checksum_to_versions.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :versions do 4 | add_column :info_checksum, String, :size => 255 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrations/09_remove_deps_md5_from_rubygems.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :rubygems do 4 | drop_column :deps_md5 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrations/10_add_yanked_at_to_versions.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :versions do 4 | add_column :yanked_at, DateTime, default: nil 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /db/migrations/11_add_yanked_info_checksum_to_versions.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :versions do 4 | add_column :yanked_info_checksum, String, size: 255, default: nil 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/bundler_api.rb: -------------------------------------------------------------------------------- 1 | module BundlerApi 2 | end 3 | -------------------------------------------------------------------------------- /lib/bundler_api/agent_reporting.rb: -------------------------------------------------------------------------------- 1 | require 'bundler_api/metriks' 2 | require 'bundler_api/redis' 3 | 4 | class BundlerApi::AgentReporting 5 | UA_REGEX = %r{^ 6 | bundler/(?#{Gem::Version::VERSION_PATTERN})\s 7 | rubygems/(?#{Gem::Version::VERSION_PATTERN})\s 8 | ruby/(?#{Gem::Version::VERSION_PATTERN})\s 9 | \((?.*)\)\s 10 | command/(?\w+)\s 11 | (?:options/(?\S+)\s)? 12 | (?:ci/(?\S+)\s)? 13 | (?.*) 14 | }x 15 | 16 | def initialize(app) 17 | @app = app 18 | end 19 | 20 | def call(env) 21 | report_user_agent(env['HTTP_USER_AGENT']) 22 | @app.call(env) 23 | end 24 | 25 | private 26 | 27 | def report_user_agent(ua_string) 28 | return unless ua_match = UA_REGEX.match(ua_string) 29 | return if known_id?(ua_match['id']) 30 | 31 | keys = [ "versions.bundler.#{ ua_match['bundler_version'] }", 32 | "versions.rubygems.#{ ua_match['gem_version'] }", 33 | "versions.ruby.#{ ua_match['ruby_version'] }", 34 | "archs.#{ ua_match['arch'] }", 35 | "commands.#{ ua_match['command'] }", 36 | ] 37 | 38 | if ua_match['options'] 39 | keys += ua_match['options'].split(",").map { |k| "options.#{ k }" } 40 | end 41 | 42 | if ua_match['ci'] 43 | keys += ua_match['ci'].split(",").map { |k| "cis.#{ k }" } 44 | end 45 | 46 | keys.each do |metric| 47 | # Librato metric keys are limited to these characters, and 255 chars total 48 | metric.gsub!(/[^.:_\-0-9a-zA-Z]/, '.') 49 | Metriks.meter(metric[0...255]).mark 50 | end 51 | end 52 | 53 | def known_id?(id) 54 | return true if BundlerApi.redis.exists(id) 55 | 56 | BundlerApi.redis.setex(id, 120, true) 57 | false 58 | rescue => e 59 | STDERR.puts "[Error][AgentReporting] `known_id?` raised #{e.class}: #{e.message}" 60 | false 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/bundler_api/cache.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'dalli' 3 | 4 | module BundlerApi 5 | FastlyClient = Struct.new(:service_id, :base_url, :api_key) do 6 | def purge_key(key) 7 | uri = URI("https://api.fastly.com/service/#{service_id}/purge/#{key}") 8 | http(uri).post uri.request_uri, nil, "Fastly-Key" => api_key 9 | end 10 | 11 | def purge_path(path) 12 | uri = URI("#{base_url}#{path}") 13 | http(uri).send_request 'PURGE', uri.path, nil, "Fastly-Key" => api_key 14 | end 15 | 16 | def http(uri) 17 | return unless ENV['RACK_ENV'] == "production" 18 | 19 | Net::HTTP.new(uri.host, uri.port).tap do |http| 20 | http.use_ssl = true 21 | http.verify_mode = OpenSSL::SSL::VERIFY_PEER 22 | end 23 | end 24 | end 25 | 26 | class CacheInvalidator 27 | def initialize(memcached: nil, cdn: nil, silent: false) 28 | @memcached_client = memcached 29 | @cdn_client = cdn 30 | @silent = silent 31 | end 32 | 33 | def log(message) 34 | puts(message) unless @silent 35 | end 36 | 37 | def verify_responses!(responses) 38 | failures = responses.compact.select {|r| r.code.to_i >= 400 } 39 | return if failures.empty? 40 | failures.map! do |response| 41 | "- #{response.uri} => #{response.code}, #{response.body}" 42 | end 43 | raise "The following cache purge requests failed:\n#{failures.join("\n")}" 44 | end 45 | 46 | def purge_specs 47 | keys = %w(dependencies) 48 | paths = %w( 49 | /latest_specs.4.8.gz 50 | /specs.4.8.gz 51 | /prerelease_specs.4.8.gz 52 | /versions 53 | /names 54 | ) 55 | log "Purging #{(keys + paths).join(', ')}" 56 | responses = keys.map {|k| cdn_client.purge_key(k) } 57 | responses += paths.map {|k| cdn_client.purge_path(k) } 58 | verify_responses!(responses) 59 | end 60 | 61 | def purge_gem(gem_helper) 62 | name = gem_helper.name 63 | full_name = gem_helper.full_name 64 | 65 | purge_memory_cache(name) 66 | 67 | paths = %W( 68 | /quick/Marshal.4.8/#{full_name}.gemspec.rz 69 | /gems/#{full_name}.gem 70 | /info/#{name} 71 | ) 72 | log "Purging #{paths.join(', ')}" 73 | responses = paths.map {|k| cdn_client.purge_path(k) } 74 | 75 | verify_responses!(responses) 76 | end 77 | 78 | def purge_memory_cache(name) 79 | memcached_client.delete "deps/v1/#{name}" 80 | memcached_client.delete "info/#{name}" 81 | memcached_client.delete "names" 82 | end 83 | 84 | def cdn_client 85 | @cdn_client ||= if ENV['FASTLY_SERVICE_ID'] 86 | FastlyClient.new( 87 | ENV['FASTLY_SERVICE_ID'], 88 | ENV['FASTLY_BASE_URL'], 89 | ENV['FASTLY_API_KEY'] 90 | ) 91 | else 92 | # Create a mock Fastly client 93 | Class.new do 94 | def purge_key(key); end 95 | def purge_path(path); end 96 | end.new 97 | end 98 | end 99 | 100 | def memcached_client 101 | @memcached_client ||= if ENV["MEMCACHEDCLOUD_SERVERS"] 102 | servers = (ENV["MEMCACHEDCLOUD_SERVERS"] || "").split(",") 103 | Dalli::Client.new( 104 | servers, { 105 | username: ENV["MEMCACHEDCLOUD_USERNAME"], 106 | password: ENV["MEMCACHEDCLOUD_PASSWORD"], 107 | failover: true, 108 | socket_timeout: 1.5, 109 | socket_failure_delay: 0.2 110 | } 111 | ) 112 | else 113 | Dalli::Client.new 114 | end 115 | end 116 | 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/bundler_api/checksum.rb: -------------------------------------------------------------------------------- 1 | class BundlerApi::Checksum 2 | attr_reader :checksum, :name 3 | 4 | def initialize(conn, name) 5 | @conn = conn 6 | @name = name 7 | row = @conn[:checksums].where(name: @name).first 8 | 9 | if row 10 | @exists = true 11 | @checksum = row[:md5] 12 | end 13 | end 14 | 15 | def checksum=(sum) 16 | if @exists 17 | @conn[:checksums].where(name: @name).update(md5: sum) 18 | else 19 | @conn[:checksums].insert(name: @name, md5: sum) 20 | end 21 | 22 | @checksum = sum 23 | end 24 | 25 | end 26 | -------------------------------------------------------------------------------- /lib/bundler_api/env.rb: -------------------------------------------------------------------------------- 1 | module BundlerApi 2 | class Env 3 | def self.load 4 | return unless local_env? 5 | require 'dotenv' 6 | Dotenv.load '.env.local', '.env' 7 | end 8 | 9 | def self.local_env? 10 | ENV['RACK_ENV'].nil? || 11 | ENV['RACK_ENV'] == 'development' or 12 | ENV['RACK_ENV'] == 'test' 13 | end 14 | end 15 | end 16 | 17 | BundlerApi::Env.load 18 | -------------------------------------------------------------------------------- /lib/bundler_api/gem_helper.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | require 'net/http' 3 | require 'bundler_api' 4 | require 'json' 5 | 6 | class BundlerApi::HTTPError < RuntimeError 7 | end 8 | 9 | class BundlerApi::GemHelper < Struct.new(:name, :version, :platform, :prerelease) 10 | RUBYGEMS_URL = ENV['RUBYGEMS_URL'] || "https://rubygems.org" 11 | 12 | REDIRECT_LIMIT = 5 13 | TRY_LIMIT = 4 14 | TRY_BACKOFF = 3 15 | 16 | def initialize(*) 17 | super 18 | @mutex = Mutex.new 19 | @gemspec = nil 20 | end 21 | 22 | def full_name 23 | full_name = "#{name}-#{version}" 24 | full_name << "-#{platform}" if platform != 'ruby' 25 | 26 | full_name 27 | end 28 | 29 | def download_spec 30 | url = download_gem_url("quick/Marshal.4.8/#{full_name}.gemspec.rz") 31 | @mutex.synchronize do 32 | @gemspec ||= Marshal.load(Gem.inflate(fetch(url))) 33 | end 34 | rescue => e 35 | raise(e) if e.is_a?(BundlerApi::HTTPError) 36 | STDERR.puts "[Error] Downloading gemspec #{full_name} failed! #{e.class}: #{e.message}" 37 | STDERR.puts e.backtrace.join("\n ") 38 | end 39 | 40 | def download_checksum 41 | url = File.join(RUBYGEMS_URL, "/api/v2/rubygems/#{name}/versions/#{version}.json") 42 | response = fetch(url) 43 | return warn("WARNING: Can't find gem #{name}-#{version} at #{url}") if response.empty? 44 | version_info = JSON.parse(response) 45 | return warn("WARNING: Gem #{name}-#{version} has no checksum!") if version_info['sha'].nil? 46 | 47 | version_info['sha'] 48 | end 49 | 50 | private 51 | 52 | def fetch(url, redirects = 0, tries = []) 53 | raise BundlerApi::HTTPError, "Too many redirects #{url}" if redirects >= REDIRECT_LIMIT 54 | raise BundlerApi::HTTPError, "Could not download #{url} (#{tries.join(", ")})" if tries.size >= TRY_LIMIT 55 | 56 | uri = URI.parse(url) 57 | response = begin 58 | Net::HTTP.get_response(uri) 59 | rescue => e 60 | "(#{url}) #{e}" 61 | end 62 | 63 | case response 64 | when Net::HTTPRedirection 65 | fetch(response["location"], redirects + 1) 66 | when Net::HTTPSuccess 67 | response.body 68 | else 69 | tries << response 70 | exp = tries.size 71 | exp *= 2 if response.is_a?(Net::HTTPTooManyRequests) 72 | sleep(TRY_BACKOFF ** exp) 73 | fetch(url, redirects, tries) 74 | end 75 | end 76 | 77 | def download_gem_url(path = nil) 78 | @base_url ||= ENV.fetch("DOWNLOAD_BASE", "https://rubygems.global.ssl.fastly.net") 79 | File.join(@base_url, path || '') 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/bundler_api/gem_info.rb: -------------------------------------------------------------------------------- 1 | require 'bundler_api' 2 | require 'compact_index' 3 | 4 | # Return data about all the gems: all gem names, all versions of all gems, all dependencies for all versions of a gem 5 | class BundlerApi::GemInfo 6 | VERSIONS_FILE_PATH = "./versions.list" 7 | DepKey = Struct.new(:name, :number, :platform, :required_ruby_version, :rubygems_version, :checksum, :created_at) 8 | 9 | def initialize(connection) 10 | @conn = connection 11 | end 12 | 13 | # @param [String] the gem name 14 | def deps_for(gem_name) 15 | dataset = @conn[<<-SQL, gem_name] 16 | SELECT rv.name, rv.number, rv.platform, rv.required_ruby_version, rv.checksum, 17 | rv.rubygems_version, d.requirements, rv.created_at, for_dep_name.name dep_name 18 | FROM 19 | (SELECT r.name, v.number, v.platform,v.rubygems_version, v.checksum, 20 | v.required_ruby_version, v.created_at, v.id AS version_id 21 | FROM rubygems AS r, versions AS v 22 | WHERE v.rubygem_id = r.id 23 | AND v.indexed is true 24 | AND r.name = ? 25 | ORDER BY v.created_at, v.number, v.platform) AS rv 26 | LEFT JOIN dependencies AS d ON 27 | d.version_id = rv.version_id 28 | LEFT JOIN rubygems AS for_dep_name ON 29 | d.rubygem_id = for_dep_name.id 30 | AND d.scope = 'runtime' 31 | ORDER BY rv.created_at, rv.number, rv.platform, for_dep_name.name; 32 | SQL 33 | 34 | deps = {} 35 | 36 | dataset.each do |row| 37 | key = DepKey.new( 38 | row[:name], 39 | row[:number], 40 | row[:platform], 41 | row[:required_ruby_version], 42 | row[:rubygems_version], 43 | row[:checksum], 44 | row[:created_at] 45 | ) 46 | deps[key] = [] unless deps[key] 47 | deps[key] << [row[:dep_name], row[:requirements]] if row[:dep_name] 48 | end 49 | 50 | deps.map do |dep_key, gem_deps| 51 | { 52 | name: dep_key.name, 53 | number: dep_key.number, 54 | platform: dep_key.platform, 55 | rubygems_version: dep_key.rubygems_version, 56 | ruby_version: dep_key.required_ruby_version, 57 | checksum: dep_key.checksum, 58 | created_at: dep_key.created_at, 59 | dependencies: gem_deps 60 | } 61 | end 62 | end 63 | 64 | # return list of gem names 65 | def names 66 | @conn[:rubygems].select(:name).order(:name).all.map {|r| r[:name] } 67 | end 68 | 69 | def versions(date,include_yanks = false) 70 | dataset = if include_yanks 71 | @conn[<<-SQL, date,date] 72 | (SELECT r.name, v.created_at as date, v.info_checksum, v.number, v.platform 73 | FROM rubygems AS r, versions AS v 74 | WHERE v.rubygem_id = r.id AND 75 | v.created_at > ?) 76 | UNION 77 | (SELECT r.name, v.yanked_at as date, v.yanked_info_checksum as info_checksum, '-'||v.number, v.platform 78 | FROM rubygems AS r, versions AS v 79 | WHERE v.rubygem_id = r.id AND 80 | v.indexed is false AND 81 | v.yanked_at > ?) 82 | ORDER BY date, number, platform, name 83 | SQL 84 | else 85 | @conn[<<-SQL, date] 86 | SELECT r.name, v.created_at, v.checksum, v.info_checksum, v.number, v.platform 87 | FROM rubygems AS r, versions AS v 88 | WHERE v.rubygem_id = r.id AND 89 | v.indexed is true AND 90 | v.created_at > ? 91 | ORDER BY v.created_at, v.number, v.platform, r.name 92 | SQL 93 | end 94 | dataset.map do |entry| 95 | CompactIndex::Gem.new(entry[:name], [ 96 | CompactIndex::GemVersion.new( 97 | entry[:number], 98 | entry[:platform], 99 | entry[:checksum], 100 | entry[:info_checksum] 101 | ) 102 | ]) 103 | end 104 | end 105 | 106 | def info(name) 107 | deps = deps_for(name) 108 | deps.map! do |dep| 109 | dependencies = dep[:dependencies].map do |d| 110 | CompactIndex::Dependency.new(d[0], d[1]) 111 | end 112 | 113 | CompactIndex::GemVersion.new( 114 | dep[:number], 115 | dep[:platform], 116 | dep[:checksum], 117 | dep[:info_checksum], 118 | dependencies, 119 | dep[:ruby_version], 120 | dep[:rubygems_version] 121 | ) 122 | end 123 | CompactIndex.info(deps) 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/bundler_api/metriks.rb: -------------------------------------------------------------------------------- 1 | require 'metriks' 2 | require 'metriks/middleware' 3 | require 'metriks-librato_metrics' 4 | 5 | module BundlerApi 6 | class Metriks 7 | def self.start(worker_index = nil) 8 | new(ENV['LIBRATO_METRICS_USER'], ENV['LIBRATO_METRICS_TOKEN'], worker_index) 9 | end 10 | 11 | def initialize(user, token, worker_index = nil) 12 | return unless user && token 13 | 14 | opts = { 15 | on_error: error_handler, 16 | source: source_name(worker_index), 17 | interval: 10, 18 | } 19 | 20 | prefix = ENV.fetch('LIBRATO_METRICS_PREFIX') do 21 | ENV['RACK_ENV'] unless ENV['RACK_ENV'] == 'production' 22 | end 23 | opts[:prefix] = prefix if prefix && !prefix.empty? 24 | 25 | ::Metriks::LibratoMetricsReporter.new(user, token, opts).start 26 | end 27 | 28 | private 29 | 30 | def source_name(worker = nil) 31 | name = ENV.fetch('DYNO') do 32 | # Fall back to hostname if DYNO isn't set. 33 | require 'socket' 34 | Socket.gethostname 35 | end 36 | 37 | worker ? "#{name}.w#{worker}" : name 38 | end 39 | 40 | def error_handler 41 | -> (e) do 42 | STDOUT.puts("[Error][Librato] #{e.class} raised during metric submission: #{e.message}") 43 | 44 | if e.is_a?(::Metriks::LibratoMetricsReporter::RequestFailedError) 45 | STDOUT.puts(" Response body: #{e.res.body}") 46 | STDOUT.puts(" Submitted data: #{e.data.inspect}") 47 | else 48 | STDOUT.puts e.backtrace.join("\n ") 49 | end 50 | end 51 | end 52 | end 53 | end 54 | 55 | -------------------------------------------------------------------------------- /lib/bundler_api/redis.rb: -------------------------------------------------------------------------------- 1 | require 'redis' 2 | 3 | module BundlerApi 4 | class << self 5 | attr_accessor :redis 6 | end 7 | end 8 | 9 | BundlerApi.redis = Redis.new(url: ENV[ENV['REDIS_ENV']]) -------------------------------------------------------------------------------- /lib/bundler_api/runtime_instrumentation.rb: -------------------------------------------------------------------------------- 1 | require 'hitimes' 2 | 3 | class BundlerApi::RuntimeInstrumentation 4 | attr_accessor :interval, :ruby_thread 5 | 6 | def initialize(options = {}) 7 | @interval = (options[:interval] || 1.0).to_f 8 | end 9 | 10 | def self.start 11 | new.start 12 | end 13 | 14 | def start 15 | return if ruby_thread && ruby_thread.alive? 16 | 17 | self.ruby_thread = Thread.new do 18 | histogram = Metriks.histogram('ruby.variance') 19 | 20 | while true 21 | ruby_interval = Hitimes::Interval.now 22 | sleep interval 23 | histogram.update(ruby_interval.duration - interval) 24 | end 25 | end 26 | end 27 | end 28 | 29 | BundlerApi::RuntimeInstrumentation.start 30 | -------------------------------------------------------------------------------- /lib/bundler_api/strategy.rb: -------------------------------------------------------------------------------- 1 | module BundlerApi 2 | class RedirectionStrategy 3 | def initialize(rubygems_url) 4 | @rubygems_url = rubygems_url 5 | end 6 | 7 | def serve_marshal(id, app) 8 | app.redirect "#{@rubygems_url}/quick/Marshal.4.8/#{id}" 9 | end 10 | 11 | def serve_actual_gem(id, app) 12 | app.redirect "#{@rubygems_url}/fetch/actual/gem/#{id}" 13 | end 14 | 15 | def serve_gem(id, app) 16 | app.redirect "#{@rubygems_url}/gems/#{id}" 17 | end 18 | 19 | def serve_latest_specs(app) 20 | app.redirect "#{@rubygems_url}/latest_specs.4.8.gz" 21 | end 22 | 23 | def serve_specs(app) 24 | app.redirect "#{@rubygems_url}/specs.4.8.gz" 25 | end 26 | 27 | def serve_prerelease_specs(app) 28 | app.redirect "#{@rubygems_url}/prerelease_specs.4.8.gz" 29 | end 30 | 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/bundler_api/update/atomic_counter.rb: -------------------------------------------------------------------------------- 1 | module BundlerApi 2 | class AtomicCounter 3 | 4 | def initialize 5 | @count = 0 6 | @mutex = Mutex.new 7 | end 8 | 9 | def count 10 | @mutex.synchronize do 11 | @count 12 | end 13 | end 14 | 15 | def increment 16 | @mutex.synchronize do 17 | @count += 1 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/bundler_api/update/consumer_pool.rb: -------------------------------------------------------------------------------- 1 | require 'thread' 2 | 3 | module BundlerApi 4 | class ConsumerPool 5 | POISON = :poison 6 | 7 | def initialize(size) 8 | @size = size 9 | @queue = Queue.new 10 | @threads = [] 11 | end 12 | 13 | def enq(job) 14 | @queue.enq(job) 15 | end 16 | 17 | def start 18 | @size.times { @threads << create_thread } 19 | end 20 | 21 | def join 22 | @threads.each {|t| t.join } 23 | end 24 | 25 | def poison 26 | @size.times { @queue.enq(POISON) } 27 | end 28 | 29 | private 30 | def create_thread 31 | Thread.new { 32 | loop do 33 | job = @queue.deq 34 | break if job == POISON 35 | 36 | job.run 37 | end 38 | } 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/bundler_api/update/fix_dep_job.rb: -------------------------------------------------------------------------------- 1 | require 'bundler_api/update/job' 2 | 3 | class BundlerApi::FixDepJob < BundlerApi::Job 4 | def initialize(db, payload, counter = nil, mutex = nil, silent: false) 5 | super(db, payload, mutex, counter, fix_deps: true, silent: silent) 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/bundler_api/update/gem_db_helper.rb: -------------------------------------------------------------------------------- 1 | require 'set' 2 | require 'bundler_api' 3 | 4 | class BundlerApi::GemDBHelper 5 | def initialize(db, gem_cache, mutex) 6 | @db = db 7 | @gem_cache = gem_cache 8 | @mutex = mutex 9 | end 10 | 11 | def exists?(payload) 12 | key = payload.full_name 13 | 14 | synchronize do 15 | return true if @gem_cache[key] 16 | end 17 | 18 | dataset = @db[<<-SQL, payload.name, payload.version.version, payload.platform] 19 | SELECT rubygems.id AS rubygem_id, versions.id AS version_id 20 | FROM rubygems, versions 21 | WHERE rubygems.id = versions.rubygem_id 22 | AND rubygems.name = ? 23 | AND versions.number = ? 24 | AND versions.platform = ? 25 | AND versions.indexed = true 26 | SQL 27 | 28 | result = dataset.first 29 | 30 | synchronize do 31 | @gem_cache[key] = result if result 32 | end 33 | 34 | !result.nil? 35 | end 36 | 37 | def find_or_insert_rubygem(spec) 38 | insert = nil 39 | rubygem_id = nil 40 | rubygem = @db[:rubygems].filter(name: spec.name.to_s).select(:id).first 41 | 42 | if rubygem 43 | insert = false 44 | rubygem_id = rubygem[:id] 45 | else 46 | insert = true 47 | rubygem_id = @db[:rubygems].insert(name: spec.name) 48 | end 49 | 50 | @db[:checksums].filter(name: "names.list").update(md5: nil) if insert 51 | 52 | [insert, rubygem_id] 53 | end 54 | 55 | def update_info_checksum(version_id, info_checksum) 56 | @db[:versions].where(id: version_id).update(info_checksum: info_checksum) 57 | end 58 | 59 | def update_yanked_info_checksum(query, info_checksum) 60 | @db[:versions].where(query).update(yanked_info_checksum: info_checksum) 61 | end 62 | 63 | def find_or_insert_version(spec, rubygem_id, platform = 'ruby', checksum = nil, indexed = nil) 64 | insert = nil 65 | version_id = nil 66 | version = @db[:versions].filter( 67 | rubygem_id: rubygem_id, 68 | number: spec.version.version, 69 | platform: platform, 70 | ).select(:id, :indexed).first 71 | 72 | if version 73 | insert = false 74 | version_id = version[:id] 75 | @db[:versions].where(id: version_id).update(indexed: indexed) if !indexed.nil? && version[:indexed] != indexed 76 | else 77 | insert = true 78 | indexed = true if indexed.nil? 79 | 80 | spec_rubygems = get_spec_rubygems(spec) 81 | spec_ruby = get_spec_ruby(spec) 82 | version_attrs = { 83 | number: spec.version.version, 84 | rubygem_id: rubygem_id, 85 | # rubygems.org actually uses the platform from the index and not from the spec 86 | platform: platform, 87 | indexed: indexed, 88 | prerelease: spec.version.prerelease?, 89 | full_name: spec.full_name, 90 | rubygems_version: (spec.required_rubygems_version || '').to_s, 91 | required_ruby_version: (spec.required_ruby_version || '').to_s, 92 | created_at: Time.now, 93 | } 94 | version_attrs[:checksum] = checksum if checksum 95 | version_id = @db[:versions].insert(version_attrs) 96 | end 97 | 98 | if insert 99 | @db[:checksums].filter(name: "versions").update(md5: nil) 100 | end 101 | 102 | [insert, version_id] 103 | end 104 | 105 | def insert_dependencies(spec, version_id) 106 | deps_added = [] 107 | 108 | spec.dependencies.each do |dep| 109 | rubygem_name = nil 110 | requirements = nil 111 | scope = nil 112 | 113 | if dep.is_a?(Gem::Dependency) 114 | rubygem_name = dep.name.to_s 115 | requirements = dep.requirement.to_s 116 | scope = dep.type.to_s 117 | else 118 | rubygem_name, requirements = dep 119 | # assume runtime for legacy deps 120 | scope = "runtime" 121 | end 122 | 123 | dep_rubygem = @db[:rubygems].filter(name: rubygem_name).select(:id).first 124 | if dep_rubygem 125 | dep = @db[:dependencies].filter(rubygem_id: dep_rubygem[:id], 126 | version_id: version_id).first 127 | if !dep || !matching_requirements?(requirements, dep[:requirements]) 128 | deps_added << "#{requirements} #{rubygem_name}" 129 | @db[:dependencies].insert( 130 | requirements: requirements, 131 | rubygem_id: dep_rubygem[:id], 132 | version_id: version_id, 133 | scope: scope 134 | ) 135 | end 136 | end 137 | end 138 | 139 | deps_added 140 | end 141 | 142 | def yank!(gem_name, version, platform) 143 | rubygem_id = @db[:rubygems].filter(name: gem_name).select(:id).first[:id] 144 | query = { 145 | rubygem_id: rubygem_id, 146 | number: version, 147 | platform: platform, 148 | } 149 | @db[:versions].where(query).update(indexed: false) 150 | info_checksum = Digest::MD5.hexdigest(BundlerApi::GemInfo.new(@db).info(gem_name)) 151 | update_yanked_info_checksum(query, info_checksum) 152 | end 153 | 154 | private 155 | def matching_requirements?(requirements1, requirements2) 156 | Set.new(requirements1.split(", ")) == Set.new(requirements2.split(", ")) 157 | end 158 | 159 | def get_spec_rubygems(spec) 160 | spec_rubygems = spec.required_rubygems_version 161 | if spec_rubygems && !spec_rubygems.to_s.empty? 162 | spec_rubygems.to_s 163 | end 164 | end 165 | 166 | def get_spec_ruby(spec) 167 | spec_ruby = spec.required_ruby_version 168 | if spec_ruby && !spec_ruby.to_s.empty? 169 | spec_ruby.to_s 170 | end 171 | end 172 | 173 | def synchronize 174 | return yield unless @mutex 175 | 176 | @mutex.synchronize do 177 | yield 178 | end 179 | end 180 | 181 | end 182 | -------------------------------------------------------------------------------- /lib/bundler_api/update/job.rb: -------------------------------------------------------------------------------- 1 | require 'bundler_api' 2 | require 'bundler_api/gem_info' 3 | require 'bundler_api/update/gem_db_helper' 4 | 5 | class BundlerApi::Job 6 | class MockMutex 7 | def synchronize 8 | yield if block_given? 9 | end 10 | end 11 | 12 | attr_reader :payload 13 | @@gem_cache = {} 14 | 15 | def initialize(db, payload, mutex = Mutex.new, gem_count = nil, fix_deps: false, silent: false, cache: nil) 16 | @db = db 17 | @payload = payload 18 | @mutex = mutex || MockMutex.new 19 | @gem_count = gem_count 20 | @db_helper = BundlerApi::GemDBHelper.new(@db, @@gem_cache, @mutex) 21 | @gem_info = BundlerApi::GemInfo.new(@db) 22 | @fix_deps = fix_deps 23 | @silent = silent 24 | @cache = cache 25 | end 26 | 27 | def run 28 | return if @db_helper.exists?(@payload) && !@fix_deps 29 | return if !@db_helper.exists?(@payload) && @fix_deps 30 | log "Adding: #{@payload.full_name}\n" 31 | 32 | spec = @payload.download_spec 33 | return unless spec 34 | 35 | checksum = @payload.download_checksum unless @fix_deps 36 | @mutex.synchronize do 37 | deps_added = insert_spec(spec, checksum) 38 | @gem_count.increment if @gem_count && (!deps_added.empty? || !@fix_deps) 39 | @cache.purge_gem(@payload) if @cache 40 | end 41 | rescue BundlerApi::HTTPError => e 42 | log "BundlerApi::Job#run gem=#{@payload.full_name.inspect} " + 43 | "message=#{e.message.inspect}" 44 | end 45 | 46 | def self.clear_cache 47 | @@gem_cache.clear 48 | end 49 | 50 | private 51 | 52 | def log(message) 53 | puts message unless @silent 54 | end 55 | 56 | def insert_spec(spec, checksum) 57 | raise "Failed to load spec" unless spec 58 | 59 | @db.transaction do 60 | rubygem_insert, rubygem_id = @db_helper.find_or_insert_rubygem(spec) 61 | version_insert, version_id = @db_helper.find_or_insert_version( 62 | spec, 63 | rubygem_id, 64 | @payload.platform, 65 | checksum, 66 | true 67 | ) 68 | info_checksum = Digest::MD5.hexdigest(@gem_info.info(spec.name)) 69 | @db_helper.update_info_checksum(version_id, info_checksum) 70 | @db_helper.insert_dependencies(spec, version_id) 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/bundler_api/update/yank_job.rb: -------------------------------------------------------------------------------- 1 | require 'bundler_api' 2 | require 'bundler_api/gem_helper' 3 | 4 | class BundlerApi::YankJob 5 | def initialize(gem_cache, payload, mutex = Mutex.new) 6 | @gem_cache = gem_cache 7 | @payload = payload 8 | @mutex = mutex 9 | end 10 | 11 | def run 12 | @mutex.synchronize do 13 | @gem_cache.delete(@payload.full_name) 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/bundler_api/web.rb: -------------------------------------------------------------------------------- 1 | require 'sinatra/base' 2 | require 'sequel' 3 | require 'json' 4 | require 'bundler_api' 5 | require 'compact_index' 6 | require 'bundler_api/agent_reporting' 7 | require 'bundler_api/checksum' 8 | require 'bundler_api/gem_info' 9 | require 'bundler_api/cache' 10 | require 'bundler_api/metriks' 11 | require 'bundler_api/runtime_instrumentation' 12 | require 'bundler_api/gem_helper' 13 | require 'bundler_api/update/job' 14 | require 'bundler_api/update/yank_job' 15 | require 'bundler_api/strategy' 16 | 17 | class BundlerApi::Web < Sinatra::Base 18 | API_REQUEST_LIMIT = 200 19 | PG_STATEMENT_TIMEOUT = ENV['PG_STATEMENT_TIMEOUT'] || 1000 20 | RUBYGEMS_URL = ENV['RUBYGEMS_URL'] || "https://www.rubygems.org" 21 | NEW_INDEX_ENABLED = ENV['NEW_INDEX_DISABLED'].nil? 22 | 23 | unless ENV['RACK_ENV'] == 'test' 24 | use Metriks::Middleware 25 | use BundlerApi::AgentReporting 26 | end 27 | 28 | def initialize(conn = nil, write_conn = nil, gem_strategy = nil, silent: false) 29 | @rubygems_token = ENV['RUBYGEMS_TOKEN'] 30 | 31 | statement_timeout = proc {|c| c.execute("SET statement_timeout = #{PG_STATEMENT_TIMEOUT}") } 32 | @conn = conn || begin 33 | Sequel.connect(ENV['FOLLOWER_DATABASE_URL'], 34 | max_connections: ENV['MAX_THREADS'], 35 | after_connect: statement_timeout) 36 | end 37 | 38 | @write_conn = write_conn || begin 39 | Sequel.connect(ENV['DATABASE_URL'], 40 | max_connections: ENV['MAX_THREADS']) 41 | end 42 | 43 | @gem_info = BundlerApi::GemInfo.new(@conn) 44 | file_path = BundlerApi::GemInfo::VERSIONS_FILE_PATH 45 | @versions_file = CompactIndex::VersionsFile.new(file_path) 46 | 47 | @silent = silent 48 | @cache = BundlerApi::CacheInvalidator.new(silent: silent) 49 | @dalli_client = @cache.memcached_client 50 | super() 51 | @gem_strategy = gem_strategy || BundlerApi::RedirectionStrategy.new(RUBYGEMS_URL) 52 | end 53 | 54 | set :root, File.join(File.dirname(__FILE__), '..', '..') 55 | 56 | not_found do 57 | status 404 58 | body JSON.dump({"error" => "Not found", "code" => 404}) 59 | end 60 | 61 | def gems 62 | halt(200) if params[:gems].nil? || params[:gems].empty? 63 | g = params[:gems].is_a?(Array) ? params[:gems] : params[:gems].split(',') 64 | g.uniq 65 | end 66 | 67 | def get_payload 68 | params = JSON.parse(request.body.read) 69 | log "webhook request: #{params.inspect}" 70 | 71 | if @rubygems_token && (params["rubygems_token"] != @rubygems_token) 72 | halt 403, "You're not Rubygems" 73 | end 74 | 75 | %w(name version platform prerelease).each do |key| 76 | halt 422, "No spec #{key} given" if params[key].nil? 77 | end 78 | 79 | version = Gem::Version.new(params["version"]) 80 | BundlerApi::GemHelper.new(params["name"], version, 81 | params["platform"], params["prerelease"]) 82 | rescue JSON::ParserError 83 | halt 422, "Invalid JSON" 84 | end 85 | 86 | def json_payload(payload) 87 | content_type 'application/json;charset=UTF-8' 88 | JSON.dump(:name => payload.name, :version => payload.version.version, 89 | :platform => payload.platform, :prerelease => payload.prerelease) 90 | end 91 | 92 | get "/" do 93 | cache_control :public, max_age: 31536000 94 | redirect 'https://www.rubygems.org' 95 | end 96 | 97 | get "/api/v1/dependencies" do 98 | halt 422, "Too many gems (use --full-index instead)" if gems.length > API_REQUEST_LIMIT 99 | 100 | content_type 'application/octet-stream' 101 | 102 | deps = with_metriks { get_cached_dependencies } 103 | Marshal.dump(deps) 104 | end 105 | 106 | get "/api/v1/dependencies.json" do 107 | halt 422, { 108 | "error" => "Too many gems (use --full-index instead)", 109 | "code" => 422 110 | }.to_json if gems.length > API_REQUEST_LIMIT 111 | 112 | content_type 'application/json;charset=UTF-8' 113 | 114 | deps = with_metriks { get_cached_dependencies } 115 | deps.to_json 116 | end 117 | 118 | post "/api/v1/add_spec.json" do 119 | Metriks.timer('webhook.add_spec').time do 120 | payload = get_payload 121 | job = BundlerApi::Job.new(@write_conn, payload, silent: @silent) 122 | job.run 123 | 124 | in_background do 125 | @cache.purge_gem(payload) 126 | @cache.purge_specs 127 | end 128 | 129 | json_payload(payload) 130 | end 131 | end 132 | 133 | post "/api/v1/remove_spec.json" do 134 | Metriks.timer('webhook.remove_spec').time do 135 | payload = get_payload 136 | BundlerApi::GemDBHelper.new(@write_conn, {}, BundlerApi::Job::MockMutex.new) 137 | .yank!(payload.name.to_s, payload.version.version, payload.platform) 138 | 139 | in_background do 140 | @cache.purge_gem(payload) 141 | @cache.purge_specs 142 | end 143 | 144 | json_payload(payload) 145 | end 146 | end 147 | 148 | get "/names" do 149 | status 404 unless NEW_INDEX_ENABLED 150 | etag_response_for("names") do 151 | @dalli_client.fetch('names') { CompactIndex.names(@gem_info.names) } 152 | end 153 | end 154 | 155 | get "/versions" do 156 | status 404 unless NEW_INDEX_ENABLED 157 | etag_response_for("versions") do 158 | from_date = @versions_file.updated_at 159 | extra_gems = @gem_info.versions(from_date, true) 160 | CompactIndex.versions(@versions_file, extra_gems) 161 | end 162 | end 163 | 164 | get "/info/:name" do 165 | status 404 unless NEW_INDEX_ENABLED 166 | etag_response_for(params[:name]) do 167 | @dalli_client.fetch("info/#{params[:name]}") { @gem_info.info(params[:name]) } 168 | end 169 | end 170 | 171 | get "/quick/Marshal.4.8/:id" do 172 | @gem_strategy.serve_marshal(params[:id], self) 173 | end 174 | 175 | get "/fetch/actual/gem/:id" do 176 | @gem_strategy.serve_actual_gem(params[:id], self) 177 | end 178 | 179 | get "/gems/:id" do 180 | @gem_strategy.serve_gem(params[:id], self) 181 | end 182 | 183 | get "/latest_specs.4.8.gz" do 184 | @gem_strategy.serve_latest_specs(self) 185 | end 186 | 187 | get "/specs.4.8.gz" do 188 | @gem_strategy.serve_specs(self) 189 | end 190 | 191 | get "/prerelease_specs.4.8.gz" do 192 | @gem_strategy.serve_prerelease_specs(self) 193 | end 194 | 195 | private 196 | 197 | def etag_response_for(name) 198 | sum = BundlerApi::Checksum.new(@write_conn, name) 199 | return if not_modified?(sum.checksum) 200 | 201 | response_body = yield 202 | sum.checksum = Digest::MD5.hexdigest(response_body) 203 | 204 | headers "ETag" => quote(sum.checksum) 205 | headers "Surrogate-Control" => "max-age=3600, stale-while-revalidate=60" 206 | content_type "text/plain" 207 | requested_range_for(response_body) 208 | end 209 | 210 | def not_modified?(checksum) 211 | etags = parse_etags(request.env["HTTP_IF_NONE_MATCH"]) 212 | 213 | return unless etags.include?(checksum) 214 | headers "ETag" => quote(checksum) 215 | status 304 216 | body "" 217 | end 218 | 219 | def requested_range_for(response_body) 220 | ranges = Rack::Utils.byte_ranges(env, response_body.bytesize) 221 | 222 | if ranges 223 | status 206 224 | body ranges.map! {|range| response_body.byteslice(range) }.join 225 | else 226 | status 200 227 | body response_body 228 | end 229 | end 230 | 231 | def with_metriks 232 | timer = Metriks.timer('dependencies').time 233 | yield.tap do |deps| 234 | Metriks.histogram('gems.size').update(gems.size) 235 | Metriks.histogram('dependencies.size').update(deps.size) 236 | end 237 | ensure 238 | timer.stop if timer 239 | end 240 | 241 | def get_cached_dependencies 242 | dependencies = [] 243 | keys = gems.map { |g| "deps/v1/#{g}" } 244 | @dalli_client.get_multi(keys) do |key, value| 245 | Metriks.meter('dependencies.memcached.hit').mark 246 | keys.delete(key) 247 | dependencies += value 248 | end 249 | 250 | keys.each do |gem| 251 | Metriks.meter('dependencies.memcached.miss').mark 252 | name = gem.gsub("deps/v1/", "") 253 | result = @gem_info.deps_for(name) 254 | @dalli_client.set(gem, result) 255 | dependencies += result 256 | end 257 | dependencies 258 | end 259 | 260 | def quote(string) 261 | '"' << string << '"' 262 | end 263 | 264 | def parse_etags(value) 265 | value ? value.split(/, ?/).select{|s| s.sub!(/"(.*)"/, '\1') } : [] 266 | end 267 | 268 | def log(message) 269 | puts message unless @silent 270 | end 271 | 272 | def in_background 273 | Thread.new do 274 | begin 275 | yield 276 | rescue => e 277 | STDOUT.puts "[Error][Web] #{e.class} raised during background task: #{e.message}" 278 | STDERR.puts e.backtrace.join("\n ") 279 | end 280 | end 281 | end 282 | 283 | end 284 | -------------------------------------------------------------------------------- /lib/puma/instrumented_cli.rb: -------------------------------------------------------------------------------- 1 | require 'puma/cli' 2 | require 'puma/single' 3 | 4 | class Puma::Single 5 | def backlog 6 | return unless @server 7 | @server.backlog 8 | end 9 | 10 | def running 11 | return unless @server 12 | @server.running 13 | end 14 | end 15 | 16 | class Puma::InstrumentedCLI < Puma::CLI 17 | attr_reader :status 18 | 19 | def backlog 20 | return unless @runner 21 | @runner.backlog 22 | end 23 | 24 | def running 25 | return unless @runner 26 | @runner.running 27 | end 28 | 29 | def run 30 | start_instrumentation 31 | super 32 | end 33 | 34 | private 35 | 36 | def start_instrumentation 37 | Thread.new do 38 | backlog_histogram = Metriks.histogram('thread_pool.backlog') 39 | running_histogram = Metriks.histogram('thread_pool.running') 40 | 41 | loop do 42 | sleep 1 43 | backlog_histogram.update(backlog) 44 | running_histogram.update(running) 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | Disallow: / 2 | -------------------------------------------------------------------------------- /puma.rb: -------------------------------------------------------------------------------- 1 | threads ENV.fetch('MIN_THREADS', 1), ENV.fetch('MAX_THREADS', 1) 2 | port ENV['PORT'] if ENV['PORT'] 3 | workers ENV.fetch('WORKER_COUNT', 1) 4 | 5 | require 'bundler_api/metriks' 6 | BundlerApi::Metriks.start 7 | 8 | require 'puma_worker_killer' 9 | PumaWorkerKiller.enable_rolling_restart 10 | 11 | on_worker_boot do |index| 12 | BundlerApi::Metriks.start(index) 13 | end 14 | -------------------------------------------------------------------------------- /script/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | lib = File.expand_path(File.join('..', '..', 'lib'), __FILE__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | 6 | require 'bundler/setup' 7 | require 'bundler_api/env' 8 | 9 | def database_url 10 | ENV['DATABASE_URL'] 11 | end 12 | 13 | def db 14 | @db ||= Sequel.connect(ENV['DATABASE_URL']) 15 | end 16 | 17 | require 'sequel' 18 | require 'json' 19 | require 'bundler_api' 20 | require 'compact_index' 21 | require 'bundler_api/checksum' 22 | require 'bundler_api/gem_info' 23 | 24 | require 'pry' 25 | Pry.start -------------------------------------------------------------------------------- /script/integration/migrate.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | lib = File.expand_path(File.join('..', '..', '..', 'lib'), __FILE__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | 6 | require 'bundler/setup' 7 | require 'bundler_api/env' 8 | require 'sequel' 9 | require 'compact_index' 10 | require 'bundler_api/gem_info' 11 | 12 | ## helpers 13 | 14 | def database_name(database_url) 15 | File.basename(database_url) 16 | end 17 | 18 | def get_temp_database_url 19 | database_url = ENV['DATABASE_URL'] 20 | abort 'DATABASE_URL environment variable required' unless database_url 21 | database_url + Time.now.to_i.to_s 22 | end 23 | 24 | ## main commands 25 | 26 | def drop_database(database_url) 27 | puts 'Dropping database' 28 | puts `dropdb --if-exists #{database_name(database_url)}` 29 | end 30 | 31 | def create_database(database_url) 32 | puts 'Creating database' 33 | puts `createdb --no-password #{database_name(database_url)}` 34 | end 35 | 36 | def import(database_url, sql_file) 37 | puts "Importing #{sql_file} data" 38 | puts `psql -d #{database_name(database_url)} -c "CREATE EXTENSION hstore"` 39 | puts `psql -d #{database_name(database_url)} < #{sql_file}` 40 | end 41 | 42 | def migrate_checksums(temp_database_url, database_url) 43 | temp_db = Sequel.connect temp_database_url 44 | db = Sequel.connect database_url 45 | 46 | puts "migrating checksum" 47 | 48 | versions_with_nil_checksum = db[:rubygems] 49 | .join(:versions, rubygem_id: :id) 50 | .where(checksum: nil) 51 | 52 | versions_with_nil_checksum.each do |entry| 53 | checksum_entry = temp_db[:versions] 54 | .select_append(:versions__created_at___versions_created_at) 55 | .join(:rubygems, id: :rubygem_id) 56 | .where(name: entry[:name], number: entry[:number]) 57 | .first 58 | if checksum_entry 59 | checksum = checksum_entry[:sha256] 60 | created_at = checksum_entry[:versions_created_at] 61 | db[:versions].where(id: entry[:id]).update(checksum: checksum, created_at: created_at) 62 | end 63 | end 64 | end 65 | 66 | def migrate_info_checksums(database_url) 67 | puts "migrating info_checksum" 68 | db = Sequel.connect database_url 69 | 70 | gem_info = BundlerApi::GemInfo.new(db) 71 | 72 | versions_with_nil_info_checksum = db[:rubygems] 73 | .select_append(:versions__id___version_id) 74 | .join(:versions, rubygem_id: :id) 75 | .where(info_checksum: nil) 76 | 77 | info_checksums = {} # table with info_checksum per gem name 78 | 79 | versions_with_nil_info_checksum.each do |entry| 80 | name = entry[:name] 81 | unless info_checksums[name] 82 | info_checksums[name] = Digest::MD5.hexdigest(gem_info.info(name)) 83 | end 84 | db[:versions].where(id: entry[:version_id]).update(info_checksum: info_checksums[name]) 85 | end 86 | end 87 | 88 | def migrate_created_at(database_url) 89 | puts "migrating created_at" 90 | db = Sequel.connect database_url 91 | 92 | gem_info = BundlerApi::GemInfo.new(db) 93 | 94 | versions_with_nil_created_at = db[:rubygems] 95 | .select_append(:versions__id___version_id) 96 | .join(:versions, rubygem_id: :id) 97 | .where(created_at: nil, indexed: true) 98 | 99 | require 'open-uri' 100 | require 'json' 101 | by_name = Hash.new do |h, name| 102 | json = JSON.load open("https://rubygems.org/api/v1/versions/#{name}.json") 103 | h[name] = json 104 | end 105 | 106 | now = Time.now 107 | versions_with_nil_created_at.each do |entry| 108 | id, name, version = entry.fetch_values(:id, :name, :number) 109 | versions = by_name[name] 110 | created_at = versions.find { |v| v["number"] == version }["created_at"] 111 | created_at = created_at ? Time.parse(created_at) : now 112 | created_at = now if now < created_at 113 | db[:versions].where(id: entry[:version_id]).update(created_at: created_at) 114 | end 115 | end 116 | 117 | def migrate_prerelease(database_url) 118 | puts "migrating prerelease" 119 | db = Sequel.connect database_url 120 | pre_release_versions = db[:versions].where(number: /[a-zA-Z]/) 121 | pre_release_versions.update(prerelease: true) 122 | non_pre_release_versions = db[:versions].exclude(number: /[a-zA-Z]/) 123 | non_pre_release_versions.update(prerelease: false) 124 | end 125 | 126 | ## main 127 | 128 | sql_file = ARGV.first 129 | temp_database_url = get_temp_database_url 130 | database_url = ENV['DATABASE_URL'] 131 | 132 | drop_database(temp_database_url) 133 | create_database(temp_database_url) 134 | begin 135 | import(temp_database_url,sql_file) 136 | migrate_checksums(temp_database_url, database_url) 137 | migrate_info_checksums(database_url) 138 | migrate_created_at(database_url) 139 | migrate_prerelease(database_url) 140 | ensure 141 | drop_database(temp_database_url) 142 | end 143 | -------------------------------------------------------------------------------- /script/integration/test.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | require 'open-uri' 4 | require 'set' 5 | require 'digest' 6 | 7 | names = Set.new 8 | 9 | #url = "bundler-api-staging.herokuapp.com" 10 | url = "localhost:5000" 11 | open("http://#{url}/versions").readlines.reverse_each do |line| 12 | name, *_, sum = line.split(' ') 13 | next unless names.add?(name) 14 | info_sum = Digest::MD5.hexdigest open("http://#{url}/info/#{name}").read 15 | unless sum == info_sum 16 | puts name 17 | puts info_sum 18 | puts sum 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /script/migrate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Create and migrate the database specified in the $DATABASE_URL environment 3 | # variable. 4 | # 5 | # Usage: script/migrate [version] 6 | # 7 | # Options: 8 | # version: migrate the database to version given 9 | 10 | $stdout.sync = true 11 | 12 | lib = File.expand_path(File.join('..', '..', 'lib'), __FILE__) 13 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 14 | 15 | require 'bundler/setup' 16 | require 'bundler_api/env' 17 | 18 | def database_url 19 | ENV['DATABASE_URL'] 20 | end 21 | 22 | def version 23 | ARGV.first 24 | end 25 | 26 | abort 'DATABASE_URL environment variable required' unless database_url 27 | 28 | puts 'Migrating database' 29 | command = *%w{sequel --migrate-directory db/migrations} 30 | command += %W{--migrate-version #{version}} if version 31 | command << database_url 32 | system *command 33 | -------------------------------------------------------------------------------- /script/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Create and migrate the database specified in the $DATABASE_URL environment 3 | # variable. 4 | # 5 | # Usage: script/setup [--verbose] [--rebuild] 6 | # 7 | # Options: 8 | # --rebuild: drop the database before creating it 9 | # --verbose: print errors and warnings from postgres 10 | 11 | $stdout.sync = true 12 | 13 | lib = File.expand_path(File.join('..', '..', 'lib'), __FILE__) 14 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 15 | 16 | require 'bundler/setup' 17 | require 'bundler_api/env' 18 | require 'bundler_api/cache' 19 | require 'bundler_api/redis' 20 | require 'sequel' 21 | require 'open3' 22 | 23 | def rebuild? 24 | @rebuild ||= ARGV.delete('--rebuild') 25 | end 26 | 27 | def verbose? 28 | @verbose ||= (ARGV.delete('-v') || ARGV.delete('--verbose')) 29 | end 30 | 31 | def run_command(*args) 32 | output, status = Open3.capture2e(*args) 33 | print_output(output, !status.success? || verbose?) 34 | abort unless status.success? 35 | end 36 | 37 | def print_output(output, verbose) 38 | return if !verbose || output.empty? 39 | output.lines.each do |line| 40 | puts " #{line}" 41 | end 42 | puts 43 | end 44 | 45 | def print_error(title, error) 46 | return unless verbose? 47 | $stderr.puts "#{title}: #{error}" 48 | 49 | error.backtrace.each do |line| 50 | $stderr.puts " #{line}" 51 | end 52 | end 53 | 54 | # must be a valid uri, e.g 55 | # postgres://user:pass@host:80/path 56 | def database_url 57 | ENV['DATABASE_URL'] 58 | end 59 | 60 | def conn_info 61 | uri = URI.parse database_url 62 | params = [] 63 | params.concat ["--host", uri.host] if uri.host 64 | params.concat ["--port", uri.port.to_s] if uri.port 65 | params.concat ["--username", uri.user] if uri.user 66 | params.concat ["--password", uri.password] if uri.password 67 | params 68 | end 69 | 70 | def database_name 71 | File.basename(database_url) 72 | end 73 | 74 | def postgres_installed? 75 | !`which psql`.strip.empty? 76 | end 77 | 78 | def database_exists? 79 | Sequel.connect(database_url) do |db| 80 | db.test_connection 81 | end 82 | 83 | true 84 | rescue 85 | false 86 | end 87 | 88 | def postgres_ready? 89 | Sequel.connect(database_url) do |db| 90 | db[:versions].first 91 | end 92 | 93 | true 94 | rescue => e 95 | print_error "Postgres error", e 96 | false 97 | end 98 | 99 | def redis_ready? 100 | BundlerApi.redis.ping 101 | true 102 | rescue => e 103 | print_error "Redis error", e 104 | false 105 | end 106 | 107 | def memcached_ready? 108 | BundlerApi::CacheInvalidator.new.memcached_client.alive! 109 | true 110 | rescue => e 111 | print_error "Memcached error", e 112 | false 113 | end 114 | 115 | def verify_dependencies 116 | puts 'Checking dependencies' 117 | unmet = [] 118 | unmet << 'postgres' unless postgres_ready? 119 | unmet << 'redis' unless redis_ready? 120 | unmet << 'memcached' unless memcached_ready? 121 | 122 | unless unmet.empty? 123 | $stderr.puts 'You have dependencies that are unmet or are not available:' 124 | 125 | unmet.each do |dependency| 126 | $stderr.puts " * #{dependency}" 127 | end 128 | end 129 | end 130 | 131 | abort 'DATABASE_URL environment variable required' unless database_url 132 | 133 | if rebuild? 134 | puts 'Dropping database' 135 | run_command(*['dropdb', '--if-exists', conn_info, database_name].flatten) 136 | end 137 | 138 | unless database_exists? 139 | unless postgres_installed? 140 | abort 'Please install postgresql or specify a connection to an existing database in .env.local' 141 | end 142 | 143 | puts "Creating database: #{database_url}" 144 | run_command(*['createdb', conn_info, database_name].flatten) 145 | end 146 | 147 | puts 'Migrating database' 148 | run_command(*%W{sequel --migrate-directory db/migrations #{database_url}}) 149 | 150 | verify_dependencies 151 | 152 | puts " 153 | Done! \ 154 | Run `rake update` and `rake fix_deps` to populate the database with \ 155 | all gems from rubygems.org." 156 | -------------------------------------------------------------------------------- /script/web: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | lib = File.expand_path('../../lib', __FILE__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | 6 | require 'bundler/setup' 7 | require 'bundler_api/env' 8 | require 'bundler_api/metriks' 9 | require 'puma/instrumented_cli' 10 | 11 | cli = Puma::InstrumentedCLI.new(ARGV + %w(--config puma.rb)) 12 | cli.run 13 | -------------------------------------------------------------------------------- /spec/agent_reporting_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'bundler_api/agent_reporting' 3 | 4 | describe BundlerApi::AgentReporting do 5 | class FakeMetriks 6 | attr_accessor :values, :key 7 | def initialize; @values = Hash.new { |hash, key| hash[key] = 0 } end 8 | def mark; @values[key] += 1 end 9 | end 10 | 11 | let(:app) { double(call: true) } 12 | let(:middleware) { described_class.new(app) } 13 | let(:metriks) { FakeMetriks.new } 14 | let(:redis) { double(exists: false, setex: true) } 15 | let(:env) { {'HTTP_USER_AGENT' => ua} } 16 | let(:ua) do 17 | [ 'bundler/1.7.3', 18 | 'rubygems/2.4.1', 19 | 'ruby/2.1.2', 20 | '(x86_64-apple-darwin13.2.0)', 21 | 'command/update', 22 | 'options/jobs,without,build.mysql', 23 | 'ci/jenkins,ci', 24 | '9d16bd9809d392ca' ].join(' ') 25 | end 26 | 27 | before do 28 | Metriks.stub(:meter) { |key| metriks.key = key; metriks } 29 | BundlerApi.stub(:redis => redis) 30 | middleware.call(env) 31 | end 32 | 33 | context "with options" do 34 | describe 'reporting metrics (valid UA)' do 35 | it 'should report the right values' do 36 | expect( metriks ).to be_incremented_for('versions.bundler.1.7.3') 37 | expect( metriks ).to be_incremented_for('versions.rubygems.2.4.1') 38 | expect( metriks ).to be_incremented_for('versions.ruby.2.1.2') 39 | expect( metriks ).to be_incremented_for('commands.update') 40 | expect( metriks ).to be_incremented_for('archs.x86_64-apple-darwin13.2.0') 41 | expect( metriks ).to be_incremented_for('options.jobs') 42 | expect( metriks ).to be_incremented_for('options.without') 43 | expect( metriks ).to be_incremented_for('options.build.mysql') 44 | expect( metriks ).to be_incremented_for('cis.jenkins') 45 | expect( metriks ).to be_incremented_for('cis.ci') 46 | end 47 | end 48 | 49 | describe 'reporting metrics (invalid UA)' do 50 | let(:ua) { 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)' } 51 | it 'should not report anything' do 52 | expect( metriks.values ).to be_empty 53 | end 54 | end 55 | end 56 | 57 | context "without options" do 58 | let(:ua) do 59 | [ 'bundler/1.7.3', 60 | 'rubygems/2.4.1', 61 | 'ruby/2.1.2', 62 | '(x86_64-apple-darwin13.2.0)', 63 | 'command/update', 64 | 'ci/semaphore', 65 | '9d16bd9809d392ca' ].join(' ') 66 | end 67 | 68 | describe 'weird version number' do 69 | let(:ua) { super().sub('bundler/1.7.3', 'bundler/1.10.4.beta.1') } 70 | it 'increments double-digit bundler versions' do 71 | expect( metriks ).to be_incremented_for('versions.bundler.1.10.4.beta.1') 72 | end 73 | end 74 | 75 | describe 'reporting metrics (valid UA, first time)' do 76 | it 'should report the right values' do 77 | expect( metriks ).to be_incremented_for('versions.bundler.1.7.3') 78 | expect( metriks ).to be_incremented_for('versions.rubygems.2.4.1') 79 | expect( metriks ).to be_incremented_for('versions.ruby.2.1.2') 80 | expect( metriks ).to be_incremented_for('commands.update') 81 | expect( metriks ).to be_incremented_for('archs.x86_64-apple-darwin13.2.0') 82 | expect( metriks ).to be_incremented_for('cis.semaphore') 83 | end 84 | end 85 | 86 | describe 'reporting metrics (valid UA, return customer)' do 87 | let(:redis) { double(exists: true) } 88 | 89 | it 'should not report anything' do 90 | expect( metriks.values ).to be_empty 91 | end 92 | end 93 | 94 | describe 'reporting metrics (invalid UA)' do 95 | let(:ua) { 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)' } 96 | it 'should not report anything' do 97 | expect( metriks.values ).to be_empty 98 | end 99 | end 100 | end 101 | 102 | context "when Redis breaks" do 103 | before do 104 | redis.stub(:exists).and_raise(Redis::CannotConnectError) 105 | end 106 | 107 | it "should not raise an exception" do 108 | expect { middleware.call(env) }.not_to raise_error 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /spec/cache_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'bundler_api/cache' 3 | 4 | describe BundlerApi::CacheInvalidator do 5 | let(:client) { double(:client, purge_path: nil, purge_key: nil) } 6 | let(:cache) { BundlerApi::CacheInvalidator.new(cdn: client, silent: true) } 7 | let(:failing_response) { double('Net::HTTPServerError', :uri => 'URI', :code => 500, :body => 'fail!') } 8 | let(:response) { double('Net::HTTPOK', :code => 200, :body => '') } 9 | 10 | describe '.purge_specs' do 11 | subject { cache.purge_specs } 12 | 13 | it 'purges dependencies key' do 14 | expect(client).to receive(:purge_key).with('dependencies').and_return(response) 15 | subject 16 | end 17 | 18 | it 'purges latest specs' do 19 | expect(client).to receive(:purge_path).with('/latest_specs.4.8.gz').and_return(response) 20 | subject 21 | end 22 | 23 | it 'purges specs' do 24 | expect(client).to receive(:purge_path).with('/specs.4.8.gz').and_return(response) 25 | subject 26 | end 27 | 28 | it 'purges prerelease specs' do 29 | expect(client).to receive(:purge_path).with('/prerelease_specs.4.8.gz').and_return(response) 30 | subject 31 | end 32 | 33 | it 'purges names' do 34 | expect(client).to receive(:purge_path).with('/names').and_return(response) 35 | subject 36 | end 37 | 38 | it 'purges versions' do 39 | expect(client).to receive(:purge_path).with('/versions').and_return(response) 40 | subject 41 | end 42 | 43 | it 'raises when a purge returns a failing HTTP response' do 44 | expect(client).to receive(:purge_path).with('/versions').and_return(failing_response) 45 | expect { subject }.to raise_error <<-E.strip 46 | The following cache purge requests failed: 47 | - URI => 500, fail! 48 | E 49 | end 50 | 51 | context 'with a nil client' do 52 | let(:client) { nil } 53 | 54 | it 'does nothing' do 55 | expect { subject }.to_not raise_error 56 | end 57 | end 58 | end 59 | 60 | describe '.purge_gem' do 61 | let(:name) { 'bundler' } 62 | let(:gem_helper) { BundlerApi::GemHelper.new(name, '1.0.0', 'ruby', false) } 63 | subject { cache.purge_gem(gem_helper) } 64 | 65 | it 'purges gemspec' do 66 | expect(client).to receive(:purge_path) 67 | .with('/quick/Marshal.4.8/bundler-1.0.0.gemspec.rz') 68 | .and_return(response) 69 | subject 70 | end 71 | 72 | it 'purges gem' do 73 | expect(client).to receive(:purge_path) 74 | .with('/gems/bundler-1.0.0.gem') 75 | .and_return(response) 76 | subject 77 | end 78 | 79 | it 'purges new index info' do 80 | expect(client).to receive(:purge_path) 81 | .with('/info/bundler') 82 | .and_return(response) 83 | subject 84 | end 85 | 86 | it 'raises when a purge returns a failing HTTP response' do 87 | expect(client).to receive(:purge_path) 88 | .with('/info/bundler') 89 | .and_return(failing_response) 90 | expect { subject }.to raise_error <<-E.strip 91 | The following cache purge requests failed: 92 | - URI => 500, fail! 93 | E 94 | end 95 | 96 | it "purges memcached gem" do 97 | cache.memcached_client.set("deps/v1/#{name}", "omg!") 98 | expect(cache.memcached_client.get("deps/v1/#{name}")).to_not be_nil 99 | subject 100 | expect(cache.memcached_client.get("deps/v1/#{name}")).to be_nil 101 | end 102 | 103 | context 'with a nil client' do 104 | let(:client) { nil } 105 | 106 | it 'does nothing' do 107 | expect { subject }.to_not raise_error 108 | end 109 | end 110 | end 111 | 112 | describe '.purge_memory_cache' do 113 | let(:name) { 'bundler-1.0.0' } 114 | subject { cache.purge_memory_cache(name) } 115 | 116 | it 'purge memcached gem api' do 117 | cache.memcached_client.set("deps/v1/#{name}", "omg!") 118 | expect(cache.memcached_client.get("deps/v1/#{name}")).to_not be_nil 119 | subject 120 | expect(cache.memcached_client.get("deps/v1/#{name}")).to be_nil 121 | end 122 | 123 | it 'purge memcached gem info' do 124 | cache.memcached_client.set("info/#{name}", "omg!") 125 | expect(cache.memcached_client.get("info/#{name}")).to_not be_nil 126 | subject 127 | expect(cache.memcached_client.get("info/#{name}")).to be_nil 128 | end 129 | 130 | it 'purge memcached gem names' do 131 | cache.memcached_client.set("names", "omg!") 132 | expect(cache.memcached_client.get("names")).to_not be_nil 133 | subject 134 | expect(cache.memcached_client.get("names")).to be_nil 135 | end 136 | end 137 | end 138 | -------------------------------------------------------------------------------- /spec/gem_helper_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'support/artifice_apps' 3 | require 'bundler_api/gem_helper' 4 | 5 | describe BundlerApi::GemHelper do 6 | let(:name) { "foo" } 7 | let(:version) { "1.0" } 8 | let(:platform) { "ruby" } 9 | let(:helper) { BundlerApi::GemHelper.new(name, version, platform) } 10 | 11 | describe "#full_name" do 12 | context "when the platform is not ruby" do 13 | let(:platform) { "java" } 14 | 15 | it "prints out the platform" do 16 | expect(helper.full_name).to eq("foo-1.0-java") 17 | end 18 | end 19 | 20 | context "when the platform is ruby" do 21 | it "doesn't print out the platform" do 22 | expect(helper.full_name).to eq("foo-1.0") 23 | end 24 | end 25 | end 26 | 27 | describe "#download_spec" do 28 | let(:gemspec) { 29 | eval(<<-GEMSPEC) 30 | Gem::Specification.new do |s| 31 | s.name = "#{name}" 32 | s.version = "#{version}" 33 | s.platform = "#{platform}" 34 | 35 | s.authors = ["Terence Lee"] 36 | s.date = "2010-10-24" 37 | s.description = "Foo" 38 | s.email = "foo@example.com" 39 | s.homepage = "http://www.foo.com" 40 | s.require_paths = ["lib"] 41 | s.rubyforge_project = "foo" 42 | s.summary = "Foo" 43 | end 44 | GEMSPEC 45 | } 46 | 47 | context "when no redirect" do 48 | before do 49 | Artifice.activate_with(GemspecGenerator) 50 | end 51 | 52 | after do 53 | Artifice.deactivate 54 | end 55 | 56 | it "returns the gemspec" do 57 | expect(helper.download_spec).to eq(gemspec) 58 | end 59 | end 60 | 61 | context "when there's a redirect" do 62 | before do 63 | Artifice.activate_with(GemspecRedirect) 64 | end 65 | 66 | after do 67 | Artifice.deactivate 68 | end 69 | 70 | it "returns the gemspec" do 71 | expect(helper.download_spec).to eq(gemspec) 72 | end 73 | end 74 | 75 | context "when we keep redirecting" do 76 | before do 77 | Artifice.activate_with(ForeverRedirect) 78 | end 79 | 80 | after do 81 | Artifice.deactivate 82 | end 83 | 84 | it "should not go on forever" do 85 | expect { helper.download_spec }.to raise_error(BundlerApi::HTTPError) 86 | end 87 | end 88 | 89 | context "when there's a http error" do 90 | before do 91 | Artifice.activate_with(GemspecHTTPError) 92 | end 93 | 94 | after do 95 | Artifice.deactivate 96 | end 97 | 98 | it "retries in case it's a hiccup" do 99 | expect(helper.download_spec).to eq(gemspec) 100 | end 101 | end 102 | 103 | context "when it's always throwing an error" do 104 | before do 105 | Artifice.activate_with(ForeverHTTPError) 106 | end 107 | 108 | after do 109 | Artifice.deactivate 110 | end 111 | 112 | it "raises an error" do 113 | expect { helper.download_spec }.to raise_error(BundlerApi::HTTPError) 114 | end 115 | end 116 | 117 | context "when using multiple threads" do 118 | let(:version) { "1" } 119 | let(:port) { 20000 } 120 | 121 | Thread.abort_on_exception = true 122 | 123 | before do 124 | @rackup_thread = Thread.new { 125 | server = Rack::Server.new(:app => NonThreadSafeGenerator, 126 | :Host => '0.0.0.0', 127 | :Port => port, 128 | :server => 'webrick', 129 | :AccessLog => [], 130 | :Logger => Logger.new("/dev/null")) 131 | server.start 132 | } 133 | @rackup_thread.run 134 | 135 | # ensure server is started 136 | require 'timeout' 137 | Timeout.timeout(15) { sleep(0.1) until @rackup_thread.status == "sleep" } 138 | sleep(1) 139 | ENV["DOWNLOAD_BASE"] = "http://localhost:#{port}" 140 | end 141 | 142 | after do 143 | @rackup_thread.kill 144 | end 145 | 146 | it "is threadsafe" do 147 | 5.times.map do 148 | Thread.new { helper.download_spec } 149 | end.each do |t| 150 | expect(t.value).to eq(gemspec) 151 | end 152 | end 153 | end 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /spec/gem_info_spec.rb: -------------------------------------------------------------------------------- 1 | require 'sequel' 2 | require 'spec_helper' 3 | require 'bundler_api/gem_info' 4 | require 'support/gem_builder' 5 | 6 | describe BundlerApi::GemInfo do 7 | let(:db) { $db } 8 | let(:builder) { GemBuilder.new(db) } 9 | let(:gem_info) { BundlerApi::GemInfo.new(db) } 10 | 11 | describe "#deps_for" do 12 | context "no gems" do 13 | it "should find the deps" do 14 | expect(gem_info.deps_for('rack')).to eq([]) 15 | end 16 | end 17 | 18 | context "no dependencies" do 19 | before do 20 | rack_id = builder.create_rubygem('rack') 21 | builder.create_version(rack_id, 'rack') 22 | end 23 | 24 | it "should return rack" do 25 | result = { 26 | name: 'rack', 27 | number: '1.0.0', 28 | platform: 'ruby' 29 | } 30 | 31 | result.each_pair do |k, v| 32 | expect(gem_info.deps_for('rack').first[k]).to eq(v) 33 | end 34 | end 35 | end 36 | 37 | context "has one dependency" do 38 | before do 39 | tomorrow = Time.at(Time.now.to_i + 86400) 40 | 41 | rack_id = builder.create_rubygem('rack') 42 | rack_version_id = builder.create_version(rack_id, 'rack') 43 | rack_version_id2 = builder.create_version(rack_id, 'rack', '1.1.9', 'ruby', time: tomorrow) 44 | rack_version_id2 = builder.create_version(rack_id, 'rack', '1.2.0', 'ruby', time: tomorrow) 45 | 46 | foo_id = builder.create_rubygem('foo') 47 | builder.create_version(foo_id, 'foo') 48 | builder.create_dependency(foo_id, rack_version_id, "= 1.0.0") 49 | end 50 | 51 | it "should return foo as a dep of rack" do 52 | result = { 53 | name: 'rack', 54 | number: '1.0.0', 55 | platform: 'ruby', 56 | dependencies: [['foo', '= 1.0.0']] 57 | } 58 | 59 | result.each_pair do |k,v| 60 | expect(gem_info.deps_for('rack').first[k]).to eq(v) 61 | end 62 | end 63 | 64 | it "order by created_at and version number" do 65 | result = %w(1.0.0 1.1.9 1.2.0) 66 | expect(gem_info.deps_for('rack').map { |x| x[:number] }).to eq(result) 67 | end 68 | end 69 | 70 | context "filters on indexed" do 71 | before do 72 | rack_id = builder.create_rubygem('rack') 73 | rack_version_id = builder.create_version(rack_id, 'rack', '1.1.0') 74 | non_indexed_rack_version_id = builder.create_version(rack_id, 'rack', '1.0.0', 'ruby', { indexed: false }) 75 | 76 | foo_id = builder.create_rubygem('foo') 77 | builder.create_version(foo_id, 'foo') 78 | builder.create_dependency(foo_id, rack_version_id, "= 1.0.0") 79 | builder.create_dependency(foo_id, non_indexed_rack_version_id, "= 1.0.0") 80 | end 81 | 82 | it "should not return nonindexed gems" do 83 | result = { 84 | name: 'rack', 85 | number: '1.1.0', 86 | platform: 'ruby', 87 | dependencies: [['foo', '= 1.0.0']] 88 | } 89 | 90 | result.each_pair do |k,v| 91 | expect(gem_info.deps_for('rack').first[k]).to eq(v) 92 | end 93 | end 94 | end 95 | end 96 | 97 | describe "#names" do 98 | before do 99 | builder.create_rubygem("a") 100 | builder.create_rubygem("c") 101 | builder.create_rubygem("b") 102 | builder.create_rubygem("d") 103 | end 104 | 105 | it "should return the list back in order" do 106 | expect(gem_info.names).to eq(%w(a b c d)) 107 | end 108 | end 109 | 110 | describe "#versions" do 111 | let(:gems) do 112 | [ 113 | CompactIndex::Gem.new( 114 | 'a', 115 | [CompactIndex::GemVersion.new('1.0.0', 'ruby', nil, 'a100')] 116 | ), 117 | CompactIndex::Gem.new( 118 | 'a', 119 | [CompactIndex::GemVersion.new('1.0.1', 'ruby', nil, 'a101')] 120 | ), 121 | CompactIndex::Gem.new( 122 | 'b', 123 | [CompactIndex::GemVersion.new('1.0.0', 'ruby', nil, 'b100')] 124 | ), 125 | CompactIndex::Gem.new( 126 | 'c', 127 | [CompactIndex::GemVersion.new('1.0.0', 'java', nil, 'c100')] 128 | ), 129 | CompactIndex::Gem.new( 130 | 'a', 131 | [CompactIndex::GemVersion.new('2.0.0', 'java', nil, 'a200')] 132 | ), 133 | CompactIndex::Gem.new( 134 | 'a', 135 | [CompactIndex::GemVersion.new('2.0.1', 'ruby', nil, 'a201')] 136 | ) 137 | ] 138 | end 139 | 140 | let(:a) { a = builder.create_rubygem("a") } 141 | 142 | before do 143 | @time = Time.now 144 | builder.create_version(a, 'a', '1.0.0', 'ruby', info_checksum: 'a100') 145 | builder.create_version(a, 'a', '1.0.1', 'ruby', info_checksum: 'a101') 146 | b = builder.create_rubygem("b") 147 | builder.create_version(b, 'b', '1.0.0', 'ruby', info_checksum: 'b100') 148 | c = builder.create_rubygem("c") 149 | builder.create_version(c, 'c', '1.0.0', 'java', info_checksum: 'c100') 150 | @a200 = builder.create_version(a, 'a', '2.0.0', 'java', info_checksum: 'a200') 151 | builder.create_version(a, 'a', '2.0.1', 'ruby', info_checksum: 'a201') 152 | end 153 | 154 | it "should return gems on compact index format" do 155 | expect(gem_info.versions(@time)).to eq(gems) 156 | end 157 | 158 | context "with yanked gems" do 159 | before do 160 | builder.yank(@a200, yanked_info_checksum: 'a200y') 161 | builder.create_version(a, 'a', '2.2.2', 'ruby', info_checksum: 'a222') 162 | end 163 | 164 | let(:gems_with_yanked) do 165 | gems + [ 166 | CompactIndex::Gem.new( 167 | 'a', 168 | [CompactIndex::GemVersion.new('-2.0.0', 'java', nil, 'a200y')] 169 | ), 170 | CompactIndex::Gem.new( 171 | 'a', 172 | [CompactIndex::GemVersion.new('2.2.2', 'ruby', nil, 'a222')] 173 | ) 174 | ] 175 | end 176 | 177 | it "return yanked gems with minus version" do 178 | expect(gem_info.versions(@time, true)).to eq(gems_with_yanked) 179 | end 180 | end 181 | end 182 | 183 | describe "#info" do 184 | before do 185 | info_test = builder.create_rubygem('info_test') 186 | builder.create_version(info_test, 'info_test', '1.0.0', 'ruby', checksum: 'abc123') 187 | 188 | info_test101= builder.create_version(info_test, 'info_test', '1.0.1', 'ruby', checksum: 'qwerty') 189 | [['foo', '= 1.0.0'], ['bar', '>= 2.1, < 3.0']].each do |dep, requirements| 190 | dep_id = builder.create_rubygem(dep) 191 | builder.create_dependency(dep_id, info_test101, requirements) 192 | end 193 | end 194 | 195 | it "return compact index info for a gem" do 196 | expected = "---\n1.0.0 |checksum:abc123\n1.0.1 bar:< 3.0&>= 2.1,foo:= 1.0.0|checksum:qwerty\n" 197 | expect(gem_info.info('info_test')).to eq(expected) 198 | end 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | ENV['RACK_ENV'] = 'test' 2 | require 'bundler_api/env' 3 | 4 | require 'dalli' 5 | require 'rspec/core' 6 | require 'rspec/mocks' 7 | 8 | require 'bundler_api/gem_helper' 9 | require 'support/database' 10 | require 'support/latch' 11 | require 'support/matchers' 12 | 13 | RSpec.configure do |config| 14 | config.filter_run :focused => true 15 | config.run_all_when_everything_filtered = true 16 | config.alias_example_to :fit, :focused => true 17 | 18 | config.expect_with :rspec do |c| 19 | c.syntax = :expect 20 | end 21 | 22 | config.mock_with :rspec do |c| 23 | c.syntax = [:should, :expect] 24 | end 25 | 26 | config.before(:each) do 27 | stub_const("BundlerApi::GemHelper::TRY_BACKOFF", 0) 28 | end 29 | 30 | config.before(:each) do 31 | Dalli::Client.new.flush 32 | end 33 | 34 | config.raise_errors_for_deprecations! 35 | end 36 | -------------------------------------------------------------------------------- /spec/support/artifice_apps.rb: -------------------------------------------------------------------------------- 1 | require 'artifice' 2 | require 'sinatra/base' 3 | require 'support/gemspec_helper' 4 | 5 | class GemspecGenerator < Sinatra::Base 6 | include GemspecHelper 7 | 8 | get "/quick/Marshal.4.8/*" do 9 | name, version, platform = parse_splat(params[:splat].first) 10 | if platform.nil? 11 | platform == "ruby" 12 | elsif platform == "jruby" 13 | platform == "java" 14 | end 15 | Gem.deflate(Marshal.dump(generate_gemspec(name, version, platform))) 16 | end 17 | 18 | get "/api/v2/rubygems/:name/versions/:version.json" do 19 | JSON.dump(name: params[:name], version: params[:version], sha: "abc123") 20 | end 21 | end 22 | 23 | class GemspecRedirect < Sinatra::Base 24 | include GemspecHelper 25 | 26 | get "/quick/Marshal.4.8/*" do 27 | redirect "/real/#{params[:splat].first}" 28 | end 29 | 30 | get "/real/*" do 31 | name, version, platform = parse_splat(params[:splat].first) 32 | platform ||= 'ruby' 33 | Gem.deflate(Marshal.dump(generate_gemspec(name, version, platform))) 34 | end 35 | end 36 | 37 | class ForeverRedirect < Sinatra::Base 38 | get "/quick/Marshal.4.8/*" do 39 | redirect "/quick/Marshal.4.8/#{params[:splat].first}" 40 | end 41 | end 42 | 43 | class GemspecHTTPError < Sinatra::Base 44 | include GemspecHelper 45 | 46 | def initialize(*) 47 | super 48 | @@run = false 49 | end 50 | 51 | get "/quick/Marshal.4.8/*" do 52 | if @@run 53 | name, version, platform = parse_splat(params[:splat].first) 54 | platform ||= 'ruby' 55 | Gem.deflate(Marshal.dump(generate_gemspec(name, version, platform))) 56 | else 57 | status 500 58 | @@run = true 59 | "OMG ERROR" 60 | end 61 | end 62 | end 63 | 64 | class ForeverHTTPError < Sinatra::Base 65 | get "/quick/Marshal.4.8/*" do 66 | status 500 67 | "OMG ERROR" 68 | end 69 | end 70 | 71 | class NonThreadSafeGenerator < Sinatra::Base 72 | include GemspecHelper 73 | 74 | def initialize(*) 75 | super 76 | 77 | @@counter = 0 78 | @@mutex = Mutex.new 79 | end 80 | 81 | get "/quick/Marshal.4.8/*" do 82 | @@counter += 1 83 | 84 | name, version, platform = parse_splat(params[:splat].first) 85 | if platform.nil? 86 | platform == "ruby" 87 | elsif platform == "jruby" 88 | platform == "java" 89 | end 90 | Gem.deflate(Marshal.dump(generate_gemspec(name, @@counter, platform))) 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /spec/support/database.rb: -------------------------------------------------------------------------------- 1 | require 'sequel' 2 | 3 | RSpec.configure do |config| 4 | config.before(:suite) do 5 | db_url = ENV["TEST_DATABASE_URL"] 6 | fail 'TEST_DATABASE_URL is required' if db_url.nil? || db_url.empty? 7 | 8 | # Drop and recreate the database 9 | Sequel.connect(ENV["TEST_DATABASE_ADMIN_URL"]) do |db| 10 | db_name = URI.parse(ENV["TEST_DATABASE_URL"]).path[1..-1] 11 | db.run("DROP DATABASE IF EXISTS #{db_name.inspect}") 12 | db.run("CREATE DATABASE #{db_name.inspect}") 13 | end 14 | 15 | # TODO: Replace global with singleton 16 | $db = Sequel.connect(db_url) 17 | Sequel.extension :migration 18 | Sequel::Migrator.run($db, 'db/migrations') 19 | end 20 | 21 | config.around(:each) do |example| 22 | $db.transaction(:rollback => :always) { example.run } 23 | $db.disconnect 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/support/etag.rb: -------------------------------------------------------------------------------- 1 | RSpec.shared_examples "return 304 on second hit" do 2 | describe "on second hit" do 3 | it "returns 304" do 4 | get url 5 | etag = last_response.header["ETag"] 6 | 7 | get url, {}, "HTTP_IF_NONE_MATCH" => etag 8 | expect(last_response.status).to eq(304) 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/support/gem_builder.rb: -------------------------------------------------------------------------------- 1 | class GemBuilder 2 | def initialize(conn) 3 | @conn = conn 4 | end 5 | 6 | def create_rubygem(name) 7 | @conn[:rubygems].insert(name: name) 8 | end 9 | 10 | def rubygem_id(name) 11 | @conn[:rubygems].select(:id).where(name: name) 12 | end 13 | 14 | def yank(version_id, yanked_info_checksum:) 15 | @conn[:versions].where(id: version_id).update(indexed: false, yanked_at: Time.now) 16 | @conn[:versions].where(id: version_id).update(yanked_info_checksum: yanked_info_checksum) 17 | end 18 | 19 | def create_version(rubygem_id, name, version = '1.0.0', platform = 'ruby', extra_args = {}) 20 | args = { 21 | indexed: true, 22 | time: Time.now, 23 | required_ruby: nil, 24 | rubygems_version: nil, 25 | info_checksum: nil, 26 | checksum: nil 27 | }.merge(extra_args) 28 | 29 | full_name = "#{name}-#{version}" 30 | full_name << "-#{platform}" if platform != 'ruby' 31 | @conn[:versions].insert( 32 | number: version, 33 | rubygem_id: rubygem_id, 34 | platform: platform, 35 | indexed: args[:indexed], 36 | prerelease: false, 37 | full_name: full_name, 38 | created_at: args[:time], 39 | required_ruby_version: args[:required_ruby], 40 | rubygems_version: args[:rubygems_version], 41 | checksum: args[:checksum], 42 | info_checksum: args[:info_checksum] 43 | ) 44 | end 45 | 46 | def create_dependency(rubygem_id, version_id, requirements, scope = 'runtime') 47 | @conn[:dependencies].insert( 48 | requirements: requirements, 49 | rubygem_id: rubygem_id, 50 | version_id: version_id, 51 | scope: scope, 52 | ) 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /spec/support/gemspec_helper.rb: -------------------------------------------------------------------------------- 1 | module GemspecHelper 2 | private 3 | def generate_gemspec(name, version, platform = 'ruby') 4 | eval(< 1.0") if name == "foo" 20 | s.add_development_dependency("#{name}-dev", ">= 1.0") 21 | yield(s) if block_given? 22 | end 23 | GEMSPEC 24 | end 25 | 26 | def parse_splat(splat) 27 | splat.sub('.gemspec.rz', '').split('-') 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /spec/support/latch.rb: -------------------------------------------------------------------------------- 1 | require 'monitor' 2 | 3 | class Latch 4 | def initialize(count = 1) 5 | @monitor = Monitor.new 6 | @cv = @monitor.new_cond 7 | @count = count 8 | end 9 | 10 | def wait 11 | @monitor.synchronize do 12 | @cv.wait_until { @count > 0 } 13 | end 14 | end 15 | 16 | def release 17 | @monitor.synchronize do 18 | @count -= 1 if @count > 0 19 | @cv.broadcast if @count.zero? 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/support/matchers.rb: -------------------------------------------------------------------------------- 1 | RSpec::Matchers.define :be_incremented_for do |expected| 2 | match do |actual| 3 | actual.values[expected] > 0 4 | end 5 | 6 | failure_message do |actual| 7 | "expected '#{ expected }' to be incremented, but it wasn't" 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /spec/update/atomic_counter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'bundler_api/update/atomic_counter' 3 | 4 | describe BundlerApi::AtomicCounter do 5 | let(:counter) { BundlerApi::AtomicCounter.new } 6 | 7 | it "starts at 0" do 8 | expect(counter.count).to eq(0) 9 | end 10 | 11 | it "increments by 1" do 12 | counter.increment 13 | 14 | expect(counter.count).to eq(1) 15 | end 16 | 17 | it "can increment more than once" do 18 | num = 12 19 | num.times { counter.increment } 20 | 21 | expect(counter.count).to eq(12) 22 | end 23 | 24 | # need to run this test in JRuby 25 | it "is atomic" do 26 | max = 10 27 | # need to create a new counter for JRuby 28 | counter = BundlerApi::AtomicCounter.new 29 | 30 | (1..max).map do 31 | Thread.new { counter.increment } 32 | end.each(&:join) 33 | 34 | expect(counter.count).to eq(max) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/update/consumer_pool_spec.rb: -------------------------------------------------------------------------------- 1 | require 'monitor' 2 | require 'timeout' 3 | require 'spec_helper' 4 | require 'bundler_api/update/consumer_pool' 5 | 6 | describe BundlerApi::ConsumerPool do 7 | class TestJob 8 | @@counter = 0 9 | 10 | def self.counter 11 | @@counter 12 | end 13 | 14 | def run 15 | @@counter += 1 16 | end 17 | end 18 | 19 | class FirstJob 20 | attr_reader :ran 21 | 22 | def initialize(test_latch, job_latch) 23 | @test_latch = test_latch 24 | @job_latch = job_latch 25 | @ran = false 26 | end 27 | 28 | def run 29 | @test_latch.release 30 | @job_latch.wait 31 | @ran = true 32 | end 33 | end 34 | 35 | class SecondJob 36 | attr_reader :ran 37 | 38 | def initialize(latch) 39 | @latch = latch 40 | @ran = false 41 | end 42 | 43 | def run 44 | @latch.release 45 | @ran = true 46 | end 47 | end 48 | 49 | it "stops the pool" do 50 | pool = BundlerApi::ConsumerPool.new(1) 51 | pool.start 52 | pool.poison 53 | pool.enq(TestJob.new) 54 | pool.join 55 | 56 | expect(TestJob.counter).to eq(0) 57 | end 58 | 59 | it "processes jobs" do 60 | pool = BundlerApi::ConsumerPool.new(1) 61 | pool.enq(TestJob.new) 62 | pool.start 63 | pool.poison 64 | pool.join 65 | 66 | expect(TestJob.counter).to eq(1) 67 | end 68 | 69 | it "works concurrently" do 70 | job_latch = Latch.new 71 | test_latch = Latch.new 72 | pool = BundlerApi::ConsumerPool.new(2) 73 | job1 = FirstJob.new(test_latch, job_latch) 74 | job2 = SecondJob.new(job_latch) 75 | 76 | pool.start 77 | pool.enq(job1) 78 | # ensure the first job is executing 79 | # requires second job to finish 80 | Timeout.timeout(1) { test_latch.release } 81 | pool.enq(job2) 82 | pool.poison 83 | 84 | pool.join 85 | 86 | expect(job1.ran).to be true 87 | expect(job2.ran).to be true 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /spec/update/fix_dep_job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'support/gemspec_helper' 3 | require 'support/artifice_apps' 4 | require 'bundler_api/update/fix_dep_job' 5 | require 'bundler_api/update/gem_db_helper' 6 | require 'bundler_api/gem_helper' 7 | 8 | describe BundlerApi::FixDepJob do 9 | describe "#run" do 10 | include GemspecHelper 11 | 12 | let(:db) { $db } 13 | let(:gem_cache) { Hash.new } 14 | let(:counter) { nil } 15 | let(:mutex) { Mutex.new } 16 | let(:helper) { BundlerApi::GemDBHelper.new(db, gem_cache, mutex) } 17 | let(:name) { "foo" } 18 | let(:version) { "1.0" } 19 | let(:platform) { "ruby" } 20 | let(:foo_spec) { generate_gemspec(name, version, platform) } 21 | let(:bar_spec) { generate_gemspec('bar', '1.0', 'ruby') } 22 | let(:payload) { BundlerApi::GemHelper.new(name, Gem::Version.new(version), platform) } 23 | let(:job) { BundlerApi::FixDepJob.new(db, payload, counter, mutex, silent: true) } 24 | 25 | before do 26 | Artifice.activate_with(GemspecGenerator) 27 | @bar_rubygem_id = helper.find_or_insert_rubygem(bar_spec).last 28 | rubygem_id = helper.find_or_insert_rubygem(foo_spec).last 29 | @foo_version_id = helper.find_or_insert_version(foo_spec, rubygem_id, platform).last 30 | end 31 | 32 | after do 33 | Artifice.deactivate 34 | end 35 | 36 | it "should fill the dependencies in if they're missing" do 37 | job.run 38 | 39 | expect(db[:dependencies].filter(rubygem_id: @bar_rubygem_id, 40 | version_id: @foo_version_id, 41 | requirements: "~> 1.0", 42 | scope: "runtime").count).to eq(1) 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/update/gem_db_helper_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'support/gemspec_helper' 3 | require 'bundler_api/update/gem_db_helper' 4 | require 'bundler_api/gem_helper' 5 | 6 | describe BundlerApi::GemDBHelper do 7 | let(:db) { $db } 8 | let(:gem_cache) { Hash.new } 9 | let(:mutex) { nil } 10 | let(:helper) { BundlerApi::GemDBHelper.new(db, gem_cache, mutex) } 11 | 12 | describe "#exists?" do 13 | let(:payload) { BundlerApi::GemHelper.new("foo", Gem::Version.new("1.0"), "ruby", false) } 14 | 15 | context "if the gem exists" do 16 | before do 17 | rubygem = db[:rubygems].insert(name: "foo") 18 | version = db[:versions].insert(rubygem_id: rubygem, number: "1.0", platform: "ruby", indexed: true) 19 | end 20 | 21 | it "returns true" do 22 | expect(helper.exists?(payload)).to be_truthy 23 | end 24 | 25 | context "when using a mutex" do 26 | let(:mutex) { Mutex.new } 27 | 28 | it "returns the rubygems and versions id from the cache when called twice" do 29 | expect(helper.exists?(payload)).to be_truthy 30 | expect(helper.exists?(payload)).to be_truthy 31 | end 32 | end 33 | end 34 | 35 | context "if the gem does not exst" do 36 | it "returns nil" do 37 | expect(helper.exists?(payload)).to be_falsey 38 | end 39 | end 40 | end 41 | 42 | describe "#find_or_insert_rubygem" do 43 | include GemspecHelper 44 | 45 | let(:name) { "foo" } 46 | let(:version) { "1.0" } 47 | let(:spec) { generate_gemspec(name, version) } 48 | 49 | context "when there is no exisitng rubygem" do 50 | it "should insert the rubygem" do 51 | insert, rubygem_id = helper.find_or_insert_rubygem(spec) 52 | 53 | expect(insert).to eq(true) 54 | expect(db[:rubygems].filter(name: spec.name).select(:id).first[:id]).to eq(rubygem_id) 55 | end 56 | 57 | context "when the md5 checksum is set on names.list" do 58 | before do 59 | $db[:checksums].insert(name: "names.list", md5: "83afeb") 60 | end 61 | 62 | it "should clear the md5 checksum" do 63 | insert, rubygem_id = helper.find_or_insert_rubygem(spec) 64 | 65 | expect($db[:checksums].select(:md5).filter(name: "names.list").first[:md5]).to eq(nil) 66 | end 67 | end 68 | end 69 | 70 | context "when the rubygem already exists" do 71 | before do 72 | @rubygem_id = db[:rubygems].insert(name: name) 73 | end 74 | 75 | it "should retrieve the existing rubygem" do 76 | insert, rubygem_id = helper.find_or_insert_rubygem(spec) 77 | 78 | expect(insert).to eq(false) 79 | expect(rubygem_id).to eq(@rubygem_id) 80 | end 81 | end 82 | end 83 | 84 | describe "#find_or_insert_version" do 85 | include GemspecHelper 86 | 87 | let(:name) { "foo" } 88 | let(:version) { "1.0" } 89 | let(:platform) { "ruby" } 90 | let(:checksum) { "checksum" } 91 | let(:indexed) { true } 92 | let(:spec) { generate_gemspec(name, version, platform) } 93 | 94 | before do 95 | @rubygem_id = helper.find_or_insert_rubygem(spec).last 96 | end 97 | 98 | context "when there is no existing version" do 99 | it "should insert the version" do 100 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform, checksum, indexed) 101 | 102 | expect(insert).to eq(true) 103 | inserted_version = db[:versions].filter(rubygem_id: @rubygem_id, 104 | number: version, 105 | platform: platform, 106 | indexed: indexed).first 107 | expect(version_id).to eq(inserted_version[:id]) 108 | expect(inserted_version[:created_at]).to be_kind_of(Time) 109 | end 110 | 111 | context "when the versions md5 is set" do 112 | before do 113 | $db[:checksums].insert(name: "versions", md5: "82f5ab51") 114 | end 115 | 116 | it "installing a new version clears it" do 117 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform, checksum, indexed) 118 | row = $db[:checksums].filter(name: "versions").first 119 | 120 | expect(row[:md5]).to eq(nil) 121 | end 122 | end 123 | 124 | context "when the platform in the index differs from the spec" do 125 | let(:platform) { "jruby" } 126 | let(:spec) { generate_gemspec(name, version, "java") } 127 | 128 | it "inserts the platform from the index and not the spec" do 129 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform,checksum, indexed) 130 | 131 | expect(insert).to eq(true) 132 | expect(version_id).to eq(db[:versions].filter(rubygem_id: @rubygem_id, 133 | number: version, 134 | platform: platform, 135 | indexed: indexed, 136 | prerelease: Gem::Version.create(version).prerelease?). 137 | select(:id).first[:id]) 138 | end 139 | end 140 | 141 | context "when indexed is nil" do 142 | let(:indexed) { nil } 143 | 144 | it "automatically indexes it" do 145 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform, checksum, indexed) 146 | 147 | expect(insert).to eq(true) 148 | expect(version_id).to eq(db[:versions].filter(rubygem_id: @rubygem_id, 149 | number: version, 150 | platform: platform, 151 | indexed: true). 152 | select(:id).first[:id]) 153 | end 154 | end 155 | 156 | context "when it's a prerelease spec" do 157 | let(:version) { "1.1.pre" } 158 | 159 | it "should insert the version" do 160 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform, checksum, indexed) 161 | 162 | expect(insert).to eq(true) 163 | expect(version_id).to eq(db[:versions].filter(rubygem_id: @rubygem_id, 164 | number: version, 165 | platform: platform, 166 | prerelease: true, 167 | indexed: indexed). 168 | select(:id).first[:id]) 169 | end 170 | end 171 | end 172 | 173 | context "when the version exists" do 174 | before do 175 | @version_id = db[:versions].insert(rubygem_id: @rubygem_id, 176 | number: version, 177 | platform: platform, 178 | indexed: indexed) 179 | end 180 | 181 | it "finds the existing version" do 182 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform) 183 | 184 | expect(insert).to eq(false) 185 | expect(version_id).to eq(@version_id) 186 | end 187 | 188 | context "when the version is not indexed" do 189 | let(:indexed) { false } 190 | 191 | it "updates the indexed value" do 192 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform, 'abc123', true) 193 | 194 | expect(db[:versions].filter(id: @version_id).select(:indexed).first[:indexed]).to eq(true) 195 | end 196 | 197 | it "does not update the indexed value" do 198 | insert, version_id = helper.find_or_insert_version(spec, @rubygem_id, platform, 'abc123', nil) 199 | 200 | expect(db[:versions].filter(id: @version_id).select(:indexed).first[:indexed]).to eq(false) 201 | end 202 | end 203 | end 204 | end 205 | 206 | describe "#insert_dependencies" do 207 | include GemspecHelper 208 | 209 | context "when the dep gem already exists" do 210 | let(:requirement) { "~> 1.0" } 211 | let(:foo_spec) { generate_gemspec('foo', '1.0') } 212 | let(:bar_spec) { generate_gemspec('bar', '1.0') } 213 | 214 | before do 215 | @bar_rubygem_id = helper.find_or_insert_rubygem(bar_spec).last 216 | foo_rubygem_id = helper.find_or_insert_rubygem(foo_spec).last 217 | @foo_version_id = helper.find_or_insert_version(foo_spec, foo_rubygem_id).last 218 | end 219 | 220 | it "should insert the dependencies" do 221 | deps_added = helper.insert_dependencies(foo_spec, @foo_version_id) 222 | 223 | expect(deps_added).to eq(["~> 1.0 bar"]) 224 | expect(db[:dependencies].filter(requirements: requirement, 225 | scope: 'runtime', 226 | rubygem_id: @bar_rubygem_id, 227 | version_id: @foo_version_id).count).to eq(1) 228 | end 229 | 230 | context "sometimes the dep name is true which gets eval'd as a TrueClass" do 231 | let(:foo_spec) do 232 | generate_gemspec('foo', '1.0') do |s| 233 | s.add_runtime_dependency(true, "> 0") 234 | end 235 | end 236 | 237 | it "should insert the dependencies and not fail on the true gem" do 238 | deps_added = helper.insert_dependencies(foo_spec, @foo_version_id) 239 | 240 | expect(deps_added).to eq(["~> 1.0 bar"]) 241 | expect(db[:dependencies].filter(requirements: requirement, 242 | scope: 'runtime', 243 | rubygem_id: @bar_rubygem_id, 244 | version_id: @foo_version_id).count).to eq(1) 245 | end 246 | end 247 | 248 | context "when the dep name is a symbol" do 249 | let(:foo_spec) do 250 | generate_gemspec('foo', '1.0') do |s| 251 | s.add_runtime_dependency(:baz, "> 0") 252 | end 253 | end 254 | let(:baz_spec) { generate_gemspec('baz', '1.0') } 255 | 256 | before do 257 | @baz_rubygem_id = helper.find_or_insert_rubygem(baz_spec).last 258 | end 259 | 260 | it "should insert the dependencies and not fail on the true gem" do 261 | deps_added = helper.insert_dependencies(foo_spec, @foo_version_id) 262 | 263 | expect(deps_added).to eq(["~> 1.0 bar", "> 0 baz"]) 264 | expect(db[:dependencies].filter(requirements: requirement, 265 | scope: 'runtime', 266 | rubygem_id: @bar_rubygem_id, 267 | version_id: @foo_version_id).count).to eq(1) 268 | expect(db[:dependencies].filter(requirements: "> 0", 269 | scope: 'runtime', 270 | rubygem_id: @baz_rubygem_id, 271 | version_id: @foo_version_id).count).to eq(1) 272 | end 273 | end 274 | 275 | context "sometimes the dep is an array" do 276 | let(:foo_spec) do 277 | spec = generate_gemspec('foo', '1.0') 278 | spec.extend(Module.new { 279 | def dependencies 280 | [["bar", "~> 1.0"]] 281 | end 282 | }) 283 | 284 | spec 285 | end 286 | 287 | it "should insert the dependencies" do 288 | deps_added = helper.insert_dependencies(foo_spec, @foo_version_id) 289 | 290 | expect(deps_added).to eq(["~> 1.0 bar"]) 291 | expect(db[:dependencies].filter(requirements: requirement, 292 | scope: 'runtime', 293 | rubygem_id: @bar_rubygem_id, 294 | version_id: @foo_version_id).count).to eq(1) 295 | end 296 | end 297 | 298 | context "when the dep db record exists" do 299 | before do 300 | db[:dependencies].insert( 301 | requirements: requirement, 302 | rubygem_id: @bar_rubygem_id, 303 | version_id: @foo_version_id, 304 | scope: 'runtime' 305 | ) 306 | end 307 | 308 | it "should just skip adding it" do 309 | deps_added = helper.insert_dependencies(foo_spec, @foo_version_id) 310 | 311 | expect(deps_added).to eq([]) 312 | expect(db[:dependencies].filter(requirements: requirement, 313 | scope: 'runtime', 314 | rubygem_id: @bar_rubygem_id, 315 | version_id: @foo_version_id).count).to eq(1) 316 | end 317 | 318 | context "when the dep order is using the legacy style" do 319 | let(:foo_spec) do 320 | generate_gemspec('foo', '1.0') do |s| 321 | s.add_runtime_dependency "baz", [">= 0","= 1.0.1"] 322 | end 323 | end 324 | let(:baz_spec) { generate_gemspec('baz', '1.0') } 325 | 326 | before do 327 | @baz_rubygem_id = helper.find_or_insert_rubygem(baz_spec).last 328 | db[:dependencies].insert( 329 | requirements: ">= 0, = 1.0.1", 330 | rubygem_id: @baz_rubygem_id, 331 | version_id: @foo_version_id, 332 | scope: 'runtime' 333 | ) 334 | end 335 | 336 | it "should just skip adding it again" do 337 | deps_added = helper.insert_dependencies(foo_spec, @foo_version_id) 338 | 339 | expect(deps_added).to eq([]) 340 | expect(db[:dependencies].filter(requirements: requirement, 341 | scope: 'runtime', 342 | rubygem_id: @bar_rubygem_id, 343 | version_id: @foo_version_id).count).to eq(1) 344 | expect(db[:dependencies].filter(requirements: ">= 0, = 1.0.1", 345 | scope: 'runtime', 346 | rubygem_id: @baz_rubygem_id, 347 | version_id: @foo_version_id).count).to eq(1) 348 | end 349 | end 350 | end 351 | end 352 | end 353 | end 354 | -------------------------------------------------------------------------------- /spec/update/job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'sinatra/base' 2 | require 'spec_helper' 3 | require 'support/artifice_apps' 4 | require 'bundler_api/gem_helper' 5 | require 'bundler_api/update/atomic_counter' 6 | require 'bundler_api/update/job' 7 | require 'bundler_api/web' 8 | require 'bundler_api/cache' 9 | 10 | describe BundlerApi::Job do 11 | include Rack::Test::Methods 12 | let(:db) { $db } 13 | let(:builder) { GemBuilder.new(db) } 14 | let(:counter) { BundlerApi::AtomicCounter.new } 15 | let(:mutex) { Mutex.new } 16 | let(:client) { double(:client, purge_path: nil, purge_key: nil) } 17 | let(:cache) { BundlerApi::CacheInvalidator.new(cdn: client) } 18 | 19 | def app 20 | BundlerApi::Web.new($db, $db, silent: true) 21 | end 22 | 23 | before do 24 | BundlerApi::Job.clear_cache 25 | end 26 | 27 | describe "#run" do 28 | before do 29 | Artifice.activate_with(GemspecGenerator) 30 | end 31 | 32 | after do 33 | Artifice.deactivate 34 | end 35 | 36 | def gem_exists?(db, name, version = '1.0', platform = 'ruby') 37 | expect(db[<<-SQL, name, version, platform].count).to eq(1) 38 | SELECT * 39 | FROM rubygems, versions 40 | WHERE rubygems.id = versions.rubygem_id 41 | AND rubygems.name = ? 42 | AND versions.number = ? 43 | AND versions.platform = ? 44 | SQL 45 | end 46 | 47 | def get_gem(attribute, db, name, version = '1.0', platform = 'ruby') 48 | db[:rubygems] 49 | .join(:versions, rubygem_id: :id) 50 | .where(name: name, number: version, platform: "ruby") 51 | .first 52 | end 53 | 54 | def dependencies(name, version = "1.0", platform = "ruby") 55 | db[:dependencies]. 56 | join(:versions, id: :version_id). 57 | join(:rubygems, id: :rubygem_id). 58 | filter(rubygems__name: name, 59 | versions__number: version, 60 | versions__platform: platform). 61 | all 62 | end 63 | 64 | it "creates a rubygem if it doesn't exist" do 65 | payload = BundlerApi::GemHelper.new("foo", Gem::Version.new("1.0"), "ruby") 66 | job = BundlerApi::Job.new(db, payload, mutex, counter, silent: true) 67 | 68 | job.run 69 | 70 | gem_exists?(db, 'foo') 71 | end 72 | 73 | it "creates different platform rubygems" do 74 | %w(ruby java).each do |platform| 75 | payload = BundlerApi::GemHelper.new("foo", Gem::Version.new("1.0"), platform) 76 | job = BundlerApi::Job.new(db, payload, mutex, counter, silent: true) 77 | job.run 78 | end 79 | 80 | gem_exists?(db, 'foo') 81 | gem_exists?(db, 'foo', '1.0', 'java') 82 | end 83 | 84 | it "doesn't dupe rubygems" do 85 | %w(ruby java ruby).each do |platform| 86 | payload = BundlerApi::GemHelper.new("foo", Gem::Version.new("1.0"), platform) 87 | job = BundlerApi::Job.new(db, payload, mutex, counter, silent: true) 88 | job.run 89 | end 90 | 91 | gem_exists?(db, 'foo') 92 | gem_exists?(db, 'foo', '1.0', 'java') 93 | end 94 | 95 | it "saves checksum for info endpoint content" do 96 | versions = %w(1.0 1.2 1.1) 97 | versions.each do |version| 98 | payload = BundlerApi::GemHelper.new("foo1", Gem::Version.new(version), 'ruby') 99 | job = BundlerApi::Job.new(db, payload, mutex, counter, silent: true) 100 | job.run 101 | cache.purge_specs 102 | cache.purge_memory_cache('foo1') 103 | 104 | get '/info/foo1' 105 | last_response.body 106 | checksum = Digest::MD5.hexdigest(last_response.body) 107 | 108 | database_checksum = get_gem(:info_checksum, db, 'foo1', version)[:info_checksum] 109 | expect(database_checksum).to eq(checksum) 110 | end 111 | 112 | expect(db[:rubygems] 113 | .join(:versions, rubygem_id: :id) 114 | .where(name: "foo1").order_by(:created_at).map { |v| v[:number] }).to eq(versions) 115 | end 116 | 117 | context "with gem dependencies" do 118 | let(:gem_payload) { BundlerApi::GemHelper.new("foo", Gem::Version.new("1.0"), "ruby") } 119 | let(:dep_payload) { BundlerApi::GemHelper.new("bar", Gem::Version.new("1.0"), "ruby") } 120 | let(:gem_job) { BundlerApi::Job.new(db, gem_payload, mutex, counter, silent: true) } 121 | let(:dep_job) { BundlerApi::Job.new(db, dep_payload, mutex, counter, silent: true) } 122 | 123 | context "when gem is added before the dependency" do 124 | before do 125 | gem_job.run 126 | dep_job.run 127 | end 128 | 129 | it "doesn't create any dependencies" do 130 | expect(dependencies("foo")).to be_empty 131 | end 132 | end 133 | 134 | context "when the dependency is added before the gem" do 135 | before do 136 | dep_job.run 137 | gem_job.run 138 | end 139 | 140 | it "creates the correct dependencies" do 141 | expect(dependencies("foo").length).to eq(1) 142 | end 143 | end 144 | end 145 | 146 | context "when the index platform is jruby" do 147 | it "handles when platform in spec is different" do 148 | jobs = 2.times.map do 149 | payload = BundlerApi::GemHelper.new("foo", Gem::Version.new("1.0"), 'jruby') 150 | BundlerApi::Job.new(db, payload, mutex, counter, silent: true) 151 | end 152 | 153 | jobs.first.run 154 | expect { jobs[1].run }.not_to raise_error() 155 | 156 | gem_exists?(db, 'foo', '1.0', 'jruby') 157 | end 158 | 159 | it "sets the indexed attribute to true" do 160 | jobs = 2.times.map do 161 | payload = BundlerApi::GemHelper.new("foo", Gem::Version.new("1.0"), 'jruby') 162 | BundlerApi::Job.new(db, payload, mutex, counter, silent: true) 163 | end 164 | jobs.first.run 165 | version_id = db[<<-SQL, 'foo', '1.0', 'jruby'].first[:id] 166 | SELECT versions.id 167 | FROM rubygems, versions 168 | WHERE rubygems.id = versions.rubygem_id 169 | AND rubygems.name = ? 170 | AND versions.number = ? 171 | AND versions.platform = ? 172 | SQL 173 | db[:versions].where(id: version_id).update(indexed: false) 174 | jobs[1].run 175 | 176 | gem_exists?(db, 'foo', '1.0', 'jruby') 177 | expect(db[:versions].filter(id: version_id).select(:indexed).first[:indexed]).to be true 178 | end 179 | end 180 | end 181 | end 182 | -------------------------------------------------------------------------------- /spec/update/yank_job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'support/artifice_apps' 3 | require 'bundler_api/update/yank_job' 4 | 5 | describe BundlerApi::YankJob do 6 | let(:mutex) { Mutex.new } 7 | let(:job) { BundlerApi::YankJob.new(gem_cache, payload, mutex) } 8 | 9 | describe "#run" do 10 | before do 11 | Artifice.activate_with(GemspecGenerator) 12 | end 13 | 14 | after do 15 | Artifice.deactivate 16 | end 17 | 18 | context "when the platform is ruby" do 19 | let(:payload) { BundlerApi::GemHelper.new('foo', '1.0', 'ruby') } 20 | let(:payload2) { BundlerApi::GemHelper.new('foo', '1.1', 'ruby') } 21 | let(:gem_cache) { 22 | { 23 | payload.full_name => 1, 24 | payload2.full_name => 2 25 | } 26 | } 27 | 28 | it "should remove the gem from the cache" do 29 | job.run 30 | 31 | expect(gem_cache).to eq({payload2.full_name => 2}) 32 | end 33 | end 34 | 35 | context "when the platform is jruby" do 36 | let(:payload) { BundlerApi::GemHelper.new('foo', '1.0', 'jruby') } 37 | let(:gem_cache) { 38 | gem_helper = BundlerApi::GemHelper.new('foo', '1.0', 'jruby') 39 | { 40 | gem_helper.full_name => 1 41 | } 42 | } 43 | 44 | it "should remove the gem from the cache" do 45 | job.run 46 | 47 | expect(gem_cache).to eq({}) 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /spec/web_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rack/test' 2 | require 'spec_helper' 3 | require 'bundler_api/web' 4 | require 'support/gem_builder' 5 | require 'support/etag' 6 | 7 | describe BundlerApi::Web do 8 | include Rack::Test::Methods 9 | 10 | let(:builder) { GemBuilder.new($db) } 11 | let(:rack_id) { builder.create_rubygem("rack") } 12 | 13 | before do 14 | builder.create_version(rack_id, "rack", "1.0.0", "ruby", info_checksum: 'racksum') 15 | end 16 | 17 | def app 18 | BundlerApi::Web.new($db, $db, silent: true) 19 | end 20 | 21 | context "GET /" do 22 | let(:request) { "/" } 23 | 24 | it "redirects to rubygems.org" do 25 | get request 26 | 27 | expect(last_response).to be_redirect 28 | expect(last_response.headers['Location']). 29 | to eq('https://www.rubygems.org') 30 | end 31 | end 32 | 33 | context "GET static files" do 34 | let(:request) { "/robots.txt" } 35 | 36 | it "returns disallow root" do 37 | get request 38 | expect(last_response).to be_ok 39 | expect(last_response.body).to eq("Disallow: /\n") 40 | end 41 | end 42 | 43 | context "GET nonexistent files'" do 44 | let(:request) { "/nonexistent" } 45 | 46 | it "returns a 404" do 47 | get request 48 | expect(last_response).to be_not_found 49 | end 50 | end 51 | 52 | context "GET /api/v1/dependencies" do 53 | let(:request) { "/api/v1/dependencies" } 54 | 55 | context "there are no gems" do 56 | it "returns an empty string" do 57 | get request 58 | 59 | expect(last_response).to be_ok 60 | expect(last_response.body).to eq("") 61 | end 62 | end 63 | 64 | context "there are gems" do 65 | it "returns a marshal dump" do 66 | result = { 67 | name: 'rack', 68 | number: '1.0.0', 69 | platform: 'ruby', 70 | } 71 | 72 | get "#{request}?gems=rack" 73 | 74 | expect(last_response).to be_ok 75 | result.each do |k,v| 76 | expect(Marshal.load(last_response.body).first[k]).to eq(v) 77 | end 78 | end 79 | end 80 | 81 | context "there are too many gems" do 82 | let(:gems) { 201.times.map { |i| "gem-#{ i }" }.join(',') } 83 | 84 | it "returns a 422" do 85 | get "#{request}?gems=#{ gems }" 86 | 87 | expect(last_response).not_to be_ok 88 | expect(last_response.status).to be 422 89 | expect(last_response.body).to eq("Too many gems (use --full-index instead)") 90 | end 91 | end 92 | end 93 | 94 | 95 | context "GET /api/v1/dependencies.json" do 96 | let(:request) { "/api/v1/dependencies.json" } 97 | 98 | context "there are no gems" do 99 | it "returns an empty string" do 100 | get request 101 | 102 | expect(last_response).to be_ok 103 | expect(last_response.body).to eq("") 104 | end 105 | end 106 | 107 | context "there are gems" do 108 | it "returns a marshal dump" do 109 | result = { 110 | "name" => 'rack', 111 | "number" => '1.0.0', 112 | "platform" => 'ruby' 113 | } 114 | 115 | get "#{request}?gems=rack" 116 | 117 | expect(last_response).to be_ok 118 | result.each do |k,v| 119 | expect(JSON.parse(last_response.body).first[k]).to eq(v) 120 | end 121 | end 122 | end 123 | 124 | context "there are too many gems" do 125 | let(:gems) { 201.times.map { |i| "gem-#{ i }" }.join(',') } 126 | 127 | it "returns a 422" do 128 | error = { 129 | "error" => "Too many gems (use --full-index instead)", 130 | "code" => 422 131 | }.to_json 132 | 133 | get "#{request}?gems=#{ gems }" 134 | 135 | expect(last_response).not_to be_ok 136 | expect(last_response.body).to eq(error) 137 | end 138 | end 139 | end 140 | 141 | context "POST /api/v1/add_spec.json" do 142 | let(:url){ "/api/v1/add_spec.json" } 143 | let(:payload){ {:name => "rack", :version => "1.0.1", 144 | :platform => "ruby", :prerelease => false} } 145 | 146 | it "adds the spec to the database" do 147 | post url, JSON.dump(payload) 148 | 149 | expect(last_response).to be_ok 150 | res = JSON.parse(last_response.body) 151 | expect(res["name"]).to eq("rack") 152 | expect(res["version"]).to eq("1.0.1") 153 | end 154 | end 155 | 156 | context "POST /api/v1/remove_spec.json" do 157 | let(:url){ "/api/v1/remove_spec.json" } 158 | let(:payload){ {:name => "rack", :version => "1.0.0", 159 | :platform => "ruby", :prerelease => false} } 160 | 161 | it "removes the spec from the database" do 162 | post url, JSON.dump(payload) 163 | 164 | expect(last_response).to be_ok 165 | res = JSON.parse(last_response.body) 166 | expect(res["name"]).to eq("rack") 167 | expect(res["version"]).to eq("1.0.0") 168 | end 169 | end 170 | 171 | context "GET /quick/Marshal.4.8/:id" do 172 | it "redirects" do 173 | get "/quick/Marshal.4.8/rack" 174 | 175 | expect(last_response).to be_redirect 176 | expect(last_response.location).to end_with("/quick/Marshal.4.8/rack") 177 | end 178 | end 179 | 180 | context "GET /fetch/actual/gem/:id" do 181 | it "redirects" do 182 | get "/fetch/actual/gem/rack" 183 | 184 | expect(last_response).to be_redirect 185 | expect(last_response.location).to end_with("/fetch/actual/gem/rack") 186 | end 187 | end 188 | 189 | context "GET /gems/:id" do 190 | it "redirects" do 191 | get "/gems/rack" 192 | 193 | expect(last_response).to be_redirect 194 | expect(last_response.location).to end_with("/gems/rack") 195 | end 196 | end 197 | 198 | context "/latest_specs.4.8.gz" do 199 | it "redirects" do 200 | get "/latest_specs.4.8.gz" 201 | 202 | expect(last_response).to be_redirect 203 | expect(last_response.location).to end_with("/latest_specs.4.8.gz") 204 | end 205 | end 206 | 207 | context "/specs.4.8.gz" do 208 | it "redirects" do 209 | get "/specs.4.8.gz" 210 | 211 | expect(last_response).to be_redirect 212 | expect(last_response.location).to end_with("/specs.4.8.gz") 213 | end 214 | end 215 | 216 | context "/prerelease_specs.4.8.gz" do 217 | it "redirects" do 218 | get "/prerelease_specs.4.8.gz" 219 | 220 | expect(last_response).to be_redirect 221 | expect(last_response.location).to end_with("/prerelease_specs.4.8.gz") 222 | end 223 | end 224 | 225 | context "/names" do 226 | it_behaves_like "return 304 on second hit" do 227 | let(:url) { "/names" } 228 | end 229 | 230 | before do 231 | %w(a b c d).each {|gem_name| builder.create_rubygem(gem_name) } 232 | end 233 | 234 | it "returns an array" do 235 | get "/names" 236 | expect(last_response).to be_ok 237 | expect(last_response.body).to eq(<<-NAMES.chomp.gsub(/^ /, '')) 238 | --- 239 | a 240 | b 241 | c 242 | d 243 | rack 244 | 245 | NAMES 246 | end 247 | end 248 | 249 | context "/versions" do 250 | it_behaves_like "return 304 on second hit" do 251 | let(:url) { "/versions" } 252 | end 253 | 254 | let :versions_file do 255 | gem_info = BundlerApi::GemInfo.new($db) 256 | file_path = BundlerApi::GemInfo::VERSIONS_FILE_PATH 257 | CompactIndex::VersionsFile.new(file_path) 258 | end 259 | 260 | before do 261 | a = builder.create_rubygem("a") 262 | builder.create_version(a, 'a', '1.0.0', 'ruby', info_checksum: 'a100') 263 | builder.create_version(a, 'a', '1.0.1', 'ruby', info_checksum: 'a101') 264 | b = builder.create_rubygem("b") 265 | builder.create_version(b, 'b', '1.0.0', 'ruby', info_checksum: 'b100', indexed: false) 266 | c = builder.create_rubygem("c") 267 | builder.create_version(c, 'c', '1.0.0-java', 'ruby', info_checksum: 'c100') 268 | a200 = builder.create_version(a, 'a', '2.0.0', 'java', info_checksum: 'a200') 269 | builder.create_version(a, 'a', '2.0.1', 'ruby', info_checksum: 'a201') 270 | builder.yank(a200, yanked_info_checksum: 'a200y') 271 | end 272 | 273 | let(:data) do 274 | versions_file.contents + 275 | "rack 1.0.0 racksum\n" + 276 | "a 1.0.0 a100\n" + 277 | "a 1.0.1 a101\n" + 278 | "b 1.0.0 b100\n" + 279 | "c 1.0.0-java c100\n" + 280 | "a 2.0.0-java a200\n" + 281 | "a 2.0.1 a201\n" + 282 | "a -2.0.0-java a200y\n" 283 | end 284 | 285 | let(:expected_etag) { '"' << Digest::MD5.hexdigest(data) << '"' } 286 | 287 | it "returns versions.list" do 288 | get "/versions" 289 | 290 | expect(last_response).to be_ok 291 | expect(last_response.body).to eq(data) 292 | expect(last_response.header["ETag"]).to eq(expected_etag) 293 | end 294 | end 295 | 296 | context "/info/:gem" do 297 | it_behaves_like "return 304 on second hit" do 298 | let(:url) { "/info/rack" } 299 | end 300 | 301 | context "when has no required ruby version" do 302 | before do 303 | info_test = builder.create_rubygem('info_test') 304 | builder.create_version(info_test, 'info_test', '1.0.0', 'ruby', checksum: 'abc123') 305 | 306 | info_test101= builder.create_version(info_test, 'info_test', '1.0.1', 'ruby', checksum: 'qwerty') 307 | [['foo', '= 1.0.0'], ['bar', '>= 2.1, < 3.0']].each do |dep, requirements| 308 | dep_id = builder.create_rubygem(dep) 309 | builder.create_dependency(dep_id, info_test101, requirements) 310 | end 311 | end 312 | 313 | let(:expected_deps) do 314 | <<-DEPS.gsub(/^ /, '') 315 | --- 316 | 1.0.0 |checksum:abc123 317 | 1.0.1 bar:< 3.0&>= 2.1,foo:= 1.0.0|checksum:qwerty 318 | DEPS 319 | end 320 | let(:expected_etag) { '"' << Digest::MD5.hexdigest(expected_deps) << '"' } 321 | 322 | it "should return the gem list" do 323 | get "/info/info_test" 324 | 325 | expect(last_response).to be_ok 326 | expect(last_response.body).to eq(expected_deps) 327 | expect(last_response.header["ETag"]).to eq(expected_etag) 328 | end 329 | end 330 | 331 | context "when has a required ruby version" do 332 | before do 333 | a = builder.create_rubygem("a") 334 | builder_args = { checksum: "abc123", required_ruby: ">1.9", rubygems_version: ">2.0" } 335 | a_version = builder.create_version(a, 'a', '1.0.1', 'ruby', builder_args ) 336 | [['a_foo', '= 1.0.0'], ['a_bar', '>= 2.1, < 3.0']].each do |dep, requirements| 337 | dep_id = builder.create_rubygem(dep) 338 | builder.create_dependency(dep_id, a_version, requirements) 339 | end 340 | end 341 | 342 | let(:expected_deps) do 343 | <<-DEPS.gsub(/^ /, '') 344 | --- 345 | 1.0.1 a_bar:< 3.0&>= 2.1,a_foo:= 1.0.0|checksum:abc123,ruby:>1.9,rubygems:>2.0 346 | DEPS 347 | end 348 | 349 | it "should return the gem list with the required ruby version" do 350 | get "/info/a" 351 | expect(last_response).to be_ok 352 | expect(last_response.body).to eq(expected_deps) 353 | end 354 | end 355 | end 356 | end 357 | --------------------------------------------------------------------------------