├── .env ├── .gitignore ├── .powrc ├── .ruby-version ├── Gemfile ├── Gemfile.lock ├── Procfile ├── README.md ├── config.ru ├── rainbows.rb ├── yarp.rb └── yarp ├── app.rb ├── cache ├── base.rb ├── file.rb ├── memcache.rb ├── null.rb └── tee.rb ├── ext └── sliceable_hash.rb ├── initializers └── new_relic.rb └── logger.rb /.env: -------------------------------------------------------------------------------- 1 | # -*- bash -*- 2 | # 3 | # .env -- 4 | # Configuration for Yarp. 5 | # Will not work if these valirable are not present in the environment. 6 | # 7 | 8 | # If a request is smaller than 50 kiB, store it it the "small" cache 9 | # (memcached by default), else in the "large" cache (filesystem by default) 10 | YARP_CACHE_THRESHOLD=50000 11 | YARP_LARGE_CACHE=file 12 | YARP_SMALL_CACHE=memcache 13 | 14 | # Cache for a day 15 | YARP_CACHE_TTL=86400 16 | 17 | # Limit the filesystem cache to 250 MiB 18 | YARP_FILECACHE_MAX_BYTES=250000000 19 | 20 | # Directory hosting the filesystem cache 21 | YARP_FILECACHE_PATH=tmp/cache 22 | 23 | # Where to redirect/fetch from. this can be another instance of Yarp, or 24 | # Rubygems. 25 | YARP_UPSTREAM=http://eu.yarp.io 26 | 27 | # Comma-separated list of host:port entries 28 | MEMCACHIER_SERVERS=localhost:11211 29 | 30 | # The port on which to run the Rack application 31 | PORT=24591 32 | 33 | # The following only apply if you run Unicorn locally 34 | # not if running Pow or Rackup directly 35 | UNICORN_KEEPALIVE=20 36 | UNICORN_THREADS=10 37 | UNICORN_TIMEOUT=30 38 | UNICORN_WORKERS=2 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | log/ 3 | -------------------------------------------------------------------------------- /.powrc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while read LINE ; do 4 | test -n "${LINE%#*}" || continue 5 | echo "$LINE" 6 | eval "export '$LINE'" 7 | done < .env 8 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.1.5 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source ENV.fetch('GEM_SOURCE', 'http://rubygems.org') 2 | ruby '2.1.5' 3 | 4 | gem 'sinatra' # lightweight web framework 5 | gem 'foreman' # process watchdog 6 | gem 'rainbows' # threaded webserver 7 | gem 'dalli' # memcache adapter 8 | 9 | gem 'pry' # a better ruby shell 10 | gem 'pry-nav' 11 | gem 'pry-remote' 12 | 13 | gem 'newrelic_rpm' # app monitoring 14 | 15 | gem 'term-ansicolor' 16 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://eu.yarp.io/ 3 | specs: 4 | coderay (1.1.0) 5 | dalli (2.7.2) 6 | dotenv (0.11.1) 7 | dotenv-deployment (~> 0.0.2) 8 | dotenv-deployment (0.0.2) 9 | foreman (0.75.0) 10 | dotenv (~> 0.11.1) 11 | thor (~> 0.19.1) 12 | kgio (2.9.2) 13 | method_source (0.8.2) 14 | newrelic_rpm (3.9.7.266) 15 | pry (0.10.1) 16 | coderay (~> 1.1.0) 17 | method_source (~> 0.8.1) 18 | slop (~> 3.4) 19 | pry-nav (0.2.4) 20 | pry (>= 0.9.10, < 0.11.0) 21 | pry-remote (0.1.8) 22 | pry (~> 0.9) 23 | slop (~> 3.0) 24 | rack (1.5.2) 25 | rack-protection (1.5.3) 26 | rack 27 | rainbows (4.6.2) 28 | kgio (~> 2.5) 29 | rack (~> 1.1) 30 | unicorn (~> 4.8) 31 | raindrops (0.13.0) 32 | sinatra (1.4.5) 33 | rack (~> 1.4) 34 | rack-protection (~> 1.4) 35 | tilt (~> 1.3, >= 1.3.4) 36 | slop (3.6.0) 37 | term-ansicolor (1.3.0) 38 | tins (~> 1.0) 39 | thor (0.19.1) 40 | tilt (1.4.1) 41 | tins (1.3.3) 42 | unicorn (4.8.3) 43 | kgio (~> 2.6) 44 | rack 45 | raindrops (~> 0.7) 46 | 47 | PLATFORMS 48 | ruby 49 | 50 | DEPENDENCIES 51 | dalli 52 | foreman 53 | newrelic_rpm 54 | pry 55 | pry-nav 56 | pry-remote 57 | rainbows 58 | sinatra 59 | term-ansicolor 60 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: bundle exec rainbows -c rainbows.rb -p $PORT 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Yet Another Rubygems Proxy 2 | 3 | > Yarp is a small [Sinatra](http://www.sinatrarb.com) app that makes your 4 | > [bundler](http://bundler.io) faster. You'll love it if you update your 5 | > apps a lot... or simply deploy a lot. 6 | 7 | On a example medium-sizes application with 34 direct gems dependencies, Yarp 8 | makes my `bundle` commands up to 80% faster: 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 |
direct Rubygemswith `yarp.io`local Yarp
bundle install (1 gem missing)170 s51 s24 s
bundle update (73 updates)140 s65 s45 s
bundle update (no change)26 s13 s8.5 s
37 | 38 | Thats a 45% percent win right there. 8 seconds shaved of my deploy times. If 39 | you deploy 20 times a day to your staging environments and 5 times a day to 40 | production, you're getting 15 minutes of your life back every week. Make 41 | those count! 42 | 43 | 44 | ### Installation and usage 45 | 46 | Deploy your own Yarp or use the one at `yarp.io`. 47 | 48 | #### For projects using `bundler` 49 | 50 | Just replace this line on top of your Gemfile: 51 | 52 | source 'http://rubygems.org' 53 | 54 | By one of the following: 55 | 56 | source 'http://us.yarp.io' 57 | source 'http://eu.yarp.io' 58 | 59 | You're done. 60 | 61 | If you want/need SSL connections, you can use the Heroku URLs: 62 | 63 | source 'https://yarp-us.herokuapp.com' 64 | source 'https://yarp-eu.herokuapp.com' 65 | 66 | 67 | #### Your own local Yarp 68 | 69 | You can make this even faster by deploying your very own, local Yarp. 70 | Example install with the excellent [Pow](http://pow.cx): 71 | 72 | curl get.pow.cx | sh # unless you already have Pow 73 | git clone https://github.com/mezis/yarp.git ~/.yarp 74 | ln -s ~/.yarp ~/.pow/yarp 75 | 76 | Then change your `Gemfile`'s' source line to: 77 | 78 | source ENV.fetch('GEM_SOURCE', 'http://eu.yarp.io') 79 | 80 | And add the GEM_SOURCE to your `~/.profile` or `~/.zshrc`: 81 | 82 | export GEM_SOURCE=http://yarp.dev 83 | 84 | Why the dance with `ENV.fetch`? Simply because your codebase may be deployed 85 | or used somewhere lacking `yarp.dev`; this gives you a fallback to another 86 | source of gems. 87 | 88 | 89 | #### Outside of `bundler` 90 | 91 | Edit the sources entry in your `~/.gemrc`: 92 | 93 | --- 94 | :sources: 95 | - http://yarp.dev 96 | 97 | assuming you've followed the Pow instructions above; or use one of the 98 | `yarp.io` servers instead. 99 | 100 | 101 | ### How it works & Caveats 102 | 103 | Yarp caches calls to Rubygem's dependency API, spec files, and gems for 24 104 | hours if using `(eu|us).yarp.io`. It redirects all other calls to Rubygems 105 | directly. 106 | 107 | This means that when gems get released or updated, you'll **lag a day 108 | behind**. 109 | 110 | 111 | ### Hacking Yarp 112 | 113 | Checkout, make sure you have a [Memcache](http://memcached.org/) running, 114 | configure `.env`, and 115 | 116 | $ bundle exec foreman run rackup 117 | 118 | Thake a long look at the [`.env`](.env) file, as most 119 | configuration options for Yarp are there. 120 | 121 | 122 | ### License 123 | 124 | Yarp is released under the MIT licence. 125 | Copyright (c) 2013 HouseTrip Ltd. 126 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | 4 | require 'dotenv' 5 | Dotenv.load! 6 | 7 | require 'yarp/app' 8 | require 'yarp/initializers/new_relic' 9 | 10 | run Yarp::App 11 | -------------------------------------------------------------------------------- /rainbows.rb: -------------------------------------------------------------------------------- 1 | # Rainbows-specific configuration 2 | 3 | Rainbows! do 4 | use :ThreadPool # concurrency model to use 5 | worker_connections ENV['UNICORN_THREADS'].to_i 6 | keepalive_timeout ENV['UNICORN_KEEPALIVE'].to_i # zero disables keepalives entirely 7 | client_max_body_size 1_024 # 1KB 8 | keepalive_requests 100 # default:100 9 | client_header_buffer_size 2_048 # 2 kilobytes 10 | end 11 | 12 | # Sample verbose configuration file for Unicorn (not Rack) 13 | # 14 | # This configuration file documents many features of Unicorn 15 | # that may not be needed for some applications. See 16 | # http://unicorn.bogomips.org/examples/unicorn.conf.minimal.rb 17 | # for a much simpler configuration file. 18 | # 19 | # See http://unicorn.bogomips.org/Unicorn/Configurator.html for complete 20 | # documentation. 21 | 22 | # Use at least one worker per core if you're on a dedicated server, 23 | # more will usually help for _short_ waits on databases/caches. 24 | worker_processes ENV['UNICORN_WORKERS'].to_i 25 | 26 | # Since Unicorn is never exposed to outside clients, it does not need to 27 | # run on the standard HTTP port (80), there is no reason to start Unicorn 28 | # as root unless it's from system init scripts. 29 | # If running the master process as root and the workers as an unprivileged 30 | # user, do this to switch euid/egid in the workers (also chowns logs): 31 | # user "unprivileged_user", "unprivileged_group" 32 | 33 | # Help ensure your application will always spawn in the symlinked 34 | # "current" directory that Capistrano sets up. 35 | # working_directory "/path/to/app/current" # available in 0.94.0+ 36 | 37 | # listen on both a Unix domain socket and a TCP port, 38 | # we use a shorter backlog for quicker failover when busy 39 | # listen "/tmp/.sock", :backlog => 64 40 | # listen 8080, :tcp_nopush => true 41 | 42 | # nuke workers after X seconds instead of 60 seconds (the default) 43 | timeout ENV['UNICORN_TIMEOUT'].to_i 44 | 45 | # feel free to point this anywhere accessible on the filesystem 46 | # pid "/path/to/app/shared/pids/unicorn.pid" 47 | 48 | # By default, the Unicorn logger will write to stderr. 49 | # Additionally, ome applications/frameworks log to stderr or stdout, 50 | # so prevent them from going to /dev/null when daemonized here: 51 | # stderr_path "/path/to/app/shared/log/unicorn.stderr.log" 52 | # stdout_path "/path/to/app/shared/log/unicorn.stdout.log" 53 | 54 | # combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings 55 | # http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow 56 | preload_app true 57 | GC.respond_to?(:copy_on_write_friendly=) and 58 | GC.copy_on_write_friendly = true 59 | 60 | # Enable this flag to have unicorn test client connections by writing the 61 | # beginning of the HTTP headers before calling the application. This 62 | # prevents calling the application for connections that have disconnected 63 | # while queued. This is only guaranteed to detect clients on the same 64 | # host unicorn runs on, and unlikely to detect disconnects even on a 65 | # fast LAN. 66 | check_client_connection false 67 | 68 | before_fork do |server, worker| 69 | Signal.trap 'TERM' do 70 | puts 'Unicorn master intercepting TERM and sending myself QUIT instead' 71 | Process.kill 'QUIT', Process.pid 72 | end 73 | end 74 | 75 | after_fork do |server, worker| 76 | Signal.trap 'TERM' do 77 | puts 'Unicorn worker intercepting TERM and doing nothing. Wait for master to sent QUIT' 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /yarp.rb: -------------------------------------------------------------------------------- 1 | module Yarp 2 | end -------------------------------------------------------------------------------- /yarp/app.rb: -------------------------------------------------------------------------------- 1 | require 'yarp' 2 | require 'yarp/ext/sliceable_hash' 3 | require 'yarp/cache/memcache' 4 | require 'yarp/cache/file' 5 | require 'yarp/cache/null' 6 | require 'yarp/cache/tee' 7 | require 'yarp/logger' 8 | 9 | require 'sinatra/base' 10 | require 'digest' 11 | require 'uri' 12 | require 'net/http' 13 | 14 | module Yarp 15 | class App < Sinatra::Base 16 | RUBYGEMS_URL = ENV['YARP_UPSTREAM'] 17 | 18 | CACHEABLE_SHORT = %r{ 19 | ^/api/v1/dependencies | 20 | ^/(prerelease_|latest_)?specs.*\.gz$ 21 | }x 22 | 23 | CACHEABLE_LONG = %r{ 24 | /quick.*gemspec\.rz$ | 25 | ^/gems/.*\.gem$ 26 | }x 27 | 28 | get CACHEABLE_SHORT do 29 | get_cached_request(request, CACHE_TTL) 30 | end 31 | 32 | get CACHEABLE_LONG do 33 | get_cached_request(request, 365*86400) 34 | end 35 | 36 | get '/cache/status.json' do 37 | content_type :json 38 | Yarp::Cache::File.new.status.to_json 39 | end 40 | 41 | get '*' do 42 | path = full_request_path 43 | Log.info "REDIRECT <#{path}>" 44 | # $stderr.flush 45 | redirect "#{RUBYGEMS_URL}#{path}" 46 | end 47 | 48 | private 49 | 50 | Log = Yarp::Logger.new(STDERR) 51 | CACHE_TTL = ENV['YARP_CACHE_TTL'].to_i 52 | CACHE_THRESHOLD = ENV['YARP_CACHE_THRESHOLD'].to_i 53 | 54 | def get_cached_request(request, ttl) 55 | path = full_request_path 56 | cache_key = Digest::SHA1.hexdigest(path) 57 | Log.debug "GET <#{path}> (#{cache_key})" 58 | 59 | headers,payload = 60 | cache.fetch(cache_key, ttl) do 61 | uri = URI("#{RUBYGEMS_URL}#{path}") 62 | Log.debug "FETCH #{uri}" 63 | response = fetch_with_redirects(uri) 64 | 65 | kept_headers = response.to_hash.slice('content-type', 'server', 'date') 66 | if response.code != '200' 67 | return [response.code.to_i, response.to_hash, response.body.to_s] 68 | end 69 | 70 | [kept_headers, response.body] 71 | end 72 | 73 | [200, headers, payload] 74 | end 75 | 76 | 77 | def full_request_path 78 | if request.query_string.length > 0 79 | "#{request.path}?#{request.query_string}" 80 | else 81 | request.path 82 | end 83 | end 84 | 85 | 86 | def fetch_with_redirects(uri_str, limit = 10) 87 | while limit > 0 88 | begin 89 | response = Net::HTTP.get_response(URI(uri_str)) 90 | rescue SocketError => e 91 | Log.error("#{SocketError}: #{e.message}") 92 | limit -= 1 93 | next 94 | end 95 | 96 | case response 97 | when Net::HTTPRedirection then 98 | uri_str = response['location'] 99 | limit -= 1 100 | else 101 | return response 102 | end 103 | end 104 | raise RuntimeError('too many HTTP redirects') if limit == 0 105 | end 106 | 107 | Cache = Yarp::Cache::Tee.new( 108 | caches: { 109 | memcache: ENV['MEMCACHIER_SERVERS'].empty? ? Yarp::Cache::File.new : Yarp::Cache::Memcache.new, 110 | file: Yarp::Cache::File.new, 111 | null: Yarp::Cache::Null.new 112 | }, 113 | condition: lambda { |key, value| 114 | value.last.length <= CACHE_THRESHOLD ? 115 | ENV['YARP_SMALL_CACHE'].to_sym : 116 | ENV['YARP_LARGE_CACHE'].to_sym 117 | }) 118 | 119 | def cache 120 | Cache 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /yarp/cache/base.rb: -------------------------------------------------------------------------------- 1 | require 'yarp' 2 | 3 | module Yarp::Cache 4 | class Base 5 | 6 | def fetch(key, ttl=nil) 7 | raise NotImplementedError("#{self.class.name} is abstract") 8 | end 9 | 10 | def get(key) 11 | raise NotImplementedError("#{self.class.name} is abstract") 12 | end 13 | 14 | end 15 | end -------------------------------------------------------------------------------- /yarp/cache/file.rb: -------------------------------------------------------------------------------- 1 | require 'yarp/cache/base' 2 | require 'yarp/logger' 3 | require 'pathname' 4 | require 'uri' 5 | 6 | module Yarp::Cache 7 | # A cache store implementation which stores everything on the filesystem. 8 | # 9 | class File < Base 10 | attr_reader :_cache_path 11 | attr_reader :_max_bytes 12 | 13 | 14 | def initialize(cache_path:nil, max_bytes:nil) 15 | @_cache_path = cache_path ? cache_path.to_s : Pathname(ENV['YARP_FILECACHE_PATH']) 16 | @_max_bytes = max_bytes ? max_bytes.to_i : ENV['YARP_FILECACHE_MAX_BYTES'].to_i 17 | end 18 | 19 | 20 | def get(key) 21 | now = Time.now.to_i 22 | _edit_metadata do |metadata| 23 | value = metadata[:files].delete(key) 24 | return unless value 25 | 26 | size,access,expiry = value 27 | if expiry < now 28 | _remove_expired 29 | return 30 | end 31 | 32 | metadata[:files][key] = [size,now,expiry] 33 | return Marshal.load(_cache_file(key).read) 34 | end 35 | end 36 | 37 | 38 | def fetch(key, ttl=nil) 39 | value = get(key) and return value 40 | value = yield 41 | _set(key, value, ttl) 42 | end 43 | 44 | def status 45 | _edit_metadata do |metadata| 46 | return { keys:metadata[:files].size, bytes:metadata[:bytes], size:_max_bytes } 47 | end 48 | end 49 | 50 | 51 | def flush 52 | _flush 53 | end 54 | 55 | 56 | # private 57 | 58 | Log = Yarp::Logger.new(STDERR) 59 | Lock = Mutex.new 60 | 61 | # schema for metadata 62 | # each file entry maps a key (the filename) to 3 integers: the size, the 63 | # timestamp of last usage and timestamp of expiry. 64 | # the first file is the least recently used. 65 | def _default_meta 66 | { bytes:0, files:{} } 67 | end 68 | 69 | 70 | # path to the file holding cache metadata 71 | def _meta_path 72 | @_meta_path ||= _cache_path.join('meta') 73 | end 74 | 75 | # path to the file used to store +key+ 76 | def _cache_file(key) 77 | escaped_key = URI.encode_www_form_component(key) 78 | _cache_path.join(escaped_key) 79 | end 80 | 81 | # empties the whole cache 82 | def _flush 83 | _edit_metadata do |metadata| 84 | _cache_path.children.each do |child| 85 | child.rmtree unless child == _meta_path 86 | end 87 | metadata.replace(DEFAULT_META) 88 | end 89 | end 90 | 91 | # write a value to the cache 92 | def _set(key, value, ttl=nil) 93 | ttl ||= 365 * 86400 # 1 year 94 | now = Time.now.to_i 95 | expiry = now + ttl 96 | data = Marshal.dump(value) 97 | size = data.bytesize 98 | 99 | # require 'pry' ; require 'pry-nav' ; binding.pry 100 | _delete(key) 101 | _make_space_for(size) 102 | _cache_file(key).open('w') { |io| io.write(data) } 103 | _edit_metadata do |metadata| 104 | metadata[:bytes] += data.bytesize 105 | metadata[:files][key] = [size,now,expiry] 106 | end 107 | 108 | return value 109 | end 110 | 111 | def _delete(key) 112 | _edit_metadata do |metadata| 113 | size,_,_ = metadata[:files][key] 114 | return false if size.nil? 115 | metadata[:bytes] -= size 116 | metadata[:files].delete(key) 117 | end 118 | _cache_file(key).delete 119 | return true 120 | end 121 | 122 | # removes expired cache entries (files) and updates metadata 123 | def _remove_expired 124 | now = Time.now.to_i 125 | _edit_metadata do |metadata| 126 | files_to_delete = [] 127 | metadata[:files].each_pair do |key,(size, _, expires)| 128 | next unless expires < now 129 | _cache_file(key).delete 130 | metadata[:bytes] -= size 131 | metadata[:files].delete(key) 132 | end 133 | end 134 | end 135 | 136 | # removes files from the cache until there is enought space for +bytes+ 137 | def _make_space_for(bytes) 138 | raise ArgumentError("cannot store files larger than the cache") if bytes > _max_bytes 139 | _edit_metadata do |metadata| 140 | while bytes > _max_bytes - metadata[:bytes] 141 | key,(size,_,_) = metadata[:files].first 142 | Log.warn "FILECACHE evicting #{key}" 143 | _cache_file(key).delete 144 | metadata[:files].delete(key) 145 | metadata[:bytes] -= size 146 | end 147 | end 148 | end 149 | 150 | # executes the given block while holding a lock on the meta file. 151 | # yield an IO for the meta file. 152 | # reentrant (yields the same descriptor if called recursively). 153 | def _with_lock 154 | return yield @_meta_fd if @_meta_fd 155 | _meta_path.parent.mkpath 156 | _meta_path.open(::File::CREAT | ::File::RDWR) do |io| 157 | Lock.synchronize do 158 | begin 159 | @_meta_fd = io 160 | io.flock(::File::LOCK_EX) 161 | yield @_meta_fd 162 | ensure 163 | io.flock(::File::LOCK_UN) 164 | @_meta_fd = nil 165 | end 166 | end 167 | end 168 | end 169 | 170 | # yields the current metadata (should be a mutable hash). 171 | # writes data back (even if unmodified) after running the block. 172 | # implicitly locks the metadata file. 173 | # reentrant (nested calls are passed the same mutable hash). 174 | def _edit_metadata 175 | return yield @_metadata if @_metadata 176 | _with_lock do |io| 177 | io.seek(0) 178 | raw = io.read() 179 | @_metadata = begin 180 | raw.length == 0 ? _default_meta.dup : Marshal.load(raw) 181 | rescue ArgumentError, TypeError 182 | Log.warn("FILECACHE resetting broken metatada") 183 | _default_meta.dup 184 | end 185 | 186 | begin 187 | yield @_metadata 188 | ensure 189 | # Log.debug("FILECACHE metadata before write #{@_metadata.inspect}") 190 | raw = Marshal.dump(@_metadata) 191 | 192 | io.seek(0) 193 | io.truncate(raw.bytesize) 194 | io.write(raw) 195 | io.flush 196 | @_metadata = nil 197 | end 198 | end 199 | end 200 | 201 | 202 | end 203 | end 204 | -------------------------------------------------------------------------------- /yarp/cache/memcache.rb: -------------------------------------------------------------------------------- 1 | require 'yarp/cache/base' 2 | require 'dalli' 3 | 4 | module Yarp::Cache 5 | class Memcache 6 | 7 | def fetch(key, ttl=nil) 8 | _connection.fetch(key, ttl) { yield or return } 9 | end 10 | 11 | def get(key) 12 | _connection.get(key) 13 | end 14 | 15 | private 16 | 17 | def _connection 18 | @_connection ||= Dalli::Client.new( 19 | ENV['MEMCACHIER_SERVERS'].split(','), 20 | username: ENV['MEMCACHIER_USERNAME'], 21 | password: ENV['MEMCACHIER_PASSWORD'], 22 | compress: true) 23 | end 24 | end 25 | end -------------------------------------------------------------------------------- /yarp/cache/null.rb: -------------------------------------------------------------------------------- 1 | require 'yarp/cache/base' 2 | 3 | module Yarp::Cache 4 | class Null 5 | 6 | def fetch(key, ttl=nil) 7 | yield 8 | end 9 | 10 | def get(key) 11 | nil 12 | end 13 | 14 | end 15 | end -------------------------------------------------------------------------------- /yarp/cache/tee.rb: -------------------------------------------------------------------------------- 1 | require 'yarp/cache/base' 2 | require 'yarp/logger' 3 | 4 | module Yarp::Cache 5 | class Tee 6 | attr_reader :_caches 7 | attr_reader :_condition 8 | 9 | Log = Yarp::Logger.new(STDERR) 10 | 11 | # - condition: proc that accepts a key and payload size, and yields a symbol 12 | # - caches: a hash of symbols to caches 13 | # 14 | # All symbols listed by the +condition+ must be in the +caches+. 15 | # 16 | def initialize(condition:nil, caches:nil) 17 | @_caches = caches 18 | @_condition = condition 19 | end 20 | 21 | 22 | def get(key) 23 | _caches.each_pair do |cache_name, cache| 24 | next unless v = cache.get(key) 25 | Log.info "TEE cache hit #{key} <- #{cache_name}" 26 | return v 27 | end 28 | nil 29 | end 30 | 31 | 32 | def fetch(key, ttl) 33 | v = get(key) and return v 34 | value = yield 35 | cache_name = _condition.call(key, value) 36 | Log.warn "TEE cache miss #{key} -> #{cache_name} (ttl #{ttl})" 37 | _caches[cache_name].fetch(key, ttl) { value } 38 | end 39 | 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /yarp/ext/sliceable_hash.rb: -------------------------------------------------------------------------------- 1 | require 'yarp' 2 | 3 | module Yarp::Ext 4 | module SliceableHash 5 | def slice(*keys) 6 | select { |k,v| keys.include?(k) } 7 | end 8 | 9 | ::Hash.send :include, self 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /yarp/initializers/new_relic.rb: -------------------------------------------------------------------------------- 1 | require 'newrelic_rpm' 2 | 3 | if defined? Unicorn 4 | NewRelic::Agent.after_fork(:force_reconnect => true) 5 | end 6 | -------------------------------------------------------------------------------- /yarp/logger.rb: -------------------------------------------------------------------------------- 1 | require 'yarp' 2 | require 'logger' 3 | require 'term/ansicolor' 4 | 5 | module Yarp 6 | class Logger < ::Logger 7 | SCHEMA = { 'DEBUG' => :uncolored, 'INFO' => :green, 'WARN' => :yellow, 'ERROR' => :red } 8 | 9 | def format_message(level, timestamp, _, message) 10 | color = SCHEMA[level] 11 | "[%s] %s\n" % [ 12 | timestamp.strftime('%F %T'), 13 | Term::ANSIColor.send(color, message) 14 | ] 15 | end 16 | end 17 | end 18 | --------------------------------------------------------------------------------