├── test ├── sasl │ ├── memcached.conf │ └── sasldb ├── test_encoding.rb ├── test_serializer.rb ├── test_compressor.rb ├── helper.rb ├── test_network.rb ├── test_ring.rb ├── test_cas_client.rb ├── test_sasl.rb ├── test_failover.rb ├── memcached_mock.rb ├── test_server.rb ├── benchmark_test.rb ├── test_rack_session.rb ├── test_active_support.rb └── test_dalli.rb ├── Gemfile ├── lib ├── dalli │ ├── version.rb │ ├── railtie.rb │ ├── compressor.rb │ ├── options.rb │ ├── cas │ │ └── client.rb │ ├── ring.rb │ ├── socket.rb │ └── client.rb ├── dalli.rb ├── action_dispatch │ └── middleware │ │ └── session │ │ └── dalli_store.rb ├── rack │ └── session │ │ └── dalli.rb └── active_support │ └── cache │ └── dalli_store.rb ├── gemfiles ├── rails4.gemfile └── rails5.gemfile ├── Appraisals ├── .travis.yml ├── .gitignore ├── Rakefile ├── dalli.gemspec ├── LICENSE ├── Performance.md ├── code_of_conduct.md ├── README.md └── History.md /test/sasl/memcached.conf: -------------------------------------------------------------------------------- 1 | mech_list: plain 2 | -------------------------------------------------------------------------------- /test/sasl/sasldb: -------------------------------------------------------------------------------- 1 | testuser:testtest::::::: 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | gem 'kgio', :platform => :mri 6 | -------------------------------------------------------------------------------- /lib/dalli/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | module Dalli 3 | VERSION = '2.7.10' 4 | end 5 | -------------------------------------------------------------------------------- /gemfiles/rails4.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source 'https://rubygems.org' 4 | 5 | gem 'kgio', platform: :mri 6 | gem 'rails', '>= 4.0.0', '< 5' 7 | 8 | gemspec path: '../' 9 | -------------------------------------------------------------------------------- /lib/dalli/railtie.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | module Dalli 3 | class Railtie < ::Rails::Railtie 4 | config.before_configuration do 5 | config.cache_store = :dalli_store 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | appraise 'rails3' do 2 | gem 'rails', '>= 3.2.0', '< 4' 3 | end 4 | 5 | appraise 'rails4' do 6 | gem 'rails', '>= 4.0.0', '< 5' 7 | end 8 | 9 | appraise 'rails5' do 10 | gem 'rails', '5.0.0.beta2' 11 | end 12 | -------------------------------------------------------------------------------- /gemfiles/rails5.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source 'https://rubygems.org' 4 | 5 | gem 'kgio', platform: :mri 6 | gem 'rails', '~> 5.0.0' 7 | gem 'minitest', '< 5.10' 8 | 9 | gemspec path: '../' 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | cache: bundler 3 | rvm: 4 | - 2.6.2 5 | - 2.5.5 6 | - 2.4.5 7 | - 2.3.8 8 | - jruby-9.1.16.0 9 | gemfile: 10 | - gemfiles/rails5.gemfile 11 | matrix: 12 | fast_finish: true 13 | env: 14 | global: 15 | - JRUBY_OPTS='--debug' 16 | script: 17 | - bundle exec rake 18 | before_install: 19 | - gem install bundler 20 | - sudo apt-get -y remove memcached 21 | - sudo apt-get install libevent-dev 22 | - wget https://memcached.org/files/memcached-1.4.15.tar.gz 23 | - tar -zxvf memcached-1.4.15.tar.gz 24 | - cd memcached-1.4.15 25 | - ./configure --enable-sasl 26 | - make 27 | - sudo make install 28 | -------------------------------------------------------------------------------- /lib/dalli/compressor.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'zlib' 3 | require 'stringio' 4 | 5 | module Dalli 6 | class Compressor 7 | def self.compress(data) 8 | Zlib::Deflate.deflate(data) 9 | end 10 | 11 | def self.decompress(data) 12 | Zlib::Inflate.inflate(data) 13 | end 14 | end 15 | 16 | class GzipCompressor 17 | def self.compress(data) 18 | io = StringIO.new(String.new(""), "w") 19 | gz = Zlib::GzipWriter.new(io) 20 | gz.write(data) 21 | gz.close 22 | io.string 23 | end 24 | 25 | def self.decompress(data) 26 | io = StringIO.new(data, "rb") 27 | Zlib::GzipReader.new(io).read 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /test/tmp/ 9 | /test/version_tmp/ 10 | /tmp/ 11 | 12 | ## Specific to RubyMotion: 13 | .dat* 14 | .repl_history 15 | build/ 16 | 17 | ## Documentation cache and generated files: 18 | /.yardoc/ 19 | /_yardoc/ 20 | /doc/ 21 | /html/ 22 | /rdoc/ 23 | 24 | ## Environment normalisation: 25 | /.bundle/ 26 | /lib/bundler/man/ 27 | 28 | # for a library or gem, you might want to ignore these files since the code is 29 | # intended to run in multiple environments; otherwise, check them in: 30 | Gemfile.lock 31 | gemfiles/*.lock 32 | .ruby-version 33 | .ruby-gemset 34 | 35 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 36 | .rvmrc 37 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'bundler/gem_tasks' 3 | require 'appraisal' 4 | require 'rake/testtask' 5 | 6 | Rake::TestTask.new(:test) do |test| 7 | test.pattern = 'test/**/test_*.rb' 8 | test.warning = true 9 | test.verbose = true 10 | end 11 | task :default => :test 12 | 13 | Rake::TestTask.new(:bench) do |test| 14 | test.pattern = 'test/benchmark_test.rb' 15 | end 16 | 17 | task :test_all do 18 | system('rake test RAILS_VERSION="~> 3.0.0"') 19 | system('rake test RAILS_VERSION=">= 3.0.0"') 20 | end 21 | 22 | # 'gem install rdoc' to upgrade RDoc if this is giving you errors 23 | require 'rdoc/task' 24 | RDoc::Task.new do |rd| 25 | rd.rdoc_files.include("lib/**/*.rb") 26 | end 27 | 28 | require 'rake/clean' 29 | CLEAN.include "**/*.rbc" 30 | CLEAN.include "**/.DS_Store" 31 | -------------------------------------------------------------------------------- /test/test_encoding.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # frozen_string_literal: true 3 | require_relative 'helper' 4 | 5 | describe 'Encoding' do 6 | describe 'using a live server' do 7 | it 'support i18n content' do 8 | memcached_persistent do |dc| 9 | key = 'foo' 10 | utf_key = utf8 = 'ƒ©åÍÎ' 11 | 12 | assert dc.set(key, utf8) 13 | assert_equal utf8, dc.get(key) 14 | 15 | dc.set(utf_key, utf8) 16 | assert_equal utf8, dc.get(utf_key) 17 | end 18 | end 19 | 20 | it 'support content expiry' do 21 | memcached_persistent do |dc| 22 | key = 'foo' 23 | assert dc.set(key, 'bar', 1) 24 | assert_equal 'bar', dc.get(key) 25 | sleep 1.2 26 | assert_nil dc.get(key) 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /test/test_serializer.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # frozen_string_literal: true 3 | require_relative 'helper' 4 | require 'json' 5 | 6 | describe 'Serializer' do 7 | it 'default to Marshal' do 8 | memcached(29198) do |dc| 9 | dc.set 1,2 10 | assert_equal Marshal, dc.instance_variable_get('@ring').servers.first.serializer 11 | end 12 | end 13 | 14 | it 'support a custom serializer' do 15 | memcached(29198) do |dc, port| 16 | memcache = Dalli::Client.new("127.0.0.1:#{port}", :serializer => JSON) 17 | memcache.set 1,2 18 | begin 19 | assert_equal JSON, memcache.instance_variable_get('@ring').servers.first.serializer 20 | 21 | memcached(21956) do |newdc| 22 | assert newdc.set("json_test", {"foo" => "bar"}) 23 | assert_equal({"foo" => "bar"}, newdc.get("json_test")) 24 | end 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /dalli.gemspec: -------------------------------------------------------------------------------- 1 | require './lib/dalli/version' 2 | 3 | Gem::Specification.new do |s| 4 | s.name = "dalli" 5 | s.version = Dalli::VERSION 6 | s.license = "MIT" 7 | 8 | s.authors = ['Peter M. Goldstein', 'Mike Perham'] 9 | s.description = s.summary = 'High performance memcached client for Ruby' 10 | s.email = ['peter.m.goldstein@gmail.com', 'mperham@gmail.com'] 11 | s.files = Dir.glob('lib/**/*') + [ 12 | 'LICENSE', 13 | 'README.md', 14 | 'History.md', 15 | 'Gemfile' 16 | ] 17 | s.homepage = 'https://github.com/petergoldstein/dalli' 18 | s.rdoc_options = ["--charset=UTF-8"] 19 | s.add_development_dependency 'minitest', '>= 4.2.0' 20 | s.add_development_dependency 'mocha' 21 | s.add_development_dependency 'rails', '~> 5' 22 | s.add_development_dependency 'rake' 23 | s.add_development_dependency 'appraisal' 24 | s.add_development_dependency 'connection_pool' 25 | s.add_development_dependency 'rdoc' 26 | s.add_development_dependency 'simplecov' 27 | end 28 | 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Peter M. Goldstein, Mike Perham 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /lib/dalli/options.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'thread' 3 | require 'monitor' 4 | 5 | module Dalli 6 | 7 | # Make Dalli threadsafe by using a lock around all 8 | # public server methods. 9 | # 10 | # Dalli::Server.extend(Dalli::Threadsafe) 11 | # 12 | module Threadsafe 13 | def self.extended(obj) 14 | obj.init_threadsafe 15 | end 16 | 17 | def request(op, *args) 18 | @lock.synchronize do 19 | super 20 | end 21 | end 22 | 23 | def alive? 24 | @lock.synchronize do 25 | super 26 | end 27 | end 28 | 29 | def close 30 | @lock.synchronize do 31 | super 32 | end 33 | end 34 | 35 | def multi_response_start 36 | @lock.synchronize do 37 | super 38 | end 39 | end 40 | 41 | def multi_response_nonblock 42 | @lock.synchronize do 43 | super 44 | end 45 | end 46 | 47 | def multi_response_abort 48 | @lock.synchronize do 49 | super 50 | end 51 | end 52 | 53 | def lock! 54 | @lock.mon_enter 55 | end 56 | 57 | def unlock! 58 | @lock.mon_exit 59 | end 60 | 61 | def init_threadsafe 62 | @lock = Monitor.new 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /lib/dalli.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'dalli/compressor' 3 | require 'dalli/client' 4 | require 'dalli/ring' 5 | require 'dalli/server' 6 | require 'dalli/socket' 7 | require 'dalli/version' 8 | require 'dalli/options' 9 | require 'dalli/railtie' if defined?(::Rails::Railtie) 10 | 11 | module Dalli 12 | # generic error 13 | class DalliError < RuntimeError; end 14 | # socket/server communication error 15 | class NetworkError < DalliError; end 16 | # no server available/alive error 17 | class RingError < DalliError; end 18 | # application error in marshalling serialization 19 | class MarshalError < DalliError; end 20 | # application error in marshalling deserialization or decompression 21 | class UnmarshalError < DalliError; end 22 | # payload too big for memcached 23 | class ValueOverMaxSize < RuntimeError; end 24 | 25 | def self.logger 26 | @logger ||= (rails_logger || default_logger) 27 | end 28 | 29 | def self.rails_logger 30 | (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) || 31 | (defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER.respond_to?(:debug) && RAILS_DEFAULT_LOGGER) 32 | end 33 | 34 | def self.default_logger 35 | require 'logger' 36 | l = Logger.new(STDOUT) 37 | l.level = Logger::INFO 38 | l 39 | end 40 | 41 | def self.logger=(logger) 42 | @logger = logger 43 | end 44 | 45 | end 46 | 47 | if defined?(RAILS_VERSION) && RAILS_VERSION < '3' 48 | raise Dalli::DalliError, "Dalli #{Dalli::VERSION} does not support Rails version < 3.0" 49 | end 50 | -------------------------------------------------------------------------------- /test/test_compressor.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # frozen_string_literal: true 3 | require_relative 'helper' 4 | require 'json' 5 | 6 | class NoopCompressor 7 | def self.compress(data) 8 | data 9 | end 10 | 11 | def self.decompress(data) 12 | data 13 | end 14 | end 15 | 16 | describe 'Compressor' do 17 | 18 | it 'default to Dalli::Compressor' do 19 | memcached(29199) do |dc| 20 | dc.set 1,2 21 | assert_equal Dalli::Compressor, dc.instance_variable_get('@ring').servers.first.compressor 22 | end 23 | end 24 | 25 | it 'support a custom compressor' do 26 | memcached(29199) do |dc| 27 | memcache = Dalli::Client.new('127.0.0.1:29199', :compressor => NoopCompressor) 28 | memcache.set 1,2 29 | begin 30 | assert_equal NoopCompressor, memcache.instance_variable_get('@ring').servers.first.compressor 31 | 32 | memcached(19127) do |newdc| 33 | assert newdc.set("string-test", "a test string") 34 | assert_equal("a test string", newdc.get("string-test")) 35 | end 36 | end 37 | end 38 | end 39 | end 40 | 41 | describe 'GzipCompressor' do 42 | 43 | it 'compress and uncompress data using Zlib::GzipWriter/Reader' do 44 | memcached(19127,nil,{:compress=>true,:compressor=>Dalli::GzipCompressor}) do |dc| 45 | data = (0...1025).map{65.+(rand(26)).chr}.join 46 | assert dc.set("test", data) 47 | assert_equal Dalli::GzipCompressor, dc.instance_variable_get('@ring').servers.first.compressor 48 | assert_equal(data, dc.get("test")) 49 | end 50 | end 51 | 52 | end 53 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | $TESTING = true 3 | require 'bundler/setup' 4 | # require 'simplecov' 5 | # SimpleCov.start 6 | require 'minitest/pride' unless RUBY_ENGINE == 'rbx' 7 | require 'minitest/autorun' 8 | require 'mocha/setup' 9 | require_relative 'memcached_mock' 10 | 11 | ENV['MEMCACHED_SASL_PWDB'] = "#{File.dirname(__FILE__)}/sasl/sasldb" 12 | ENV['SASL_CONF_PATH'] = "#{File.dirname(__FILE__)}/sasl/memcached.conf" 13 | 14 | require 'rails' 15 | puts "Testing with Rails #{Rails.version}" 16 | 17 | require 'dalli' 18 | require 'logger' 19 | 20 | require 'active_support/time' 21 | require 'active_support/cache/dalli_store' 22 | 23 | Dalli.logger = Logger.new(STDOUT) 24 | Dalli.logger.level = Logger::ERROR 25 | 26 | class MiniTest::Spec 27 | include MemcachedMock::Helper 28 | 29 | def assert_error(error, regexp=nil, &block) 30 | ex = assert_raises(error, &block) 31 | assert_match(regexp, ex.message, "#{ex.class.name}: #{ex.message}\n#{ex.backtrace.join("\n\t")}") 32 | end 33 | 34 | def op_cas_succeeds(rsp) 35 | rsp.is_a?(Integer) && rsp > 0 36 | end 37 | 38 | def op_replace_succeeds(rsp) 39 | rsp.is_a?(Integer) && rsp > 0 40 | end 41 | 42 | # add and set must have the same return value because of DalliStore#write_entry 43 | def op_addset_succeeds(rsp) 44 | rsp.is_a?(Integer) && rsp > 0 45 | end 46 | 47 | def with_activesupport 48 | require 'active_support/all' 49 | require 'active_support/cache/dalli_store' 50 | yield 51 | end 52 | 53 | def with_actionpack 54 | require 'action_dispatch' 55 | require 'action_controller' 56 | yield 57 | end 58 | 59 | def with_connectionpool 60 | require 'connection_pool' 61 | yield 62 | end 63 | 64 | def with_nil_logger 65 | old = Dalli.logger 66 | Dalli.logger = Logger.new(nil) 67 | begin 68 | yield 69 | ensure 70 | Dalli.logger = old 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/dalli/cas/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'dalli/client' 3 | 4 | module Dalli 5 | class Client 6 | ## 7 | # Get the value and CAS ID associated with the key. If a block is provided, 8 | # value and CAS will be passed to the block. 9 | def get_cas(key) 10 | (value, cas) = perform(:cas, key) 11 | value = (!value || value == 'Not found') ? nil : value 12 | if block_given? 13 | yield value, cas 14 | else 15 | [value, cas] 16 | end 17 | end 18 | 19 | ## 20 | # Fetch multiple keys efficiently, including available metadata such as CAS. 21 | # If a block is given, yields key/data pairs one a time. Data is an array: 22 | # [value, cas_id] 23 | # If no block is given, returns a hash of 24 | # { 'key' => [value, cas_id] } 25 | def get_multi_cas(*keys) 26 | if block_given? 27 | get_multi_yielder(keys) {|*args| yield(*args)} 28 | else 29 | Hash.new.tap do |hash| 30 | get_multi_yielder(keys) {|k, data| hash[k] = data} 31 | end 32 | end 33 | end 34 | 35 | ## 36 | # Set the key-value pair, verifying existing CAS. 37 | # Returns the resulting CAS value if succeeded, and falsy otherwise. 38 | def set_cas(key, value, cas, ttl=nil, options=nil) 39 | ttl ||= @options[:expires_in].to_i 40 | perform(:set, key, value, ttl, cas, options) 41 | end 42 | 43 | ## 44 | # Conditionally add a key/value pair, verifying existing CAS, only if the 45 | # key already exists on the server. Returns the new CAS value if the 46 | # operation succeeded, or falsy otherwise. 47 | def replace_cas(key, value, cas, ttl=nil, options=nil) 48 | ttl ||= @options[:expires_in].to_i 49 | perform(:replace, key, value, ttl, cas, options) 50 | end 51 | 52 | # Delete a key/value pair, verifying existing CAS. 53 | # Returns true if succeeded, and falsy otherwise. 54 | def delete_cas(key, cas=0) 55 | perform(:delete, key, cas) 56 | end 57 | 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /test/test_network.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | 4 | describe 'Network' do 5 | 6 | describe 'assuming a bad network' do 7 | 8 | it 'handle no server available' do 9 | assert_raises Dalli::RingError, :message => "No server available" do 10 | dc = Dalli::Client.new 'localhost:19333' 11 | dc.get 'foo' 12 | end 13 | end 14 | 15 | describe 'with a fake server' do 16 | it 'handle connection reset' do 17 | memcached_mock(lambda {|sock| sock.close }) do 18 | assert_raises Dalli::RingError, :message => "No server available" do 19 | dc = Dalli::Client.new('localhost:19123') 20 | dc.get('abc') 21 | end 22 | end 23 | end 24 | 25 | it 'handle connection reset with unix socket' do 26 | socket_path = MemcachedMock::UNIX_SOCKET_PATH 27 | memcached_mock(lambda {|sock| sock.close }, :start_unix, socket_path) do 28 | assert_raises Dalli::RingError, :message => "No server available" do 29 | dc = Dalli::Client.new(socket_path) 30 | dc.get('abc') 31 | end 32 | end 33 | end 34 | 35 | it 'handle malformed response' do 36 | memcached_mock(lambda {|sock| sock.write('123') }) do 37 | assert_raises Dalli::RingError, :message => "No server available" do 38 | dc = Dalli::Client.new('localhost:19123') 39 | dc.get('abc') 40 | end 41 | end 42 | end 43 | 44 | it 'handle connect timeouts' do 45 | memcached_mock(lambda {|sock| sleep(0.6); sock.close }, :delayed_start) do 46 | assert_raises Dalli::RingError, :message => "No server available" do 47 | dc = Dalli::Client.new('localhost:19123') 48 | dc.get('abc') 49 | end 50 | end 51 | end 52 | 53 | it 'handle read timeouts' do 54 | memcached_mock(lambda {|sock| sleep(0.6); sock.write('giraffe') }) do 55 | assert_raises Dalli::RingError, :message => "No server available" do 56 | dc = Dalli::Client.new('localhost:19123') 57 | dc.get('abc') 58 | end 59 | end 60 | end 61 | 62 | end 63 | 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /Performance.md: -------------------------------------------------------------------------------- 1 | Performance 2 | ==================== 3 | 4 | Caching is all about performance, so I carefully track Dalli performance to ensure no regressions. 5 | You can optionally use kgio to give Dalli a 10-20% performance boost: `gem install kgio`. 6 | 7 | Note I've added some benchmarks over time to Dalli that the other libraries don't necessarily have. 8 | 9 | memcache-client 10 | --------------- 11 | 12 | Testing 1.8.5 with ruby 1.9.3p0 (2011-10-30 revision 33570) [x86_64-darwin11.2.0] 13 | 14 | user system total real 15 | set:plain:memcache-client 1.860000 0.310000 2.170000 ( 2.188030) 16 | set:ruby:memcache-client 1.830000 0.290000 2.120000 ( 2.130212) 17 | get:plain:memcache-client 1.830000 0.340000 2.170000 ( 2.176156) 18 | get:ruby:memcache-client 1.900000 0.330000 2.230000 ( 2.235045) 19 | multiget:ruby:memcache-client 0.860000 0.120000 0.980000 ( 0.987348) 20 | missing:ruby:memcache-client 1.630000 0.320000 1.950000 ( 1.954867) 21 | mixed:ruby:memcache-client 3.690000 0.670000 4.360000 ( 4.364469) 22 | 23 | 24 | dalli 25 | ----- 26 | 27 | Testing with Rails 3.2.1 28 | Using kgio socket IO 29 | Testing 2.0.0 with ruby 1.9.3p125 (2012-02-16 revision 34643) [x86_64-darwin11.3.0] 30 | 31 | user system total real 32 | mixed:rails:dalli 1.580000 0.570000 2.150000 ( 3.008839) 33 | set:plain:dalli 0.730000 0.300000 1.030000 ( 1.567098) 34 | setq:plain:dalli 0.520000 0.120000 0.640000 ( 0.634402) 35 | set:ruby:dalli 0.800000 0.300000 1.100000 ( 1.640348) 36 | get:plain:dalli 0.840000 0.330000 1.170000 ( 1.668425) 37 | get:ruby:dalli 0.850000 0.330000 1.180000 ( 1.665716) 38 | multiget:ruby:dalli 0.700000 0.260000 0.960000 ( 0.965423) 39 | missing:ruby:dalli 0.720000 0.320000 1.040000 ( 1.511720) 40 | mixed:ruby:dalli 1.660000 0.640000 2.300000 ( 3.320743) 41 | mixedq:ruby:dalli 1.630000 0.510000 2.140000 ( 2.629734) 42 | incr:ruby:dalli 0.270000 0.100000 0.370000 ( 0.547618) 43 | -------------------------------------------------------------------------------- /code_of_conduct.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of 4 | fostering an open and welcoming community, we pledge to respect all people who 5 | contribute through reporting issues, posting feature requests, updating 6 | documentation, submitting pull requests or patches, and other activities. 7 | 8 | We are committed to making participation in this project a harassment-free 9 | experience for everyone, regardless of level of experience, gender, gender 10 | identity and expression, sexual orientation, disability, personal appearance, 11 | body size, race, ethnicity, age, religion, or nationality. 12 | 13 | Examples of unacceptable behavior by participants include: 14 | 15 | * The use of sexualized language or imagery 16 | * Personal attacks 17 | * Trolling or insulting/derogatory comments 18 | * Public or private harassment 19 | * Publishing other's private information, such as physical or electronic 20 | addresses, without explicit permission 21 | * Other unethical or unprofessional conduct 22 | 23 | Project maintainers have the right and responsibility to remove, edit, or 24 | reject comments, commits, code, wiki edits, issues, and other contributions 25 | that are not aligned to this Code of Conduct, or to ban temporarily or 26 | permanently any contributor for other behaviors that they deem inappropriate, 27 | threatening, offensive, or harmful. 28 | 29 | By adopting this Code of Conduct, project maintainers commit themselves to 30 | fairly and consistently applying these principles to every aspect of managing 31 | this project. Project maintainers who do not follow or enforce the Code of 32 | Conduct may be permanently removed from the project team. 33 | 34 | This Code of Conduct applies both within project spaces and in public spaces 35 | when an individual is representing the project or its community. 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 38 | reported by contacting the project maintainer at peter.m.goldstein AT gmail.com. All 39 | complaints will be reviewed and investigated and will result in a response that 40 | is deemed necessary and appropriate to the circumstances. Maintainers are 41 | obligated to maintain confidentiality with regard to the reporter of an 42 | incident. 43 | 44 | 45 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 46 | version 1.3.0, available at 47 | [http://contributor-covenant.org/version/1/3/0/][version] 48 | 49 | [homepage]: http://contributor-covenant.org 50 | [version]: http://contributor-covenant.org/version/1/3/0/ 51 | -------------------------------------------------------------------------------- /lib/action_dispatch/middleware/session/dalli_store.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'active_support/cache' 3 | require 'action_dispatch/middleware/session/abstract_store' 4 | require 'dalli' 5 | 6 | # Dalli-based session store for Rails 3.0. 7 | module ActionDispatch 8 | module Session 9 | class DalliStore < AbstractStore 10 | def initialize(app, options = {}) 11 | # Support old :expires option 12 | options[:expire_after] ||= options[:expires] 13 | 14 | super 15 | 16 | @default_options = { :namespace => 'rack:session' }.merge(@default_options) 17 | 18 | @pool = options[:cache] || begin 19 | Dalli::Client.new( 20 | @default_options[:memcache_server], @default_options) 21 | end 22 | @namespace = @default_options[:namespace] 23 | 24 | @raise_errors = !!@default_options[:raise_errors] 25 | 26 | super 27 | end 28 | 29 | def reset 30 | @pool.reset 31 | end 32 | 33 | private 34 | 35 | def get_session(env, sid) 36 | sid = generate_sid unless sid and !sid.empty? 37 | begin 38 | session = @pool.get(sid) || {} 39 | rescue Dalli::DalliError => ex 40 | # re-raise ArgumentError so Rails' session abstract_store.rb can autoload any missing models 41 | raise ArgumentError, ex.message if ex.message =~ /unmarshal/ 42 | Rails.logger.warn("Session::DalliStore#get: #{ex.message}") 43 | session = {} 44 | end 45 | [sid, session] 46 | end 47 | 48 | def set_session(env, sid, session_data, options = nil) 49 | options ||= env[ENV_SESSION_OPTIONS_KEY] 50 | expiry = options[:expire_after] 51 | @pool.set(sid, session_data, expiry) 52 | sid 53 | rescue Dalli::DalliError 54 | Rails.logger.warn("Session::DalliStore#set: #{$!.message}") 55 | raise if @raise_errors 56 | false 57 | end 58 | 59 | def destroy_session(env, session_id, options) 60 | begin 61 | @pool.delete(session_id) 62 | rescue Dalli::DalliError 63 | Rails.logger.warn("Session::DalliStore#destroy_session: #{$!.message}") 64 | raise if @raise_errors 65 | end 66 | return nil if options[:drop] 67 | generate_sid 68 | end 69 | 70 | def destroy(env) 71 | if sid = current_session_id(env) 72 | @pool.delete(sid) 73 | end 74 | rescue Dalli::DalliError 75 | Rails.logger.warn("Session::DalliStore#destroy: #{$!.message}") 76 | raise if @raise_errors 77 | false 78 | end 79 | 80 | end 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /test/test_ring.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | 4 | describe 'Ring' do 5 | 6 | describe 'a ring of servers' do 7 | 8 | it "have the continuum sorted by value" do 9 | servers = [stub(:name => "localhost:11211", :weight => 1), 10 | stub(:name => "localhost:9500", :weight => 1)] 11 | ring = Dalli::Ring.new(servers, {}) 12 | previous_value = 0 13 | ring.continuum.each do |entry| 14 | assert entry.value > previous_value 15 | previous_value = entry.value 16 | end 17 | end 18 | 19 | it 'raise when no servers are available/defined' do 20 | ring = Dalli::Ring.new([], {}) 21 | assert_raises Dalli::RingError, :message => "No server available" do 22 | ring.server_for_key('test') 23 | end 24 | end 25 | 26 | describe 'containing only a single server' do 27 | it "raise correctly when it's not alive" do 28 | servers = [ 29 | Dalli::Server.new("localhost:12345"), 30 | ] 31 | ring = Dalli::Ring.new(servers, {}) 32 | assert_raises Dalli::RingError, :message => "No server available" do 33 | ring.server_for_key('test') 34 | end 35 | end 36 | 37 | it "return the server when it's alive" do 38 | servers = [ 39 | Dalli::Server.new("localhost:19191"), 40 | ] 41 | ring = Dalli::Ring.new(servers, {}) 42 | memcached(19191) do |mc| 43 | ring = mc.send(:ring) 44 | assert_equal ring.servers.first.port, ring.server_for_key('test').port 45 | end 46 | end 47 | end 48 | 49 | describe 'containing multiple servers' do 50 | it "raise correctly when no server is alive" do 51 | servers = [ 52 | Dalli::Server.new("localhost:12345"), 53 | Dalli::Server.new("localhost:12346"), 54 | ] 55 | ring = Dalli::Ring.new(servers, {}) 56 | assert_raises Dalli::RingError, :message => "No server available" do 57 | ring.server_for_key('test') 58 | end 59 | end 60 | 61 | it "return an alive server when at least one is alive" do 62 | servers = [ 63 | Dalli::Server.new("localhost:12346"), 64 | Dalli::Server.new("localhost:19191"), 65 | ] 66 | ring = Dalli::Ring.new(servers, {}) 67 | memcached(19191) do |mc| 68 | ring = mc.send(:ring) 69 | assert_equal ring.servers.first.port, ring.server_for_key('test').port 70 | end 71 | end 72 | end 73 | 74 | it 'detect when a dead server is up again' do 75 | memcached(19997) do 76 | down_retry_delay = 0.5 77 | dc = Dalli::Client.new(['localhost:19997', 'localhost:19998'], :down_retry_delay => down_retry_delay) 78 | assert_equal 1, dc.stats.values.compact.count 79 | 80 | memcached(19998) do 81 | assert_equal 2, dc.stats.values.compact.count 82 | end 83 | end 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /test/test_cas_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | 4 | describe 'Dalli::Cas::Client' do 5 | describe 'using a live server' do 6 | it 'supports get with CAS' do 7 | memcached_cas_persistent do |dc| 8 | dc.flush 9 | 10 | expected = { 'blah' => 'blerg!' } 11 | get_block_called = false 12 | stored_value = stored_cas = nil 13 | # Validate call-with-block 14 | dc.get_cas('gets_key') do |v, cas| 15 | get_block_called = true 16 | stored_value = v 17 | stored_cas = cas 18 | end 19 | assert get_block_called 20 | assert_nil stored_value 21 | 22 | dc.set('gets_key', expected) 23 | 24 | # Validate call-with-return-value 25 | stored_value, stored_cas = dc.get_cas('gets_key') 26 | assert_equal stored_value, expected 27 | assert(stored_cas != 0) 28 | end 29 | end 30 | 31 | it 'supports multi-get with CAS' do 32 | memcached_cas_persistent do |dc| 33 | dc.close 34 | dc.flush 35 | 36 | expected_hash = {'a' => 'foo', 'b' => 123} 37 | expected_hash.each_pair do |k, v| 38 | dc.set(k, v) 39 | end 40 | 41 | # Invocation without block 42 | resp = dc.get_multi_cas(%w(a b c d e f)) 43 | resp.each_pair do |k, data| 44 | value, cas = [data.first, data[1]] 45 | assert_equal expected_hash[k], value 46 | assert(cas && cas != 0) 47 | end 48 | 49 | # Invocation with block 50 | dc.get_multi_cas(%w(a b c d e f)) do |k, data| 51 | value, cas = [data.first, data[1]] 52 | assert_equal expected_hash[k], value 53 | assert(cas && cas != 0) 54 | end 55 | end 56 | end 57 | 58 | it 'supports replace-with-CAS operation' do 59 | memcached_cas_persistent do |dc| 60 | dc.flush 61 | cas = dc.set('key', 'value') 62 | 63 | # Accepts CAS, replaces, and returns new CAS 64 | cas = dc.replace_cas('key', 'value2', cas) 65 | assert cas.is_a?(Integer) 66 | 67 | assert_equal 'value2', dc.get('key') 68 | end 69 | end 70 | 71 | it 'supports delete with CAS' do 72 | memcached_cas_persistent do |dc| 73 | cas = dc.set('some_key', 'some_value') 74 | dc.delete_cas('some_key', cas) 75 | assert_nil dc.get('some_key') 76 | end 77 | end 78 | 79 | it 'handles CAS round-trip operations' do 80 | memcached_cas_persistent do |dc| 81 | dc.flush 82 | 83 | expected = {'blah' => 'blerg!'} 84 | dc.set('some_key', expected) 85 | 86 | value, cas = dc.get_cas('some_key') 87 | assert_equal value, expected 88 | assert(!cas.nil? && cas != 0) 89 | 90 | # Set operation, first with wrong then with correct CAS 91 | expected = {'blah' => 'set succeeded'} 92 | assert(dc.set_cas('some_key', expected, cas+1) == false) 93 | assert op_addset_succeeds(cas = dc.set_cas('some_key', expected, cas)) 94 | 95 | # Replace operation, first with wrong then with correct CAS 96 | expected = {'blah' => 'replace succeeded'} 97 | assert(dc.replace_cas('some_key', expected, cas+1) == false) 98 | assert op_addset_succeeds(cas = dc.replace_cas('some_key', expected, cas)) 99 | 100 | # Delete operation, first with wrong then with correct CAS 101 | assert(dc.delete_cas('some_key', cas+1) == false) 102 | assert dc.delete_cas('some_key', cas) 103 | end 104 | end 105 | 106 | end 107 | end 108 | -------------------------------------------------------------------------------- /test/test_sasl.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | 4 | describe 'Sasl' do 5 | 6 | # https://github.com/seattlerb/minitest/issues/298 7 | def self.xit(msg, &block) 8 | end 9 | 10 | describe 'a server requiring authentication' do 11 | before do 12 | @server = mock() 13 | @server.stubs(:request).returns(true) 14 | @server.stubs(:weight).returns(1) 15 | @server.stubs(:name).returns("localhost:19124") 16 | end 17 | 18 | describe 'without authentication credentials' do 19 | before do 20 | ENV['MEMCACHE_USERNAME'] = 'foo' 21 | ENV['MEMCACHE_PASSWORD'] = 'wrongpwd' 22 | end 23 | 24 | after do 25 | ENV['MEMCACHE_USERNAME'] = nil 26 | ENV['MEMCACHE_PASSWORD'] = nil 27 | end 28 | 29 | xit 'gracefully handle authentication failures' do 30 | memcached_sasl_persistent do |dc| 31 | assert_error Dalli::DalliError, /32/ do 32 | dc.set('abc', 123) 33 | end 34 | end 35 | end 36 | end 37 | 38 | xit 'fail SASL authentication with wrong options' do 39 | memcached_sasl_persistent do |dc, port| 40 | dc = Dalli::Client.new("localhost:#{port}", :username => 'testuser', :password => 'testtest') 41 | assert_error Dalli::DalliError, /32/ do 42 | dc.set('abc', 123) 43 | end 44 | end 45 | end 46 | 47 | # OSX: Create a SASL user for the memcached application like so: 48 | # 49 | # saslpasswd2 -a memcached -c testuser 50 | # 51 | # with password 'testtest' 52 | describe 'in an authenticated environment' do 53 | before do 54 | ENV['MEMCACHE_USERNAME'] = 'testuser' 55 | ENV['MEMCACHE_PASSWORD'] = 'testtest' 56 | end 57 | 58 | after do 59 | ENV['MEMCACHE_USERNAME'] = nil 60 | ENV['MEMCACHE_PASSWORD'] = nil 61 | end 62 | 63 | xit 'pass SASL authentication' do 64 | memcached_sasl_persistent do |dc| 65 | # I get "Dalli::DalliError: Error authenticating: 32" in OSX 66 | # but SASL works on Heroku servers. YMMV. 67 | assert_equal true, dc.set('abc', 123) 68 | assert_equal 123, dc.get('abc') 69 | results = dc.stats 70 | assert_equal 1, results.size 71 | assert_equal 38, results.values.first.size 72 | end 73 | end 74 | end 75 | 76 | xit 'pass SASL authentication with options' do 77 | memcached_sasl_persistent do |dc, port| 78 | dc = Dalli::Client.new("localhost:#{port}", sasl_credentials) 79 | # I get "Dalli::DalliError: Error authenticating: 32" in OSX 80 | # but SASL works on Heroku servers. YMMV. 81 | assert_equal true, dc.set('abc', 123) 82 | assert_equal 123, dc.get('abc') 83 | results = dc.stats 84 | assert_equal 1, results.size 85 | assert_equal 38, results.values.first.size 86 | end 87 | end 88 | 89 | it 'pass SASL as URI' do 90 | Dalli::Server.expects(:new).with("localhost:19124", 91 | :username => "testuser", :password => "testtest").returns(@server) 92 | dc = Dalli::Client.new('memcached://testuser:testtest@localhost:19124') 93 | dc.flush_all 94 | end 95 | 96 | it 'pass SASL as ring of URIs' do 97 | Dalli::Server.expects(:new).with("localhost:19124", 98 | :username => "testuser", :password => "testtest").returns(@server) 99 | Dalli::Server.expects(:new).with("otherhost:19125", 100 | :username => "testuser2", :password => "testtest2").returns(@server) 101 | dc = Dalli::Client.new(['memcached://testuser:testtest@localhost:19124', 102 | 'memcached://testuser2:testtest2@otherhost:19125']) 103 | dc.flush_all 104 | end 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /lib/dalli/ring.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'digest/sha1' 3 | require 'zlib' 4 | 5 | module Dalli 6 | class Ring 7 | POINTS_PER_SERVER = 160 # this is the default in libmemcached 8 | 9 | attr_accessor :servers, :continuum 10 | 11 | def initialize(servers, options) 12 | @servers = servers 13 | @continuum = nil 14 | if servers.size > 1 15 | total_weight = servers.inject(0) { |memo, srv| memo + srv.weight } 16 | continuum = [] 17 | servers.each do |server| 18 | entry_count_for(server, servers.size, total_weight).times do |idx| 19 | hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}") 20 | value = Integer("0x#{hash[0..7]}") 21 | continuum << Dalli::Ring::Entry.new(value, server) 22 | end 23 | end 24 | @continuum = continuum.sort_by(&:value) 25 | end 26 | 27 | threadsafe! unless options[:threadsafe] == false 28 | @failover = options[:failover] != false 29 | end 30 | 31 | def server_for_key(key) 32 | if @continuum 33 | hkey = hash_for(key) 34 | 20.times do |try| 35 | entryidx = binary_search(@continuum, hkey) 36 | server = @continuum[entryidx].server 37 | return server if server.alive? 38 | break unless @failover 39 | hkey = hash_for("#{try}#{key}") 40 | end 41 | else 42 | server = @servers.first 43 | return server if server && server.alive? 44 | end 45 | 46 | raise Dalli::RingError, "No server available" 47 | end 48 | 49 | def lock 50 | @servers.each(&:lock!) 51 | begin 52 | return yield 53 | ensure 54 | @servers.each(&:unlock!) 55 | end 56 | end 57 | 58 | private 59 | 60 | def threadsafe! 61 | @servers.each do |s| 62 | s.extend(Dalli::Threadsafe) 63 | end 64 | end 65 | 66 | def hash_for(key) 67 | Zlib.crc32(key) 68 | end 69 | 70 | def entry_count_for(server, total_servers, total_weight) 71 | ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor 72 | end 73 | 74 | # Native extension to perform the binary search within the continuum 75 | # space. Fallback to a pure Ruby version if the compilation doesn't work. 76 | # optional for performance and only necessary if you are using multiple 77 | # memcached servers. 78 | begin 79 | require 'inline' 80 | inline do |builder| 81 | builder.c <<-EOM 82 | int binary_search(VALUE ary, unsigned int r) { 83 | long upper = RARRAY_LEN(ary) - 1; 84 | long lower = 0; 85 | long idx = 0; 86 | ID value = rb_intern("value"); 87 | VALUE continuumValue; 88 | unsigned int l; 89 | 90 | while (lower <= upper) { 91 | idx = (lower + upper) / 2; 92 | 93 | continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0); 94 | l = NUM2UINT(continuumValue); 95 | if (l == r) { 96 | return idx; 97 | } 98 | else if (l > r) { 99 | upper = idx - 1; 100 | } 101 | else { 102 | lower = idx + 1; 103 | } 104 | } 105 | return upper; 106 | } 107 | EOM 108 | end 109 | rescue LoadError 110 | # Find the closest index in the Ring with value <= the given value 111 | def binary_search(ary, value) 112 | upper = ary.size - 1 113 | lower = 0 114 | 115 | while (lower <= upper) do 116 | idx = (lower + upper) / 2 117 | comp = ary[idx].value <=> value 118 | 119 | if comp == 0 120 | return idx 121 | elsif comp > 0 122 | upper = idx - 1 123 | else 124 | lower = idx + 1 125 | end 126 | end 127 | upper 128 | end 129 | end 130 | 131 | class Entry 132 | attr_reader :value 133 | attr_reader :server 134 | 135 | def initialize(val, srv) 136 | @value = val 137 | @server = srv 138 | end 139 | end 140 | 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /test/test_failover.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | 4 | describe 'failover' do 5 | 6 | describe 'timeouts' do 7 | it 'not lead to corrupt sockets' do 8 | memcached_persistent do |dc| 9 | value = {:test => "123"} 10 | begin 11 | Timeout.timeout 0.01 do 12 | start_time = Time.now 13 | 10_000.times do 14 | dc.set("test_123", value) 15 | end 16 | flunk("Did not timeout in #{Time.now - start_time}") 17 | end 18 | rescue Timeout::Error 19 | end 20 | 21 | assert_equal(value, dc.get("test_123")) 22 | end 23 | end 24 | end 25 | 26 | 27 | describe 'assuming some bad servers' do 28 | 29 | it 'silently reconnect if server hiccups' do 30 | server_port = 30124 31 | memcached_persistent(server_port) do |dc, port| 32 | dc.set 'foo', 'bar' 33 | foo = dc.get 'foo' 34 | assert_equal foo, 'bar' 35 | 36 | memcached_kill(port) 37 | memcached_persistent(port) do 38 | 39 | foo = dc.get 'foo' 40 | assert_nil foo 41 | 42 | memcached_kill(port) 43 | end 44 | end 45 | end 46 | 47 | it 'handle graceful failover' do 48 | port_1 = 31777 49 | port_2 = 32113 50 | memcached_persistent(port_1) do |first_dc, first_port| 51 | memcached_persistent(port_2) do |second_dc, second_port| 52 | dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] 53 | dc.set 'foo', 'bar' 54 | foo = dc.get 'foo' 55 | assert_equal foo, 'bar' 56 | 57 | memcached_kill(first_port) 58 | 59 | dc.set 'foo', 'bar' 60 | foo = dc.get 'foo' 61 | assert_equal foo, 'bar' 62 | 63 | memcached_kill(second_port) 64 | 65 | assert_raises Dalli::RingError, :message => "No server available" do 66 | dc.set 'foo', 'bar' 67 | end 68 | end 69 | end 70 | end 71 | 72 | it 'handle them gracefully in get_multi' do 73 | port_1 = 32971 74 | port_2 = 34312 75 | memcached_persistent(port_1) do |first_dc, first_port| 76 | memcached(port_2) do |second_dc, second_port| 77 | dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] 78 | dc.set 'a', 'a1' 79 | result = dc.get_multi ['a'] 80 | assert_equal result, {'a' => 'a1'} 81 | 82 | memcached_kill(first_port) 83 | 84 | result = dc.get_multi ['a'] 85 | assert_equal result, {'a' => 'a1'} 86 | end 87 | end 88 | end 89 | 90 | it 'handle graceful failover in get_multi' do 91 | port_1 = 34541 92 | port_2 = 33044 93 | memcached_persistent(port_1) do |first_dc, first_port| 94 | memcached_persistent(port_2) do |second_dc, second_port| 95 | dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] 96 | dc.set 'foo', 'foo1' 97 | dc.set 'bar', 'bar1' 98 | result = dc.get_multi ['foo', 'bar'] 99 | assert_equal result, {'foo' => 'foo1', 'bar' => 'bar1'} 100 | 101 | memcached_kill(first_port) 102 | 103 | dc.set 'foo', 'foo1' 104 | dc.set 'bar', 'bar1' 105 | result = dc.get_multi ['foo', 'bar'] 106 | assert_equal result, {'foo' => 'foo1', 'bar' => 'bar1'} 107 | 108 | memcached_kill(second_port) 109 | 110 | result = dc.get_multi ['foo', 'bar'] 111 | assert_equal result, {} 112 | end 113 | end 114 | end 115 | 116 | it 'stats it still properly report' do 117 | port_1 = 34547 118 | port_2 = 33219 119 | memcached_persistent(port_1) do |first_dc, first_port| 120 | memcached_persistent(port_2) do |second_dc, second_port| 121 | dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] 122 | result = dc.stats 123 | assert_instance_of Hash, result["localhost:#{first_port}"] 124 | assert_instance_of Hash, result["localhost:#{second_port}"] 125 | 126 | memcached_kill(first_port) 127 | 128 | dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] 129 | result = dc.stats 130 | assert_instance_of NilClass, result["localhost:#{first_port}"] 131 | assert_instance_of Hash, result["localhost:#{second_port}"] 132 | 133 | memcached_kill(second_port) 134 | end 135 | end 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /lib/dalli/socket.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'rbconfig' 3 | 4 | module Dalli::Server::TCPSocketOptions 5 | def setsockopts(sock, options) 6 | sock.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true) 7 | sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if options[:keepalive] 8 | sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF, options[:rcvbuf]) if options[:rcvbuf] 9 | sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, options[:sndbuf]) if options[:sndbuf] 10 | end 11 | end 12 | 13 | begin 14 | require 'kgio' 15 | puts "Using kgio socket IO" if defined?($TESTING) && $TESTING 16 | 17 | class Dalli::Server::KSocket < Kgio::Socket 18 | attr_accessor :options, :server 19 | 20 | def kgio_wait_readable 21 | IO.select([self], nil, nil, options[:socket_timeout]) || raise(Timeout::Error, "IO timeout") 22 | end 23 | 24 | def kgio_wait_writable 25 | IO.select(nil, [self], nil, options[:socket_timeout]) || raise(Timeout::Error, "IO timeout") 26 | end 27 | 28 | alias :write :kgio_write 29 | 30 | def readfull(count) 31 | value = String.new('') 32 | while true 33 | value << kgio_read!(count - value.bytesize) 34 | break if value.bytesize == count 35 | end 36 | value 37 | end 38 | 39 | def read_available 40 | value = String.new('') 41 | while true 42 | ret = kgio_tryread(8196) 43 | case ret 44 | when nil 45 | raise EOFError, 'end of stream' 46 | when :wait_readable 47 | break 48 | else 49 | value << ret 50 | end 51 | end 52 | value 53 | end 54 | end 55 | 56 | class Dalli::Server::KSocket::TCP < Dalli::Server::KSocket 57 | extend Dalli::Server::TCPSocketOptions 58 | 59 | def self.open(host, port, server, options = {}) 60 | addr = Socket.pack_sockaddr_in(port, host) 61 | sock = start(addr) 62 | setsockopts(sock, options) 63 | sock.options = options 64 | sock.server = server 65 | sock.kgio_wait_writable 66 | sock 67 | rescue Timeout::Error 68 | sock.close if sock 69 | raise 70 | end 71 | end 72 | 73 | class Dalli::Server::KSocket::UNIX < Dalli::Server::KSocket 74 | def self.open(path, server, options = {}) 75 | addr = Socket.pack_sockaddr_un(path) 76 | sock = start(addr) 77 | sock.options = options 78 | sock.server = server 79 | sock.kgio_wait_writable 80 | sock 81 | rescue Timeout::Error 82 | sock.close if sock 83 | raise 84 | end 85 | end 86 | 87 | if ::Kgio.respond_to?(:wait_readable=) 88 | ::Kgio.wait_readable = :kgio_wait_readable 89 | ::Kgio.wait_writable = :kgio_wait_writable 90 | end 91 | 92 | rescue LoadError 93 | 94 | puts "Using standard socket IO (#{RUBY_DESCRIPTION})" if defined?($TESTING) && $TESTING 95 | module Dalli::Server::KSocket 96 | module InstanceMethods 97 | def readfull(count) 98 | value = String.new('') 99 | begin 100 | while true 101 | value << read_nonblock(count - value.bytesize) 102 | break if value.bytesize == count 103 | end 104 | rescue Errno::EAGAIN, Errno::EWOULDBLOCK 105 | if IO.select([self], nil, nil, options[:socket_timeout]) 106 | retry 107 | else 108 | safe_options = options.reject{|k,v| [:username, :password].include? k} 109 | raise Timeout::Error, "IO timeout: #{safe_options.inspect}" 110 | end 111 | end 112 | value 113 | end 114 | 115 | def read_available 116 | value = String.new('') 117 | while true 118 | begin 119 | value << read_nonblock(8196) 120 | rescue Errno::EAGAIN, Errno::EWOULDBLOCK 121 | break 122 | end 123 | end 124 | value 125 | end 126 | end 127 | 128 | def self.included(receiver) 129 | receiver.send(:attr_accessor, :options, :server) 130 | receiver.send(:include, InstanceMethods) 131 | end 132 | end 133 | 134 | class Dalli::Server::KSocket::TCP < TCPSocket 135 | extend Dalli::Server::TCPSocketOptions 136 | include Dalli::Server::KSocket 137 | 138 | def self.open(host, port, server, options = {}) 139 | Timeout.timeout(options[:socket_timeout]) do 140 | sock = new(host, port) 141 | setsockopts(sock, options) 142 | sock.options = {:host => host, :port => port}.merge(options) 143 | sock.server = server 144 | sock 145 | end 146 | end 147 | end 148 | 149 | if RbConfig::CONFIG['host_os'] =~ /mingw|mswin/ 150 | class Dalli::Server::KSocket::UNIX 151 | def initialize(*args) 152 | raise Dalli::DalliError, "Unix sockets are not supported on Windows platform." 153 | end 154 | end 155 | else 156 | class Dalli::Server::KSocket::UNIX < UNIXSocket 157 | include Dalli::Server::KSocket 158 | 159 | def self.open(path, server, options = {}) 160 | Timeout.timeout(options[:socket_timeout]) do 161 | sock = new(path) 162 | sock.options = {:path => path}.merge(options) 163 | sock.server = server 164 | sock 165 | end 166 | end 167 | end 168 | 169 | end 170 | end 171 | -------------------------------------------------------------------------------- /test/memcached_mock.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require "socket" 3 | require "tempfile" 4 | 5 | $started = {} 6 | 7 | module MemcachedMock 8 | UNIX_SOCKET_PATH = (f = Tempfile.new('dalli_test'); f.close; f.path) 9 | 10 | def self.start(port=19123) 11 | server = TCPServer.new("localhost", port) 12 | session = server.accept 13 | yield(session) 14 | end 15 | 16 | def self.start_unix(path=UNIX_SOCKET_PATH) 17 | begin 18 | File.delete(path) 19 | rescue Errno::ENOENT 20 | end 21 | server = UNIXServer.new(path) 22 | session = server.accept 23 | yield(session) 24 | end 25 | 26 | def self.delayed_start(port=19123, wait=1) 27 | server = TCPServer.new("localhost", port) 28 | sleep wait 29 | yield(server) 30 | end 31 | 32 | module Helper 33 | # Forks the current process and starts a new mock Memcached server on 34 | # port 22122. 35 | # 36 | # memcached_mock(lambda {|sock| socket.write('123') }) do 37 | # assert_equal "PONG", Dalli::Client.new('localhost:22122').get('abc') 38 | # end 39 | # 40 | def memcached_mock(proc, meth=:start, meth_args=[]) 41 | return unless supports_fork? 42 | begin 43 | pid = fork do 44 | trap("TERM") { exit } 45 | 46 | MemcachedMock.send(meth, *meth_args) do |*args| 47 | proc.call(*args) 48 | end 49 | end 50 | 51 | sleep 0.3 # Give time for the socket to start listening. 52 | yield 53 | ensure 54 | if pid 55 | Process.kill("TERM", pid) 56 | Process.wait(pid) 57 | end 58 | end 59 | end 60 | 61 | PATHS = %w( 62 | /usr/local/bin/ 63 | /opt/local/bin/ 64 | /usr/bin/ 65 | ) 66 | 67 | def find_memcached 68 | output = `memcached -h | head -1`.strip 69 | if output && output =~ /^memcached (\d.\d.\d+)/ && $1 > '1.4' 70 | return (puts "Found #{output} in PATH"; '') 71 | end 72 | PATHS.each do |path| 73 | output = `memcached -h | head -1`.strip 74 | if output && output =~ /^memcached (\d\.\d\.\d+)/ && $1 > '1.4' 75 | return (puts "Found #{output} in #{path}"; path) 76 | end 77 | end 78 | 79 | raise Errno::ENOENT, "Unable to find memcached 1.4+ locally" 80 | end 81 | 82 | def memcached_persistent(port=21345, options={}) 83 | dc = start_and_flush_with_retry(port, '', options) 84 | yield dc, port if block_given? 85 | end 86 | 87 | def sasl_credentials 88 | { :username => 'testuser', :password => 'testtest' } 89 | end 90 | 91 | def sasl_env 92 | { 93 | 'MEMCACHED_SASL_PWDB' => "#{File.dirname(__FILE__)}/sasl/sasldb", 94 | 'SASL_CONF_PATH' => "#{File.dirname(__FILE__)}/sasl/memcached.conf" 95 | } 96 | end 97 | 98 | def memcached_sasl_persistent(port=21397) 99 | dc = start_and_flush_with_retry(port, '-S', sasl_credentials) 100 | yield dc, port if block_given? 101 | end 102 | 103 | def memcached_cas_persistent(port = 25662) 104 | require 'dalli/cas/client' 105 | dc = start_and_flush_with_retry(port) 106 | yield dc, port if block_given? 107 | end 108 | 109 | 110 | def memcached_low_mem_persistent(port = 19128) 111 | dc = start_and_flush_with_retry(port, '-m 1 -M') 112 | yield dc, port if block_given? 113 | end 114 | 115 | def start_and_flush_with_retry(port, args = '', client_options = {}) 116 | dc = nil 117 | retry_count = 0 118 | while dc.nil? do 119 | begin 120 | dc = start_and_flush(port, args, client_options, (retry_count == 0)) 121 | rescue StandardError => e 122 | $started[port] = nil 123 | retry_count += 1 124 | raise e if retry_count >= 3 125 | end 126 | end 127 | dc 128 | end 129 | 130 | def start_and_flush(port, args = '', client_options = {}, flush = true) 131 | memcached_server(port, args) 132 | if "#{port}" =~ /\A\// 133 | # unix socket 134 | dc = Dalli::Client.new(port, client_options) 135 | else 136 | dc = Dalli::Client.new(["localhost:#{port}", "127.0.0.1:#{port}"], client_options) 137 | end 138 | dc.flush_all if flush 139 | dc 140 | end 141 | 142 | def memcached(port, args='', client_options={}) 143 | dc = start_and_flush_with_retry(port, args, client_options) 144 | yield dc, port if block_given? 145 | memcached_kill(port) 146 | end 147 | 148 | def memcached_server(port, args='') 149 | Memcached.path ||= find_memcached 150 | if "#{port}" =~ /\A\// 151 | # unix socket 152 | port_socket_arg = '-s' 153 | begin 154 | File.delete(port) 155 | rescue Errno::ENOENT 156 | end 157 | else 158 | port_socket_arg = '-p' 159 | port = port.to_i 160 | end 161 | 162 | cmd = "#{Memcached.path}memcached #{args} #{port_socket_arg} #{port}" 163 | 164 | $started[port] ||= begin 165 | pid = IO.popen(cmd).pid 166 | at_exit do 167 | begin 168 | Process.kill("TERM", pid) 169 | Process.wait(pid) 170 | rescue Errno::ECHILD, Errno::ESRCH 171 | end 172 | end 173 | wait_time = (args && args =~ /\-S/) ? 0.1 : 0.1 174 | sleep wait_time 175 | pid 176 | end 177 | end 178 | 179 | def supports_fork? 180 | Process.respond_to?(:fork) 181 | end 182 | 183 | def memcached_kill(port) 184 | pid = $started.delete(port) 185 | if pid 186 | begin 187 | Process.kill("TERM", pid) 188 | Process.wait(pid) 189 | rescue Errno::ECHILD, Errno::ESRCH => e 190 | puts e.inspect 191 | end 192 | end 193 | end 194 | 195 | end 196 | end 197 | 198 | module Memcached 199 | class << self 200 | attr_accessor :path 201 | end 202 | end 203 | -------------------------------------------------------------------------------- /lib/rack/session/dalli.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'rack/session/abstract/id' 3 | require 'dalli' 4 | 5 | module Rack 6 | module Session 7 | class Dalli < defined?(Abstract::Persisted) ? Abstract::Persisted : Abstract::ID 8 | attr_reader :pool, :mutex 9 | 10 | DEFAULT_DALLI_OPTIONS = { 11 | :namespace => 'rack:session', 12 | :memcache_server => 'localhost:11211' 13 | } 14 | 15 | # Brings in a new Rack::Session::Dalli middleware with the given 16 | # `:memcache_server`. The server is either a hostname, or a 17 | # host-with-port string in the form of "host_name:port", or an array of 18 | # such strings. For example: 19 | # 20 | # use Rack::Session::Dalli, 21 | # :memcache_server => "mc.example.com:1234" 22 | # 23 | # If no `:memcache_server` option is specified, Rack::Session::Dalli will 24 | # connect to localhost, port 11211 (the default memcached port). If 25 | # `:memcache_server` is set to nil, Dalli::Client will look for 26 | # ENV['MEMCACHE_SERVERS'] and use that value if it is available, or fall 27 | # back to the same default behavior described above. 28 | # 29 | # Rack::Session::Dalli is intended to be a drop-in replacement for 30 | # Rack::Session::Memcache. It accepts additional options that control the 31 | # behavior of Rack::Session, Dalli::Client, and an optional 32 | # ConnectionPool. First and foremost, if you wish to instantiate your own 33 | # Dalli::Client (or ConnectionPool) and use that instead of letting 34 | # Rack::Session::Dalli instantiate it on your behalf, simply pass it in 35 | # as the `:cache` option. Please note that you will be responsible for 36 | # setting the namespace and any other options on Dalli::Client. 37 | # 38 | # Secondly, if you're not using the `:cache` option, Rack::Session::Dalli 39 | # accepts the same options as Dalli::Client, so it's worth reviewing its 40 | # documentation. Perhaps most importantly, if you don't specify a 41 | # `:namespace` option, Rack::Session::Dalli will default to using 42 | # "rack:session". 43 | # 44 | # Whether you are using the `:cache` option or not, it is not recommend 45 | # to set `:expires_in`. Instead, use `:expire_after`, which will control 46 | # both the expiration of the client cookie as well as the expiration of 47 | # the corresponding entry in memcached. 48 | # 49 | # Rack::Session::Dalli also accepts a host of options that control how 50 | # the sessions and session cookies are managed, including the 51 | # aforementioned `:expire_after` option. Please see the documentation for 52 | # Rack::Session::Abstract::Persisted for a detailed explanation of these 53 | # options and their default values. 54 | # 55 | # Finally, if your web application is multithreaded, the 56 | # Rack::Session::Dalli middleware can become a source of contention. You 57 | # can use a connection pool of Dalli clients by passing in the 58 | # `:pool_size` and/or `:pool_timeout` options. For example: 59 | # 60 | # use Rack::Session::Dalli, 61 | # :memcache_server => "mc.example.com:1234", 62 | # :pool_size => 10 63 | # 64 | # You must include the `connection_pool` gem in your project if you wish 65 | # to use pool support. Please see the documentation for ConnectionPool 66 | # for more information about it and its default options (which would only 67 | # be applicable if you supplied one of the two options, but not both). 68 | # 69 | def initialize(app, options={}) 70 | # Parent uses DEFAULT_OPTIONS to build @default_options for Rack::Session 71 | super 72 | 73 | # Determine the default TTL for newly-created sessions 74 | @default_ttl = ttl @default_options[:expire_after] 75 | 76 | # Normalize and validate passed options 77 | cache, mserv, mopts, popts = extract_dalli_options options 78 | 79 | @pool = 80 | if cache # caller passed a Dalli::Client or ConnectionPool instance 81 | cache 82 | elsif popts # caller passed ConnectionPool options 83 | ConnectionPool.new(popts) { ::Dalli::Client.new(mserv, mopts) } 84 | else 85 | ::Dalli::Client.new(mserv, mopts) 86 | end 87 | 88 | if @pool.respond_to?(:alive!) # is a Dalli::Client 89 | @mutex = Mutex.new 90 | 91 | @pool.alive! 92 | end 93 | end 94 | 95 | def get_session(env, sid) 96 | with_block(env, [nil, {}]) do |dc| 97 | unless sid and !sid.empty? and session = dc.get(sid) 98 | old_sid, sid, session = sid, generate_sid_with(dc), {} 99 | unless dc.add(sid, session, @default_ttl) 100 | sid = old_sid 101 | redo # generate a new sid and try again 102 | end 103 | end 104 | [sid, session] 105 | end 106 | end 107 | 108 | def set_session(env, session_id, new_session, options) 109 | return false unless session_id 110 | 111 | with_block(env, false) do |dc| 112 | dc.set(session_id, new_session, ttl(options[:expire_after])) 113 | session_id 114 | end 115 | end 116 | 117 | def destroy_session(env, session_id, options) 118 | with_block(env) do |dc| 119 | dc.delete(session_id) 120 | generate_sid_with(dc) unless options[:drop] 121 | end 122 | end 123 | 124 | if defined?(Abstract::Persisted) 125 | def find_session(req, sid) 126 | get_session req.env, sid 127 | end 128 | 129 | def write_session(req, sid, session, options) 130 | set_session req.env, sid, session, options 131 | end 132 | 133 | def delete_session(req, sid, options) 134 | destroy_session req.env, sid, options 135 | end 136 | end 137 | 138 | private 139 | 140 | def extract_dalli_options(options) 141 | return [options[:cache]] if options[:cache] 142 | 143 | # Filter out Rack::Session-specific options and apply our defaults 144 | mopts = DEFAULT_DALLI_OPTIONS.merge \ 145 | options.reject {|k, _| DEFAULT_OPTIONS.key? k } 146 | mserv = mopts.delete :memcache_server 147 | 148 | if mopts[:pool_size] || mopts[:pool_timeout] 149 | popts = {} 150 | popts[:size] = mopts.delete :pool_size if mopts[:pool_size] 151 | popts[:timeout] = mopts.delete :pool_timeout if mopts[:pool_timeout] 152 | 153 | # For a connection pool, locking is handled at the pool level 154 | mopts[:threadsafe] = false unless mopts.key? :threadsafe 155 | end 156 | 157 | [nil, mserv, mopts, popts] 158 | end 159 | 160 | def generate_sid_with(dc) 161 | while true 162 | sid = generate_sid 163 | break sid unless dc.get(sid) 164 | end 165 | end 166 | 167 | def with_block(env, default=nil, &block) 168 | @mutex.lock if @mutex and env['rack.multithread'] 169 | @pool.with(&block) 170 | rescue ::Dalli::DalliError, Errno::ECONNREFUSED 171 | raise if $!.message =~ /undefined class/ 172 | if $VERBOSE 173 | warn "#{self} is unable to find memcached server." 174 | warn $!.inspect 175 | end 176 | default 177 | ensure 178 | @mutex.unlock if @mutex and @mutex.locked? 179 | end 180 | 181 | def ttl(expire_after) 182 | expire_after.nil? ? 0 : expire_after + 1 183 | end 184 | end 185 | end 186 | end 187 | -------------------------------------------------------------------------------- /test/test_server.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | 4 | describe Dalli::Server do 5 | describe 'hostname parsing' do 6 | it 'handles unix socket with no weight' do 7 | s = Dalli::Server.new('/var/run/memcached/sock') 8 | assert_equal '/var/run/memcached/sock', s.hostname 9 | assert_equal 1, s.weight 10 | assert_equal :unix, s.socket_type 11 | end 12 | 13 | it 'handles unix socket with a weight' do 14 | s = Dalli::Server.new('/var/run/memcached/sock:2') 15 | assert_equal '/var/run/memcached/sock', s.hostname 16 | assert_equal 2, s.weight 17 | assert_equal :unix, s.socket_type 18 | end 19 | 20 | it 'handles no port or weight' do 21 | s = Dalli::Server.new('localhost') 22 | assert_equal 'localhost', s.hostname 23 | assert_equal 11211, s.port 24 | assert_equal 1, s.weight 25 | assert_equal :tcp, s.socket_type 26 | end 27 | 28 | it 'handles a port, but no weight' do 29 | s = Dalli::Server.new('localhost:11212') 30 | assert_equal 'localhost', s.hostname 31 | assert_equal 11212, s.port 32 | assert_equal 1, s.weight 33 | assert_equal :tcp, s.socket_type 34 | end 35 | 36 | it 'handles a port and a weight' do 37 | s = Dalli::Server.new('localhost:11212:2') 38 | assert_equal 'localhost', s.hostname 39 | assert_equal 11212, s.port 40 | assert_equal 2, s.weight 41 | assert_equal :tcp, s.socket_type 42 | end 43 | 44 | it 'handles ipv4 addresses' do 45 | s = Dalli::Server.new('127.0.0.1') 46 | assert_equal '127.0.0.1', s.hostname 47 | assert_equal 11211, s.port 48 | assert_equal 1, s.weight 49 | assert_equal :tcp, s.socket_type 50 | end 51 | 52 | it 'handles ipv6 addresses' do 53 | s = Dalli::Server.new('[::1]') 54 | assert_equal '::1', s.hostname 55 | assert_equal 11211, s.port 56 | assert_equal 1, s.weight 57 | assert_equal :tcp, s.socket_type 58 | end 59 | 60 | it 'handles ipv6 addresses with port' do 61 | s = Dalli::Server.new('[::1]:11212') 62 | assert_equal '::1', s.hostname 63 | assert_equal 11212, s.port 64 | assert_equal 1, s.weight 65 | assert_equal :tcp, s.socket_type 66 | end 67 | 68 | it 'handles ipv6 addresses with port and weight' do 69 | s = Dalli::Server.new('[::1]:11212:2') 70 | assert_equal '::1', s.hostname 71 | assert_equal 11212, s.port 72 | assert_equal 2, s.weight 73 | assert_equal :tcp, s.socket_type 74 | end 75 | 76 | it 'handles a FQDN' do 77 | s = Dalli::Server.new('my.fqdn.com') 78 | assert_equal 'my.fqdn.com', s.hostname 79 | assert_equal 11211, s.port 80 | assert_equal 1, s.weight 81 | assert_equal :tcp, s.socket_type 82 | end 83 | 84 | it 'handles a FQDN with port and weight' do 85 | s = Dalli::Server.new('my.fqdn.com:11212:2') 86 | assert_equal 'my.fqdn.com', s.hostname 87 | assert_equal 11212, s.port 88 | assert_equal 2, s.weight 89 | assert_equal :tcp, s.socket_type 90 | end 91 | 92 | it 'throws an exception if the hostname cannot be parsed' do 93 | lambda { Dalli::Server.new('[]') }.must_raise Dalli::DalliError 94 | lambda { Dalli::Server.new('my.fqdn.com:') }.must_raise Dalli::DalliError 95 | lambda { Dalli::Server.new('my.fqdn.com:11212,:2') }.must_raise Dalli::DalliError 96 | lambda { Dalli::Server.new('my.fqdn.com:11212:abc') }.must_raise Dalli::DalliError 97 | end 98 | end 99 | 100 | describe 'ttl translation' do 101 | it 'does not translate ttls under 30 days' do 102 | s = Dalli::Server.new('localhost') 103 | assert_equal s.send(:sanitize_ttl, 30*24*60*60), 30*24*60*60 104 | end 105 | 106 | it 'translates ttls over 30 days into timestamps' do 107 | s = Dalli::Server.new('localhost') 108 | assert_equal s.send(:sanitize_ttl, 30*24*60*60 + 1), Time.now.to_i + 30*24*60*60+1 109 | end 110 | 111 | it 'does not translate ttls which are already timestamps' do 112 | s = Dalli::Server.new('localhost') 113 | timestamp_ttl = Time.now.to_i + 60 114 | assert_equal s.send(:sanitize_ttl, timestamp_ttl), timestamp_ttl 115 | end 116 | end 117 | 118 | describe 'guard_max_value' do 119 | it 'yields when size is under max' do 120 | s = Dalli::Server.new('127.0.0.1') 121 | value = OpenStruct.new(:bytesize => 1_048_576) 122 | 123 | yielded = false 124 | s.send(:guard_max_value, :foo, value) do 125 | yielded = true 126 | end 127 | 128 | assert_equal yielded, true 129 | end 130 | 131 | it 'warns when size is over max' do 132 | s = Dalli::Server.new('127.0.0.1') 133 | value = OpenStruct.new(:bytesize => 1_048_577) 134 | 135 | Dalli.logger.expects(:error).once.with("Value for foo over max size: 1048576 <= 1048577 - this value may be truncated by memcached") 136 | 137 | s.send(:guard_max_value, :foo, value) 138 | end 139 | 140 | it 'throws when size is over max and error_over_max_size true' do 141 | s = Dalli::Server.new('127.0.0.1', :error_when_over_max_size => true) 142 | value = OpenStruct.new(:bytesize => 1_048_577) 143 | 144 | lambda do 145 | s.send(:guard_max_value, :foo, value) 146 | end.must_raise Dalli::ValueOverMaxSize 147 | end 148 | end 149 | 150 | describe 'deserialize' do 151 | subject { Dalli::Server.new('127.0.0.1') } 152 | 153 | it 'uses Marshal as default serializer' do 154 | assert_equal subject.serializer, Marshal 155 | end 156 | 157 | it 'deserializes serialized value' do 158 | value = 'some_value' 159 | deserialized = subject.send(:deserialize, Marshal.dump(value), Dalli::Server::FLAG_SERIALIZED) 160 | assert_equal value, deserialized 161 | end 162 | 163 | it 'raises UnmarshalError for broken data' do 164 | assert_raises Dalli::UnmarshalError do 165 | subject.send(:deserialize, :not_marshaled_value, Dalli::Server::FLAG_SERIALIZED) 166 | end 167 | end 168 | 169 | describe 'custom serializer' do 170 | let(:serializer) { Minitest::Mock.new } 171 | subject { Dalli::Server.new('127.0.0.1', serializer: serializer) } 172 | 173 | it 'uses custom serializer' do 174 | assert subject.serializer === serializer 175 | end 176 | 177 | it 'reraises general NameError' do 178 | serializer.expect(:load, nil) do 179 | raise NameError, 'ddd' 180 | end 181 | assert_raises NameError do 182 | subject.send(:deserialize, :some_value, Dalli::Server::FLAG_SERIALIZED) 183 | end 184 | end 185 | 186 | it 'raises UnmarshalError on uninitialized constant' do 187 | serializer.expect(:load, nil) do 188 | raise NameError, 'uninitialized constant Ddd' 189 | end 190 | assert_raises Dalli::UnmarshalError do 191 | subject.send(:deserialize, :some_value, Dalli::Server::FLAG_SERIALIZED) 192 | end 193 | end 194 | 195 | it 'reraises general ArgumentError' do 196 | serializer.expect(:load, nil) do 197 | raise ArgumentError, 'ddd' 198 | end 199 | assert_raises ArgumentError do 200 | subject.send(:deserialize, :some_value, Dalli::Server::FLAG_SERIALIZED) 201 | end 202 | end 203 | 204 | it 'raises UnmarshalError on undefined class' do 205 | serializer.expect(:load, nil) do 206 | raise ArgumentError, 'undefined class Ddd' 207 | end 208 | assert_raises Dalli::UnmarshalError do 209 | subject.send(:deserialize, :some_value, Dalli::Server::FLAG_SERIALIZED) 210 | end 211 | end 212 | end 213 | end 214 | end 215 | -------------------------------------------------------------------------------- /test/benchmark_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | require 'benchmark' 4 | require 'active_support/cache/dalli_store' 5 | 6 | describe 'performance' do 7 | before do 8 | puts "Testing #{Dalli::VERSION} with #{RUBY_DESCRIPTION}" 9 | # We'll use a simple @value to try to avoid spending time in Marshal, 10 | # which is a constant penalty that both clients have to pay 11 | @value = [] 12 | @marshalled = Marshal.dump(@value) 13 | 14 | @port = 23417 15 | @servers = ["127.0.0.1:#{@port}", "localhost:#{@port}"] 16 | @key1 = "Short" 17 | @key2 = "Sym1-2-3::45"*8 18 | @key3 = "Long"*40 19 | @key4 = "Medium"*8 20 | # 5 and 6 are only used for multiget miss test 21 | @key5 = "Medium2"*8 22 | @key6 = "Long3"*40 23 | @counter = 'counter' 24 | end 25 | 26 | it 'runs benchmarks' do 27 | memcached(@port) do 28 | 29 | Benchmark.bm(37) do |x| 30 | 31 | n = 2500 32 | 33 | @ds = ActiveSupport::Cache::DalliStore.new(@servers) 34 | x.report("mixed:rails:dalli") do 35 | n.times do 36 | @ds.read @key1 37 | @ds.write @key2, @value 38 | @ds.fetch(@key3) { @value } 39 | @ds.fetch(@key2) { @value } 40 | @ds.fetch(@key1) { @value } 41 | @ds.write @key2, @value, :unless_exists => true 42 | @ds.delete @key2 43 | @ds.increment @counter, 1, :initial => 100 44 | @ds.increment @counter, 1, :expires_in => 12 45 | @ds.decrement @counter, 1 46 | end 47 | end 48 | 49 | x.report("mixed:rails-localcache:dalli") do 50 | n.times do 51 | @ds.with_local_cache do 52 | @ds.read @key1 53 | @ds.write @key2, @value 54 | @ds.fetch(@key3) { @value } 55 | @ds.fetch(@key2) { @value } 56 | @ds.fetch(@key1) { @value } 57 | @ds.write @key2, @value, :unless_exists => true 58 | @ds.delete @key2 59 | @ds.increment @counter, 1, :initial => 100 60 | @ds.increment @counter, 1, :expires_in => 12 61 | @ds.decrement @counter, 1 62 | end 63 | end 64 | end 65 | 66 | @ds.clear 67 | sizeable_data = "some view partial data" * 50 68 | [@key1, @key2, @key3, @key4, @key5, @key6].each do |key| 69 | @ds.write(key, sizeable_data) 70 | end 71 | 72 | x.report("read_multi_big:rails:dalli") do 73 | n.times do 74 | @ds.read_multi @key1, @key2, @key3, @key4 75 | @ds.read @key1 76 | @ds.read @key2 77 | @ds.read @key3 78 | @ds.read @key4 79 | @ds.read @key1 80 | @ds.read @key2 81 | @ds.read @key3 82 | @ds.read_multi @key1, @key2, @key3 83 | end 84 | end 85 | 86 | x.report("read_multi_big:rails-localcache:dalli") do 87 | n.times do 88 | @ds.with_local_cache do 89 | @ds.read_multi @key1, @key2, @key3, @key4 90 | @ds.read @key1 91 | @ds.read @key2 92 | @ds.read @key3 93 | @ds.read @key4 94 | end 95 | @ds.with_local_cache do 96 | @ds.read @key1 97 | @ds.read @key2 98 | @ds.read @key3 99 | @ds.read_multi @key1, @key2, @key3 100 | end 101 | end 102 | end 103 | 104 | @m = Dalli::Client.new(@servers) 105 | x.report("set:plain:dalli") do 106 | n.times do 107 | @m.set @key1, @marshalled, 0, :raw => true 108 | @m.set @key2, @marshalled, 0, :raw => true 109 | @m.set @key3, @marshalled, 0, :raw => true 110 | @m.set @key1, @marshalled, 0, :raw => true 111 | @m.set @key2, @marshalled, 0, :raw => true 112 | @m.set @key3, @marshalled, 0, :raw => true 113 | end 114 | end 115 | 116 | @m = Dalli::Client.new(@servers) 117 | x.report("setq:plain:dalli") do 118 | @m.multi do 119 | n.times do 120 | @m.set @key1, @marshalled, 0, :raw => true 121 | @m.set @key2, @marshalled, 0, :raw => true 122 | @m.set @key3, @marshalled, 0, :raw => true 123 | @m.set @key1, @marshalled, 0, :raw => true 124 | @m.set @key2, @marshalled, 0, :raw => true 125 | @m.set @key3, @marshalled, 0, :raw => true 126 | end 127 | end 128 | end 129 | 130 | @m = Dalli::Client.new(@servers) 131 | x.report("set:ruby:dalli") do 132 | n.times do 133 | @m.set @key1, @value 134 | @m.set @key2, @value 135 | @m.set @key3, @value 136 | @m.set @key1, @value 137 | @m.set @key2, @value 138 | @m.set @key3, @value 139 | end 140 | end 141 | 142 | @m = Dalli::Client.new(@servers) 143 | x.report("get:plain:dalli") do 144 | n.times do 145 | @m.get @key1, :raw => true 146 | @m.get @key2, :raw => true 147 | @m.get @key3, :raw => true 148 | @m.get @key1, :raw => true 149 | @m.get @key2, :raw => true 150 | @m.get @key3, :raw => true 151 | end 152 | end 153 | 154 | @m = Dalli::Client.new(@servers) 155 | x.report("get:ruby:dalli") do 156 | n.times do 157 | @m.get @key1 158 | @m.get @key2 159 | @m.get @key3 160 | @m.get @key1 161 | @m.get @key2 162 | @m.get @key3 163 | end 164 | end 165 | 166 | @m = Dalli::Client.new(@servers) 167 | x.report("multiget:ruby:dalli") do 168 | n.times do 169 | # We don't use the keys array because splat is slow 170 | @m.get_multi @key1, @key2, @key3, @key4, @key5, @key6 171 | end 172 | end 173 | 174 | @m = Dalli::Client.new(@servers) 175 | x.report("missing:ruby:dalli") do 176 | n.times do 177 | begin @m.delete @key1; rescue; end 178 | begin @m.get @key1; rescue; end 179 | begin @m.delete @key2; rescue; end 180 | begin @m.get @key2; rescue; end 181 | begin @m.delete @key3; rescue; end 182 | begin @m.get @key3; rescue; end 183 | end 184 | end 185 | 186 | @m = Dalli::Client.new(@servers) 187 | x.report("mixed:ruby:dalli") do 188 | n.times do 189 | @m.set @key1, @value 190 | @m.set @key2, @value 191 | @m.set @key3, @value 192 | @m.get @key1 193 | @m.get @key2 194 | @m.get @key3 195 | @m.set @key1, @value 196 | @m.get @key1 197 | @m.set @key2, @value 198 | @m.get @key2 199 | @m.set @key3, @value 200 | @m.get @key3 201 | end 202 | end 203 | 204 | @m = Dalli::Client.new(@servers) 205 | x.report("mixedq:ruby:dalli") do 206 | @m.multi do 207 | n.times do 208 | @m.set @key1, @value 209 | @m.set @key2, @value 210 | @m.set @key3, @value 211 | @m.get @key1 212 | @m.get @key2 213 | @m.get @key3 214 | @m.set @key1, @value 215 | @m.get @key1 216 | @m.set @key2, @value 217 | @m.replace @key2, @value 218 | @m.delete @key3 219 | @m.add @key3, @value 220 | @m.get @key2 221 | @m.set @key3, @value 222 | @m.get @key3 223 | end 224 | end 225 | end 226 | 227 | @m = Dalli::Client.new(@servers) 228 | x.report("incr:ruby:dalli") do 229 | counter = 'foocount' 230 | n.times do 231 | @m.incr counter, 1, 0, 1 232 | end 233 | n.times do 234 | @m.decr counter, 1 235 | end 236 | 237 | assert_equal 0, @m.incr(counter, 0) 238 | end 239 | 240 | end 241 | end 242 | 243 | end 244 | end 245 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Dalli [![Build Status](https://secure.travis-ci.org/petergoldstein/dalli.svg)](http://travis-ci.org/petergoldstein/dalli) [![Dependency Status](https://gemnasium.com/petergoldstein/dalli.svg)](https://gemnasium.com/petergoldstein/dalli) [![Code Climate](https://codeclimate.com/github/petergoldstein/dalli.svg)](https://codeclimate.com/github/petergoldstein/dalli) 2 | ===== 3 | 4 | Dalli is a high performance pure Ruby client for accessing memcached servers. It works with memcached 1.4+ only as it uses the newer binary protocol. It should be considered a replacement for the memcache-client gem. 5 | 6 | The name is a variant of Salvador Dali for his famous painting [The Persistence of Memory](http://en.wikipedia.org/wiki/The_Persistence_of_Memory). 7 | 8 | ![Persistence of Memory](http://www.virtualdali.com/assets/paintings/31PersistenceOfMemory.jpg) 9 | 10 | Dalli's initial development was sponsored by [CouchBase](http://www.couchbase.com/). Many thanks to them! 11 | 12 | 13 | Design 14 | ------------ 15 | 16 | Mike Perham decided to write Dalli after maintaining memcache-client for two years for a few specific reasons: 17 | 18 | 0. The code is mostly old and gross. The bulk of the code is a single 1000 line .rb file. 19 | 1. It has a lot of options that are infrequently used which complicate the codebase. 20 | 2. The implementation has no single point to attach monitoring hooks. 21 | 3. Uses the old text protocol, which hurts raw performance. 22 | 23 | So a few notes. Dalli: 24 | 25 | 0. uses the exact same algorithm to choose a server so existing memcached clusters with TBs of data will work identically to memcache-client. 26 | 1. is approximately 20% faster than memcache-client (which itself was heavily optimized) in Ruby 1.9.2. 27 | 2. contains explicit "chokepoint" methods which handle all requests; these can be hooked into by monitoring tools (NewRelic, Rack::Bug, etc) to track memcached usage. 28 | 3. supports SASL for use in managed environments, e.g. Heroku. 29 | 4. provides proper failover with recovery and adjustable timeouts 30 | 31 | 32 | Supported Ruby versions and implementations 33 | ------------------------------------------------ 34 | 35 | Dalli should work identically on: 36 | 37 | * JRuby 1.6+ 38 | * Ruby 1.9.3+ 39 | * Rubinius 2.0 40 | 41 | If you have problems, please enter an issue. 42 | 43 | 44 | Installation and Usage 45 | ------------------------ 46 | 47 | Remember, Dalli **requires** memcached 1.4+. You can check the version with `memcached -h`. Please note that the memcached version that *Mac OS X Snow Leopard* ships with is 1.2.8 and it won't work. Install memcached 1.4.x using Homebrew with 48 | 49 | brew install memcached 50 | 51 | On Ubuntu you can install it by running: 52 | 53 | apt-get install memcached 54 | 55 | You can verify your installation using this piece of code: 56 | 57 | ```bash 58 | gem install dalli 59 | ``` 60 | 61 | ```ruby 62 | require 'dalli' 63 | options = { :namespace => "app_v1", :compress => true } 64 | dc = Dalli::Client.new('localhost:11211', options) 65 | dc.set('abc', 123) 66 | value = dc.get('abc') 67 | ``` 68 | 69 | The test suite requires memcached 1.4.3+ with SASL enabled (`brew install memcached --enable-sasl ; mv /usr/bin/memcached /usr/bin/memcached.old`). Currently only supports the PLAIN mechanism. 70 | 71 | Dalli has no runtime dependencies and never will. If you are using Ruby <2.3, 72 | you can optionally install the '[kgio](https://bogomips.org/kgio/)' gem to 73 | give Dalli a 20-30% performance boost. 74 | 75 | 76 | Usage with Rails 3.x and 4.x 77 | --------------------------- 78 | 79 | In your Gemfile: 80 | 81 | ```ruby 82 | gem 'dalli' 83 | ``` 84 | 85 | In `config/environments/production.rb`: 86 | 87 | ```ruby 88 | config.cache_store = :dalli_store 89 | ``` 90 | 91 | Here's a more comprehensive example that sets a reasonable default for maximum cache entry lifetime (one day), enables compression for large values and namespaces all entries for this rails app. Remove the namespace if you have multiple apps which share cached values. 92 | 93 | ```ruby 94 | config.cache_store = :dalli_store, 'cache-1.example.com', 'cache-2.example.com:11211:2', 95 | { :namespace => NAME_OF_RAILS_APP, :expires_in => 1.day, :compress => true } 96 | ``` 97 | 98 | You can specify a port and a weight by appending to the server name. You may wish to increase the weight of a server with more memory configured. (e.g. to specify port 11211 and a weight of 2, append `:11211:2` ) 99 | 100 | If your servers are specified in `ENV["MEMCACHE_SERVERS"]` (e.g. on Heroku when using a third-party hosted addon), simply provide `nil` for the servers: 101 | 102 | ```ruby 103 | config.cache_store = :dalli_store, nil, { :namespace => NAME_OF_RAILS_APP, :expires_in => 1.day, :compress => true } 104 | ``` 105 | 106 | To use Dalli for Rails session storage that times out after 20 minutes, in `config/initializers/session_store.rb`: 107 | 108 | For Rails >= 3.2.4: 109 | 110 | ```ruby 111 | Rails.application.config.session_store ActionDispatch::Session::CacheStore, :expire_after => 20.minutes 112 | ``` 113 | 114 | For Rails 3.x: 115 | 116 | ```ruby 117 | require 'action_dispatch/middleware/session/dalli_store' 118 | Rails.application.config.session_store :dalli_store, :memcache_server => ['host1', 'host2'], :namespace => 'sessions', :key => '_foundation_session', :expire_after => 20.minutes 119 | ``` 120 | 121 | Dalli does not support Rails 2.x. 122 | 123 | 124 | Multithreading and Rails 125 | -------------------------- 126 | 127 | If you use Puma or another threaded app server, as of Dalli 2.7, you can use a pool 128 | of Dalli clients with Rails to ensure the `Rails.cache` singleton does not become a 129 | source of thread contention. You must add `gem 'connection_pool'` to your Gemfile and 130 | add :pool\_size to your `dalli_store` config: 131 | 132 | ```ruby 133 | config.cache_store = :dalli_store, 'cache-1.example.com', { :pool_size => 5 } 134 | ``` 135 | 136 | You can then use the Rails cache as normal and Rails.cache will use the pool transparently under the covers, or you can check out a Dalli client directly from the pool: 137 | 138 | ```ruby 139 | Rails.cache.fetch('foo', :expires_in => 300) do 140 | 'bar' 141 | end 142 | 143 | Rails.cache.dalli.with do |client| 144 | # client is a Dalli::Client instance which you can 145 | # use ONLY within this block 146 | end 147 | ``` 148 | 149 | 150 | Configuration 151 | ------------------------ 152 | 153 | **servers**: An Array of "host:port:weight" where weight allows you to distribute cache unevenly. 154 | 155 | Dalli::Client accepts the following options. All times are in seconds. 156 | 157 | **expires_in**: Global default for key TTL. Default is 0, which means no expiry. 158 | 159 | **namespace**: If specified, prepends each key with this value to provide simple namespacing. Default is nil. 160 | 161 | **failover**: Boolean, if true Dalli will failover to another server if the main server for a key is down. Default is true. 162 | 163 | **threadsafe**: Boolean. If true Dalli ensures that only one thread is using a socket at a given time. Default is true. Set to false at your own peril. 164 | 165 | **serializer**: The serializer to use for objects being stored (ex. JSON). 166 | Default is Marshal. 167 | 168 | **compress**: Boolean, if true Dalli will gzip-compress values larger than 1K. Default is false. 169 | 170 | **compression_min_size**: Minimum value byte size for which to attempt compression. Default is 1K. 171 | 172 | **compression_max_size**: Maximum value byte size for which to attempt compression. Default is unlimited. 173 | 174 | **compressor**: The compressor to use for objects being stored. 175 | Default is zlib, implemented under `Dalli::Compressor`. 176 | If serving compressed data using nginx's HttpMemcachedModule, set `memcached_gzip_flag 2` and use `Dalli::GzipCompressor` 177 | 178 | **keepalive**: Boolean. If true, Dalli will enable keep-alive for socket connections. Default is true. 179 | 180 | **socket_timeout**: Timeout for all socket operations (connect, read, write). Default is 0.5. 181 | 182 | **socket_max_failures**: When a socket operation fails after socket_timeout, the same operation is retried. This is to not immediately mark a server down when there's a very slight network problem. Default is 2. 183 | 184 | **socket_failure_delay**: Before retrying a socket operation, the process sleeps for this amount of time. Default is 0.01. Set to nil for no delay. 185 | 186 | **down_retry_delay**: When a server has been marked down due to many failures, the server will be checked again for being alive only after this amount of time. Don't set this value too low, otherwise each request which tries the failed server might hang for the maximum **socket_timeout**. Default is 60 seconds. 187 | 188 | **value_max_bytes**: The maximum size of a value in memcached. Defaults to 1MB, this can be increased with memcached's -I parameter. You must also configure Dalli to allow the larger size here. 189 | 190 | **error_when_over_max_size**: Boolean. If true, Dalli will throw a Dalli::ValueOverMaxSize exception when trying to store data larger than **value_max_bytes**. Defaults to false, meaning only a warning is logged. 191 | 192 | **username**: The username to use for authenticating this client instance against a SASL-enabled memcached server. Heroku users should not need to use this normally. 193 | 194 | **password**: The password to use for authenticating this client instance against a SASL-enabled memcached server. Heroku users should not need to use this normally. 195 | 196 | **sndbuf**: In bytes, set the socket SO_SNDBUF. Defaults to operating system default. 197 | 198 | **rcvbuf**: In bytes, set the socket SO_RCVBUF. Defaults to operating system default. 199 | 200 | **cache_nils**: Boolean. If true Dalli will not treat cached `nil` values as 'not found' for `#fetch` operations. Default is false. 201 | 202 | **raise_errors**: Boolean. When true DalliStore will reraise Dalli:DalliError instead swallowing the error. Default is false. 203 | 204 | **instrument_errors**: Boolean. When true DalliStore will send notification of Dalli::DalliError via a 'cache_error.active_support' event. Default is false. 205 | 206 | Features and Changes 207 | ------------------------ 208 | 209 | By default, Dalli is thread-safe. Disable thread-safety at your own peril. 210 | 211 | Dalli does not need anything special in Unicorn/Passenger since 2.0.4. 212 | It will detect sockets shared with child processes and gracefully reopen the 213 | socket. 214 | 215 | Note that Dalli does not require ActiveSupport or Rails. You can safely use it in your own Ruby projects. 216 | 217 | [View the Client API](http://www.rubydoc.info/github/mperham/dalli/Dalli/Client) 218 | 219 | Helping Out 220 | ------------- 221 | 222 | If you have a fix you wish to provide, please fork the code, fix in your local project and then send a pull request on github. Please ensure that you include a test which verifies your fix and update `History.md` with a one sentence description of your fix so you get credit as a contributor. 223 | 224 | We're not accepting new compressors. They are trivial to add in an initializer. See #385 (LZ4), #406 (Snappy) 225 | 226 | Thanks 227 | ------------ 228 | 229 | Mike Perham - for originally authoring the Dalli project and serving as maintainer and primary contributor 230 | 231 | Eric Wong - for help using his [kgio](http://bogomips.org/kgio/) library. 232 | 233 | Brian Mitchell - for his remix-stash project which was helpful when implementing and testing the binary protocol support. 234 | 235 | [CouchBase](http://couchbase.com) - for their project sponsorship 236 | 237 | Authors 238 | ---------- 239 | 240 | * [Peter M. Goldstein](https://github.com/petergoldstein) - current maintainer 241 | * [Mike Perham](https://github.com/mperham) and contributors 242 | 243 | 244 | Copyright 245 | ----------- 246 | 247 | Copyright (c) Mike Perham, Peter M. Goldstein. See LICENSE for details. 248 | -------------------------------------------------------------------------------- /test/test_rack_session.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | 4 | require 'rack/session/dalli' 5 | require 'rack/lint' 6 | require 'rack/mock' 7 | require 'thread' 8 | 9 | describe Rack::Session::Dalli do 10 | 11 | before do 12 | @port = 19129 13 | memcached_persistent(@port) 14 | Rack::Session::Dalli::DEFAULT_DALLI_OPTIONS[:memcache_server] = "localhost:#{@port}" 15 | 16 | # test memcache connection 17 | Rack::Session::Dalli.new(incrementor) 18 | end 19 | 20 | let(:session_key) { Rack::Session::Dalli::DEFAULT_OPTIONS[:key] } 21 | let(:session_match) do 22 | /#{session_key}=([0-9a-fA-F]+);/ 23 | end 24 | let(:incrementor_proc) do 25 | lambda do |env| 26 | env["rack.session"]["counter"] ||= 0 27 | env["rack.session"]["counter"] += 1 28 | Rack::Response.new(env["rack.session"].inspect).to_a 29 | end 30 | end 31 | let(:drop_session) do 32 | Rack::Lint.new(proc do |env| 33 | env['rack.session.options'][:drop] = true 34 | incrementor_proc.call(env) 35 | end) 36 | end 37 | let(:renew_session) do 38 | Rack::Lint.new(proc do |env| 39 | env['rack.session.options'][:renew] = true 40 | incrementor_proc.call(env) 41 | end) 42 | end 43 | let(:defer_session) do 44 | Rack::Lint.new(proc do |env| 45 | env['rack.session.options'][:defer] = true 46 | incrementor_proc.call(env) 47 | end) 48 | end 49 | let(:skip_session) do 50 | Rack::Lint.new(proc do |env| 51 | env['rack.session.options'][:skip] = true 52 | incrementor_proc.call(env) 53 | end) 54 | end 55 | let(:incrementor) { Rack::Lint.new(incrementor_proc) } 56 | 57 | it "faults on no connection" do 58 | assert_raises Dalli::RingError do 59 | Rack::Session::Dalli.new(incrementor, :memcache_server => 'nosuchserver') 60 | end 61 | end 62 | 63 | it "connects to existing server" do 64 | assert_silent do 65 | rsd = Rack::Session::Dalli.new(incrementor, :namespace => 'test:rack:session') 66 | rsd.pool.set('ping', '') 67 | end 68 | end 69 | 70 | it "passes options to MemCache" do 71 | opts = { 72 | :namespace => 'test:rack:session', 73 | :compression_min_size => 1234 74 | } 75 | 76 | rsd = Rack::Session::Dalli.new(incrementor, opts) 77 | assert_equal(opts[:namespace], rsd.pool.instance_eval { @options[:namespace] }) 78 | assert_equal(opts[:compression_min_size], rsd.pool.instance_eval { @options[:compression_min_size] }) 79 | end 80 | 81 | it "accepts and prioritizes a :cache option" do 82 | server = Rack::Session::Dalli::DEFAULT_DALLI_OPTIONS[:memcache_server] 83 | cache = Dalli::Client.new(server, :namespace => 'test:rack:session') 84 | rsd = Rack::Session::Dalli.new(incrementor, :cache => cache, :namespace => 'foobar') 85 | assert_equal('test:rack:session', rsd.pool.instance_eval { @options[:namespace] }) 86 | end 87 | 88 | it "generates sids without an existing Dalli::Client" do 89 | rsd = Rack::Session::Dalli.new(incrementor) 90 | assert rsd.send :generate_sid 91 | end 92 | 93 | it "upgrades to a connection pool" do 94 | opts = { 95 | :namespace => 'test:rack:session', 96 | :pool_size => 10 97 | } 98 | 99 | with_connectionpool do 100 | rsd = Rack::Session::Dalli.new(incrementor, opts) 101 | assert rsd.pool.is_a? ConnectionPool 102 | rsd.pool.with do |mc| 103 | assert mc.instance_eval { !@options[:threadsafe] } 104 | assert_equal(opts[:namespace], mc.instance_eval { @options[:namespace] }) 105 | end 106 | end 107 | end 108 | 109 | it "creates a new cookie" do 110 | rsd = Rack::Session::Dalli.new(incrementor) 111 | res = Rack::MockRequest.new(rsd).get("/") 112 | assert res["Set-Cookie"].include?("#{session_key}=") 113 | assert_equal '{"counter"=>1}', res.body 114 | end 115 | 116 | it "determines session from a cookie" do 117 | rsd = Rack::Session::Dalli.new(incrementor) 118 | req = Rack::MockRequest.new(rsd) 119 | res = req.get("/") 120 | cookie = res["Set-Cookie"] 121 | assert_equal '{"counter"=>2}', req.get("/", "HTTP_COOKIE" => cookie).body 122 | assert_equal '{"counter"=>3}', req.get("/", "HTTP_COOKIE" => cookie).body 123 | end 124 | 125 | it "determines session only from a cookie by default" do 126 | rsd = Rack::Session::Dalli.new(incrementor) 127 | req = Rack::MockRequest.new(rsd) 128 | res = req.get("/") 129 | sid = res["Set-Cookie"][session_match, 1] 130 | assert_equal '{"counter"=>1}', req.get("/?rack.session=#{sid}").body 131 | assert_equal '{"counter"=>1}', req.get("/?rack.session=#{sid}").body 132 | end 133 | 134 | it "determines session from params" do 135 | rsd = Rack::Session::Dalli.new(incrementor, :cookie_only => false) 136 | req = Rack::MockRequest.new(rsd) 137 | res = req.get("/") 138 | sid = res["Set-Cookie"][session_match, 1] 139 | assert_equal '{"counter"=>2}', req.get("/?rack.session=#{sid}").body 140 | assert_equal '{"counter"=>3}', req.get("/?rack.session=#{sid}").body 141 | end 142 | 143 | it "survives nonexistant cookies" do 144 | bad_cookie = "rack.session=blarghfasel" 145 | rsd = Rack::Session::Dalli.new(incrementor) 146 | res = Rack::MockRequest.new(rsd). 147 | get("/", "HTTP_COOKIE" => bad_cookie) 148 | assert_equal '{"counter"=>1}', res.body 149 | cookie = res["Set-Cookie"][session_match] 150 | refute_match(/#{bad_cookie}/, cookie) 151 | end 152 | 153 | it "survives nonexistant blank cookies" do 154 | bad_cookie = "rack.session=" 155 | rsd = Rack::Session::Dalli.new(incrementor) 156 | res = Rack::MockRequest.new(rsd). 157 | get("/", "HTTP_COOKIE" => bad_cookie) 158 | cookie = res["Set-Cookie"][session_match] 159 | refute_match(/#{bad_cookie}$/, cookie) 160 | end 161 | 162 | it "sets an expiration on new sessions" do 163 | rsd = Rack::Session::Dalli.new(incrementor, :expire_after => 3) 164 | res = Rack::MockRequest.new(rsd).get('/') 165 | assert res.body.include?('"counter"=>1') 166 | cookie = res["Set-Cookie"] 167 | puts 'Sleeping to expire session' if $DEBUG 168 | sleep 4 169 | res = Rack::MockRequest.new(rsd).get('/', "HTTP_COOKIE" => cookie) 170 | refute_equal cookie, res["Set-Cookie"] 171 | assert res.body.include?('"counter"=>1') 172 | end 173 | 174 | it "maintains freshness of existing sessions" do 175 | rsd = Rack::Session::Dalli.new(incrementor, :expire_after => 3) 176 | res = Rack::MockRequest.new(rsd).get('/') 177 | assert res.body.include?('"counter"=>1') 178 | cookie = res["Set-Cookie"] 179 | res = Rack::MockRequest.new(rsd).get('/', "HTTP_COOKIE" => cookie) 180 | assert_equal cookie, res["Set-Cookie"] 181 | assert res.body.include?('"counter"=>2') 182 | puts 'Sleeping to expire session' if $DEBUG 183 | sleep 4 184 | res = Rack::MockRequest.new(rsd).get('/', "HTTP_COOKIE" => cookie) 185 | refute_equal cookie, res["Set-Cookie"] 186 | assert res.body.include?('"counter"=>1') 187 | end 188 | 189 | it "does not send the same session id if it did not change" do 190 | rsd = Rack::Session::Dalli.new(incrementor) 191 | req = Rack::MockRequest.new(rsd) 192 | 193 | res0 = req.get("/") 194 | cookie = res0["Set-Cookie"][session_match] 195 | assert_equal '{"counter"=>1}', res0.body 196 | 197 | res1 = req.get("/", "HTTP_COOKIE" => cookie) 198 | assert_nil res1["Set-Cookie"] 199 | assert_equal '{"counter"=>2}', res1.body 200 | 201 | res2 = req.get("/", "HTTP_COOKIE" => cookie) 202 | assert_nil res2["Set-Cookie"] 203 | assert_equal '{"counter"=>3}', res2.body 204 | end 205 | 206 | it "deletes cookies with :drop option" do 207 | rsd = Rack::Session::Dalli.new(incrementor) 208 | req = Rack::MockRequest.new(rsd) 209 | drop = Rack::Utils::Context.new(rsd, drop_session) 210 | dreq = Rack::MockRequest.new(drop) 211 | 212 | res1 = req.get("/") 213 | session = (cookie = res1["Set-Cookie"])[session_match] 214 | assert_equal '{"counter"=>1}', res1.body 215 | 216 | res2 = dreq.get("/", "HTTP_COOKIE" => cookie) 217 | assert_nil res2["Set-Cookie"] 218 | assert_equal '{"counter"=>2}', res2.body 219 | 220 | res3 = req.get("/", "HTTP_COOKIE" => cookie) 221 | refute_equal session, res3["Set-Cookie"][session_match] 222 | assert_equal '{"counter"=>1}', res3.body 223 | end 224 | 225 | it "provides new session id with :renew option" do 226 | rsd = Rack::Session::Dalli.new(incrementor) 227 | req = Rack::MockRequest.new(rsd) 228 | renew = Rack::Utils::Context.new(rsd, renew_session) 229 | rreq = Rack::MockRequest.new(renew) 230 | 231 | res1 = req.get("/") 232 | session = (cookie = res1["Set-Cookie"])[session_match] 233 | assert_equal '{"counter"=>1}', res1.body 234 | 235 | res2 = rreq.get("/", "HTTP_COOKIE" => cookie) 236 | new_cookie = res2["Set-Cookie"] 237 | new_session = new_cookie[session_match] 238 | refute_equal session, new_session 239 | assert_equal '{"counter"=>2}', res2.body 240 | 241 | res3 = req.get("/", "HTTP_COOKIE" => new_cookie) 242 | assert_equal '{"counter"=>3}', res3.body 243 | 244 | # Old cookie was deleted 245 | res4 = req.get("/", "HTTP_COOKIE" => cookie) 246 | assert_equal '{"counter"=>1}', res4.body 247 | end 248 | 249 | it "omits cookie with :defer option but still updates the state" do 250 | rsd = Rack::Session::Dalli.new(incrementor) 251 | count = Rack::Utils::Context.new(rsd, incrementor) 252 | defer = Rack::Utils::Context.new(rsd, defer_session) 253 | dreq = Rack::MockRequest.new(defer) 254 | creq = Rack::MockRequest.new(count) 255 | 256 | res0 = dreq.get("/") 257 | assert_nil res0["Set-Cookie"] 258 | assert_equal '{"counter"=>1}', res0.body 259 | 260 | res0 = creq.get("/") 261 | res1 = dreq.get("/", "HTTP_COOKIE" => res0["Set-Cookie"]) 262 | assert_equal '{"counter"=>2}', res1.body 263 | res2 = dreq.get("/", "HTTP_COOKIE" => res0["Set-Cookie"]) 264 | assert_equal '{"counter"=>3}', res2.body 265 | end 266 | 267 | it "omits cookie and state update with :skip option" do 268 | rsd = Rack::Session::Dalli.new(incrementor) 269 | count = Rack::Utils::Context.new(rsd, incrementor) 270 | skip = Rack::Utils::Context.new(rsd, skip_session) 271 | sreq = Rack::MockRequest.new(skip) 272 | creq = Rack::MockRequest.new(count) 273 | 274 | res0 = sreq.get("/") 275 | assert_nil res0["Set-Cookie"] 276 | assert_equal '{"counter"=>1}', res0.body 277 | 278 | res0 = creq.get("/") 279 | res1 = sreq.get("/", "HTTP_COOKIE" => res0["Set-Cookie"]) 280 | assert_equal '{"counter"=>2}', res1.body 281 | res2 = sreq.get("/", "HTTP_COOKIE" => res0["Set-Cookie"]) 282 | assert_equal '{"counter"=>2}', res2.body 283 | end 284 | 285 | it "updates deep hashes correctly" do 286 | hash_check = proc do |env| 287 | session = env['rack.session'] 288 | unless session.include? 'test' 289 | session.update :a => :b, :c => { :d => :e }, 290 | :f => { :g => { :h => :i} }, 'test' => true 291 | else 292 | session[:f][:g][:h] = :j 293 | end 294 | [200, {}, [session.inspect]] 295 | end 296 | rsd = Rack::Session::Dalli.new(hash_check) 297 | req = Rack::MockRequest.new(rsd) 298 | 299 | res0 = req.get("/") 300 | session_id = (cookie = res0["Set-Cookie"])[session_match, 1] 301 | ses0 = rsd.pool.get(session_id, true) 302 | 303 | req.get("/", "HTTP_COOKIE" => cookie) 304 | ses1 = rsd.pool.get(session_id, true) 305 | 306 | refute_equal ses0, ses1 307 | end 308 | 309 | # anyone know how to do this better? 310 | it "cleanly merges sessions when multithreaded" do 311 | unless $DEBUG 312 | assert_equal 1, 1 # fake assertion to appease the mighty bacon 313 | next 314 | end 315 | warn 'Running multithread test for Session::Dalli' 316 | rsd = Rack::Session::Dalli.new(incrementor) 317 | req = Rack::MockRequest.new(rsd) 318 | 319 | res = req.get('/') 320 | assert_equal '{"counter"=>1}', res.body 321 | cookie = res["Set-Cookie"] 322 | session_id = cookie[session_match, 1] 323 | 324 | delta_incrementor = lambda do |env| 325 | # emulate disconjoinment of threading 326 | env['rack.session'] = env['rack.session'].dup 327 | Thread.stop 328 | env['rack.session'][(Time.now.usec*rand).to_i] = true 329 | incrementor.call(env) 330 | end 331 | tses = Rack::Utils::Context.new rsd, delta_incrementor 332 | treq = Rack::MockRequest.new(tses) 333 | tnum = rand(7).to_i+5 334 | r = Array.new(tnum) do 335 | Thread.new(treq) do |run| 336 | run.get('/', "HTTP_COOKIE" => cookie, 'rack.multithread' => true) 337 | end 338 | end.reverse.map{|t| t.run.join.value } 339 | r.each do |request| 340 | assert_equal cookie, request['Set-Cookie'] 341 | assert request.body.include?('"counter"=>2') 342 | end 343 | 344 | session = rsd.pool.get(session_id) 345 | assert_equal tnum+1, session.size # counter 346 | assert_equal 2, session['counter'] # meeeh 347 | 348 | tnum = rand(7).to_i+5 349 | r = Array.new(tnum) do |i| 350 | app = Rack::Utils::Context.new rsd, time_delta 351 | req = Rack::MockRequest.new app 352 | Thread.new(req) do |run| 353 | run.get('/', "HTTP_COOKIE" => cookie, 'rack.multithread' => true) 354 | end 355 | end.reverse.map{|t| t.run.join.value } 356 | r.each do |request| 357 | assert_equal cookie, request['Set-Cookie'] 358 | assert request.body.include?('"counter"=>3') 359 | end 360 | 361 | session = rsd.pool.get(session_id) 362 | assert_equal tnum+1, session.size 363 | assert_equal 3, session['counter'] 364 | 365 | drop_counter = proc do |env| 366 | env['rack.session'].delete 'counter' 367 | env['rack.session']['foo'] = 'bar' 368 | [200, {'Content-Type'=>'text/plain'}, env['rack.session'].inspect] 369 | end 370 | tses = Rack::Utils::Context.new rsd, drop_counter 371 | treq = Rack::MockRequest.new(tses) 372 | tnum = rand(7).to_i+5 373 | r = Array.new(tnum) do 374 | Thread.new(treq) do |run| 375 | run.get('/', "HTTP_COOKIE" => cookie, 'rack.multithread' => true) 376 | end 377 | end.reverse.map{|t| t.run.join.value } 378 | r.each do |request| 379 | assert_equal cookie, request['Set-Cookie'] 380 | assert request.body.include?('"foo"=>"bar"') 381 | end 382 | 383 | session = rsd.pool.get(session_id) 384 | assert_equal r.size+1, session.size 385 | assert_nil session['counter'] 386 | assert_equal 'bar', session['foo'] 387 | end 388 | end 389 | -------------------------------------------------------------------------------- /lib/active_support/cache/dalli_store.rb: -------------------------------------------------------------------------------- 1 | # encoding: ascii 2 | # frozen_string_literal: true 3 | require 'dalli' 4 | 5 | module ActiveSupport 6 | module Cache 7 | class DalliStore 8 | 9 | attr_reader :silence, :options 10 | alias_method :silence?, :silence 11 | 12 | def self.supports_cache_versioning? 13 | true 14 | end 15 | 16 | # Silence the logger. 17 | def silence! 18 | @silence = true 19 | self 20 | end 21 | 22 | # Silence the logger within a block. 23 | def mute 24 | previous_silence, @silence = defined?(@silence) && @silence, true 25 | yield 26 | ensure 27 | @silence = previous_silence 28 | end 29 | 30 | ESCAPE_KEY_CHARS = /[\x00-\x20%\x7F-\xFF]/ 31 | 32 | # Creates a new DalliStore object, with the given memcached server 33 | # addresses. Each address is either a host name, or a host-with-port string 34 | # in the form of "host_name:port". For example: 35 | # 36 | # ActiveSupport::Cache::DalliStore.new("localhost", "server-downstairs.localnetwork:8229") 37 | # 38 | # If no addresses are specified, then DalliStore will connect to 39 | # localhost port 11211 (the default memcached port). 40 | # 41 | # Connection Pool support 42 | # 43 | # If you are using multithreaded Rails, the Rails.cache singleton can become a source 44 | # of contention. You can use a connection pool of Dalli clients with Rails.cache by 45 | # passing :pool_size and/or :pool_timeout: 46 | # 47 | # config.cache_store = :dalli_store, 'localhost:11211', :pool_size => 10 48 | # 49 | # Both pool options default to 5. You must include the `connection_pool` gem if you 50 | # wish to use pool support. 51 | # 52 | def initialize(*addresses) 53 | addresses = addresses.flatten 54 | options = addresses.extract_options! 55 | @options = options.dup 56 | 57 | pool_options = {} 58 | pool_options[:size] = options[:pool_size] if options[:pool_size] 59 | pool_options[:timeout] = options[:pool_timeout] if options[:pool_timeout] 60 | 61 | @options[:compress] ||= @options[:compression] 62 | 63 | addresses.compact! 64 | servers = if addresses.empty? 65 | nil # use the default from Dalli::Client 66 | else 67 | addresses 68 | end 69 | if pool_options.empty? 70 | @data = Dalli::Client.new(servers, @options) 71 | else 72 | @data = ::ConnectionPool.new(pool_options) { Dalli::Client.new(servers, @options.merge(:threadsafe => false)) } 73 | end 74 | 75 | extend Strategy::LocalCache 76 | extend LocalCacheEntryUnwrapAndRaw 77 | end 78 | 79 | ## 80 | # Access the underlying Dalli::Client or ConnectionPool instance for 81 | # access to get_multi, etc. 82 | def dalli 83 | @data 84 | end 85 | 86 | def with(&block) 87 | @data.with(&block) 88 | end 89 | 90 | # Fetch the value associated with the key. 91 | # If a value is found, then it is returned. 92 | # 93 | # If a value is not found and no block is given, then nil is returned. 94 | # 95 | # If a value is not found (or if the found value is nil and :cache_nils is false) 96 | # and a block is given, the block will be invoked and its return value 97 | # written to the cache and returned. 98 | def fetch(name, options=nil) 99 | options ||= {} 100 | options[:cache_nils] = true if @options[:cache_nils] 101 | namespaced_name = namespaced_key(name, options) 102 | not_found = options[:cache_nils] ? Dalli::Server::NOT_FOUND : nil 103 | if block_given? 104 | entry = not_found 105 | unless options[:force] 106 | entry = instrument_with_log(:read, namespaced_name, options) do |payload| 107 | read_entry(namespaced_name, options).tap do |result| 108 | if payload 109 | payload[:super_operation] = :fetch 110 | payload[:hit] = not_found != result 111 | end 112 | end 113 | end 114 | end 115 | 116 | if not_found == entry 117 | result = instrument_with_log(:generate, namespaced_name, options) do |payload| 118 | yield(name) 119 | end 120 | write(name, result, options) 121 | result 122 | else 123 | instrument_with_log(:fetch_hit, namespaced_name, options) { |payload| } 124 | entry 125 | end 126 | else 127 | read(name, options) 128 | end 129 | end 130 | 131 | def read(name, options=nil) 132 | options ||= {} 133 | name = namespaced_key(name, options) 134 | 135 | instrument_with_log(:read, name, options) do |payload| 136 | entry = read_entry(name, options) 137 | payload[:hit] = !entry.nil? if payload 138 | entry 139 | end 140 | end 141 | 142 | def write(name, value, options=nil) 143 | options ||= {} 144 | name = namespaced_key(name, options) 145 | 146 | instrument_with_log(:write, name, options) do |payload| 147 | with do |connection| 148 | options = options.merge(:connection => connection) 149 | write_entry(name, value, options) 150 | end 151 | end 152 | end 153 | 154 | def exist?(name, options=nil) 155 | options ||= {} 156 | name = namespaced_key(name, options) 157 | 158 | log(:exist, name, options) 159 | !read_entry(name, options).nil? 160 | end 161 | 162 | def delete(name, options=nil) 163 | options ||= {} 164 | name = namespaced_key(name, options) 165 | 166 | instrument_with_log(:delete, name, options) do |payload| 167 | delete_entry(name, options) 168 | end 169 | end 170 | 171 | # Reads multiple keys from the cache using a single call to the 172 | # servers for all keys. Keys must be Strings. 173 | def read_multi(*names) 174 | options = names.extract_options! 175 | mapping = names.inject({}) { |memo, name| memo[namespaced_key(name, options)] = name; memo } 176 | instrument_with_log(:read_multi, mapping.keys) do 177 | results = {} 178 | if local_cache 179 | mapping.each_key do |key| 180 | if value = local_cache.read_entry(key, options) 181 | results[key] = value 182 | end 183 | end 184 | end 185 | 186 | data = with { |c| c.get_multi(mapping.keys - results.keys) } 187 | results.merge!(data) 188 | results.inject({}) do |memo, (inner, _)| 189 | entry = results[inner] 190 | # NB Backwards data compatibility, to be removed at some point 191 | value = (entry.is_a?(ActiveSupport::Cache::Entry) ? entry.value : entry) 192 | memo[mapping[inner]] = value 193 | local_cache.write_entry(inner, value, options) if local_cache 194 | memo 195 | end 196 | end 197 | end 198 | 199 | # Fetches data from the cache, using the given keys. If there is data in 200 | # the cache with the given keys, then that data is returned. Otherwise, 201 | # the supplied block is called for each key for which there was no data, 202 | # and the result will be written to the cache and returned. 203 | def fetch_multi(*names) 204 | options = names.extract_options! 205 | mapping = names.inject({}) { |memo, name| memo[namespaced_key(name, options)] = name; memo } 206 | 207 | instrument_with_log(:fetch_multi, mapping.keys) do 208 | with do |connection| 209 | results = connection.get_multi(mapping.keys) 210 | 211 | connection.multi do 212 | mapping.inject({}) do |memo, (expanded, name)| 213 | memo[name] = results[expanded] 214 | if memo[name].nil? 215 | value = yield(name) 216 | memo[name] = value 217 | options = options.merge(:connection => connection) 218 | write_entry(expanded, value, options) 219 | end 220 | 221 | memo 222 | end 223 | end 224 | end 225 | end 226 | end 227 | 228 | # Increment a cached value. This method uses the memcached incr atomic 229 | # operator and can only be used on values written with the :raw option. 230 | # Calling it on a value not stored with :raw will fail. 231 | # :initial defaults to the amount passed in, as if the counter was initially zero. 232 | # memcached counters cannot hold negative values. 233 | def increment(name, amount = 1, options=nil) 234 | options ||= {} 235 | name = namespaced_key(name, options) 236 | initial = options.has_key?(:initial) ? options[:initial] : amount 237 | expires_in = options[:expires_in] 238 | instrument_with_log(:increment, name, :amount => amount) do 239 | with { |c| c.incr(name, amount, expires_in, initial) } 240 | end 241 | rescue Dalli::DalliError => e 242 | log_dalli_error(e) 243 | instrument_error(e) if instrument_errors? 244 | raise if raise_errors? 245 | nil 246 | end 247 | 248 | # Decrement a cached value. This method uses the memcached decr atomic 249 | # operator and can only be used on values written with the :raw option. 250 | # Calling it on a value not stored with :raw will fail. 251 | # :initial defaults to zero, as if the counter was initially zero. 252 | # memcached counters cannot hold negative values. 253 | def decrement(name, amount = 1, options=nil) 254 | options ||= {} 255 | name = namespaced_key(name, options) 256 | initial = options.has_key?(:initial) ? options[:initial] : 0 257 | expires_in = options[:expires_in] 258 | instrument_with_log(:decrement, name, :amount => amount) do 259 | with { |c| c.decr(name, amount, expires_in, initial) } 260 | end 261 | rescue Dalli::DalliError => e 262 | log_dalli_error(e) 263 | instrument_error(e) if instrument_errors? 264 | raise if raise_errors? 265 | nil 266 | end 267 | 268 | # Clear the entire cache on all memcached servers. This method should 269 | # be used with care when using a shared cache. 270 | def clear(options=nil) 271 | instrument_with_log(:clear, 'flushing all keys') do 272 | with { |c| c.flush_all } 273 | end 274 | rescue Dalli::DalliError => e 275 | log_dalli_error(e) 276 | instrument_error(e) if instrument_errors? 277 | raise if raise_errors? 278 | nil 279 | end 280 | 281 | # Clear any local cache 282 | def cleanup(options=nil) 283 | end 284 | 285 | # Get the statistics from the memcached servers. 286 | def stats 287 | with { |c| c.stats } 288 | end 289 | 290 | def reset 291 | with { |c| c.reset } 292 | end 293 | 294 | def logger 295 | Dalli.logger 296 | end 297 | 298 | def logger=(new_logger) 299 | Dalli.logger = new_logger 300 | end 301 | 302 | protected 303 | 304 | # Read an entry from the cache. 305 | def read_entry(key, options) # :nodoc: 306 | entry = with { |c| c.get(key, options) } 307 | # NB Backwards data compatibility, to be removed at some point 308 | entry.is_a?(ActiveSupport::Cache::Entry) ? entry.value : entry 309 | rescue Dalli::DalliError => e 310 | log_dalli_error(e) 311 | instrument_error(e) if instrument_errors? 312 | raise if raise_errors? 313 | nil 314 | end 315 | 316 | # Write an entry to the cache. 317 | def write_entry(key, value, options) # :nodoc: 318 | # cleanup LocalCache 319 | cleanup if options[:unless_exist] 320 | method = options[:unless_exist] ? :add : :set 321 | expires_in = options[:expires_in] 322 | connection = options.delete(:connection) 323 | connection.send(method, key, value, expires_in, options) 324 | rescue Dalli::DalliError => e 325 | log_dalli_error(e) 326 | instrument_error(e) if instrument_errors? 327 | raise if raise_errors? 328 | false 329 | end 330 | 331 | # Delete an entry from the cache. 332 | def delete_entry(key, options) # :nodoc: 333 | with { |c| c.delete(key) } 334 | rescue Dalli::DalliError => e 335 | log_dalli_error(e) 336 | instrument_error(e) if instrument_errors? 337 | raise if raise_errors? 338 | false 339 | end 340 | 341 | private 342 | 343 | def namespaced_key(key, options) 344 | key = expanded_key(key) 345 | namespace = options[:namespace] if options 346 | prefix = namespace.is_a?(Proc) ? namespace.call : namespace 347 | key = "#{prefix}:#{key}" if prefix 348 | key = "#{key[0, 213]}:md5:#{@options[:digest_class].hexdigest(key)}" if key && key.size > 250 349 | key 350 | end 351 | alias :normalize_key :namespaced_key 352 | 353 | # Expand key to be a consistent string value. Invokes +cache_key_with_version+ 354 | # first to support Rails 5.2 cache versioning. 355 | # Invoke +cache_key+ if object responds to +cache_key+. Otherwise, to_param method 356 | # will be called. If the key is a Hash, then keys will be sorted alphabetically. 357 | def expanded_key(key) # :nodoc: 358 | return key.cache_key_with_version.to_s if key.respond_to?(:cache_key_with_version) 359 | return key.cache_key.to_s if key.respond_to?(:cache_key) 360 | 361 | case key 362 | when Array 363 | if key.size > 1 364 | key = key.collect{|element| expanded_key(element)} 365 | else 366 | key = key.first 367 | end 368 | when Hash 369 | key = key.sort_by { |k,_| k.to_s }.collect{|k,v| "#{k}=#{v}"} 370 | end 371 | 372 | key = key.to_param 373 | if key.respond_to? :force_encoding 374 | key = key.dup 375 | key.force_encoding('binary') 376 | end 377 | key 378 | end 379 | 380 | def log_dalli_error(error) 381 | logger.error("DalliError: #{error.message}") if logger 382 | end 383 | 384 | def instrument_with_log(operation, key, options=nil) 385 | log(operation, key, options) 386 | 387 | payload = { :key => key } 388 | payload.merge!(options) if options.is_a?(Hash) 389 | instrument(operation, payload) { |p| yield(p) } 390 | end 391 | 392 | def instrument_error(error) 393 | instrument(:error, { :key => 'DalliError', :message => error.message }) 394 | end 395 | 396 | def instrument(operation, payload) 397 | ActiveSupport::Notifications.instrument("cache_#{operation}.active_support", payload) do 398 | yield(payload) if block_given? 399 | end 400 | end 401 | 402 | def log(operation, key, options=nil) 403 | return unless logger && logger.debug? && !silence? 404 | logger.debug("Cache #{operation}: #{key}#{options.blank? ? "" : " (#{options.inspect})"}") 405 | end 406 | 407 | def raise_errors? 408 | !!@options[:raise_errors] 409 | end 410 | 411 | def instrument_errors? 412 | !!@options[:instrument_errors] 413 | end 414 | 415 | # Make sure LocalCache is giving raw values, not `Entry`s, and 416 | # respect `raw` option. 417 | module LocalCacheEntryUnwrapAndRaw # :nodoc: 418 | protected 419 | def read_entry(key, options) 420 | retval = super 421 | if retval.is_a? ActiveSupport::Cache::Entry 422 | # Must have come from LocalStore, unwrap it 423 | if options[:raw] 424 | retval.value.to_s 425 | else 426 | retval.value 427 | end 428 | else 429 | retval 430 | end 431 | end 432 | end 433 | end 434 | end 435 | end 436 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | Dalli Changelog 2 | ===================== 3 | 4 | 2.7.10 5 | ========== 6 | - Revert frozen string change (schneems) 7 | - Advertise supports_cached_versioning? in DalliStore (schneems) 8 | - Better detection of fork support, to allow specs to run under Truffle Ruby (deepj) 9 | - Update logging for over max size to log as error (aeroastro) 10 | 11 | 2.7.9 12 | ========== 13 | - Fix behavior for Rails 5.2+ cache_versioning (GriwMF) 14 | - Ensure fetch provides the key to the fallback block as an argument (0exp) 15 | - Assorted performance improvements (schneems) 16 | 17 | 2.7.8 18 | ========== 19 | - Rails 5.2 compatibility (pbougie) 20 | - Fix Session Cache compatibility (pixeltrix) 21 | 22 | 2.7.7 23 | ========== 24 | - Support large cache keys on fetch multi (sobrinho) 25 | - Not found checks no longer trigger the result's equality method (dannyfallon) 26 | - Use SVG build badges (olleolleolle) 27 | - Travis updates (junaruga, tiarly, petergoldstein) 28 | - Update default down_retry_delay (jaredhales) 29 | - Close kgio socket after IO.select timeouts 30 | - Documentation updates (tipair) 31 | - Instrument DalliStore errors with instrument_errors configuration option. (btatnall) 32 | 33 | 2.7.6 34 | ========== 35 | - Rails 5.0.0.beta2 compatibility (yui-knk, petergoldstein) 36 | - Add cas!, a variant of the #cas method that yields to the block whether or not the key already exist (mwpastore) 37 | - Performance improvements (nateberkopec) 38 | - Add Ruby 2.3.0 to support matrix (tricknotes) 39 | 40 | 2.7.5 41 | ========== 42 | 43 | - Support rcvbuff and sndbuff byte configuration. (btatnall) 44 | - Add `:cache_nils` option to support nil values in `DalliStore#fetch` and `Dalli::Client#fetch` (wjordan, #559) 45 | - Log retryable server errors with 'warn' instead of 'info' (phrinx) 46 | - Fix timeout issue with Dalli::Client#get_multi_yielder (dspeterson) 47 | - Escape namespaces with special regexp characters (Steven Peckins) 48 | - Ensure LocalCache supports the `:raw` option and Entry unwrapping (sj26) 49 | - Ensure bad ttl values don't cause Dalli::RingError (eagletmt, petergoldstein) 50 | - Always pass namespaced key to instrumentation API (kaorimatz) 51 | - Replace use of deprecated TimeoutError with Timeout::Error (eagletmt) 52 | - Clean up gemspec, and use Bundler for loading (grosser) 53 | - Dry up local cache testing (grosser) 54 | 55 | 2.7.4 56 | ========== 57 | 58 | - Restore Windows compatibility (dfens, #524) 59 | 60 | 2.7.3 61 | ========== 62 | 63 | - Assorted spec improvements 64 | - README changes to specify defaults for failover and compress options (keen99, #470) 65 | - SASL authentication changes to deal with Unicode characters (flypiggy, #477) 66 | - Call to_i on ttl to accomodate ActiveSupport::Duration (#494) 67 | - Change to implicit blocks for performance (glaucocustodio, #495) 68 | - Change to each_key for performance (jastix, #496) 69 | - Support stats settings - (dterei, #500) 70 | - Raise DallError if hostname canno be parsed (dannyfallon, #501) 71 | - Fix instrumentation for falsey values (AlexRiedler, #514) 72 | - Support UNIX socket configurations (r-stu31, #515) 73 | 74 | 2.7.2 75 | ========== 76 | 77 | - The fix for #423 didn't make it into the released 2.7.1 gem somehow. 78 | 79 | 2.7.1 80 | ========== 81 | 82 | - Rack session will check if servers are up on initialization (arthurnn, #423) 83 | - Add support for IPv6 addresses in hex form, ie: "[::1]:11211" (dplummer, #428) 84 | - Add symbol support for namespace (jingkai #431) 85 | - Support expiration intervals longer than 30 days (leonid-shevtsov #436) 86 | 87 | 2.7.0 88 | ========== 89 | 90 | - BREAKING CHANGE: 91 | Dalli::Client#add and #replace now return a truthy value, not boolean true or false. 92 | - Multithreading support with dalli\_store: 93 | Use :pool\_size to create a pool of shared, threadsafe Dalli clients in Rails: 94 | ```ruby 95 | config.cache_store = :dalli_store, "cache-1.example.com", "cache-2.example.com", :compress => true, :pool_size => 5, :expires_in => 300 96 | ``` 97 | This will ensure the Rails.cache singleton does not become a source of contention. 98 | **PLEASE NOTE** Rails's :mem\_cache\_store does not support pooling as of 99 | Rails 4.0. You must use :dalli\_store. 100 | 101 | - Implement `version` for retrieving version of connected servers [dterei, #384] 102 | - Implement `fetch_multi` for batched read/write [sorentwo, #380] 103 | - Add more support for safe updates with multiple writers: [philipmw, #395] 104 | `require 'dalli/cas/client'` augments Dalli::Client with the following methods: 105 | * Get value with CAS: `[value, cas] = get_cas(key)` 106 | `get_cas(key) {|value, cas| ...}` 107 | * Get multiple values with CAS: `get_multi_cas(k1, k2, ...) {|value, metadata| cas = metadata[:cas]}` 108 | * Set value with CAS: `new_cas = set_cas(key, value, cas, ttl, options)` 109 | * Replace value with CAS: `replace_cas(key, new_value, cas, ttl, options)` 110 | * Delete value with CAS: `delete_cas(key, cas)` 111 | - Fix bug with get key with "Not found" value [uzzz, #375] 112 | 113 | 2.6.4 114 | ======= 115 | 116 | - Fix ADD command, aka `write(unless_exist: true)` (pitr, #365) 117 | - Upgrade test suite from mini\_shoulda to minitest. 118 | - Even more performance improvements for get\_multi (xaop, #331) 119 | 120 | 2.6.3 121 | ======= 122 | 123 | - Support specific stats by passing `:items` or `:slabs` to `stats` method [bukhamseen] 124 | - Fix 'can't modify frozen String' errors in `ActiveSupport::Cache::DalliStore` [dblock] 125 | - Protect against objects with custom equality checking [theron17] 126 | - Warn if value for key is too large to store [locriani] 127 | 128 | 2.6.2 129 | ======= 130 | 131 | - Properly handle missing RubyInline 132 | 133 | 2.6.1 134 | ======= 135 | 136 | - Add optional native C binary search for ring, add: 137 | 138 | gem 'RubyInline' 139 | 140 | to your Gemfile to get a 10% speedup when using many servers. 141 | You will see no improvement if you are only using one server. 142 | 143 | - More get_multi performance optimization [xaop, #315] 144 | - Add lambda support for cache namespaces [joshwlewis, #311] 145 | 146 | 2.6.0 147 | ======= 148 | 149 | - read_multi optimization, now checks local_cache [chendo, #306] 150 | - Re-implement get_multi to be non-blocking [tmm1, #295] 151 | - Add `dalli` accessor to dalli_store to access the underlying 152 | Dalli::Client, for things like `get_multi`. 153 | - Add `Dalli::GzipCompressor`, primarily for compatibility with nginx's HttpMemcachedModule using `memcached_gzip_flag` 154 | 155 | 2.5.0 156 | ======= 157 | 158 | - Don't escape non-ASCII keys, memcached binary protocol doesn't care. [#257] 159 | - :dalli_store now implements LocalCache [#236] 160 | - Removed lots of old session_store test code, tests now all run without a default memcached server [#275] 161 | - Changed Dalli ActiveSupport adapter to always attempt instrumentation [brianmario, #284] 162 | - Change write operations (add/set/replace) to return false when value is too large to store [brianmario, #283] 163 | - Allowing different compressors per client [naseem] 164 | 165 | 2.4.0 166 | ======= 167 | - Added the ability to swap out the compressed used to [de]compress cache data [brianmario, #276] 168 | - Fix get\_multi performance issues with lots of memcached servers [tmm1] 169 | - Throw more specific exceptions [tmm1] 170 | - Allowing different types of serialization per client [naseem] 171 | 172 | 2.3.0 173 | ======= 174 | - Added the ability to swap out the serializer used to [de]serialize cache data [brianmario, #274] 175 | 176 | 2.2.1 177 | ======= 178 | 179 | - Fix issues with ENV-based connections. [#266] 180 | - Fix problem with SessionStore in Rails 4.0 [#265] 181 | 182 | 2.2.0 183 | ======= 184 | 185 | - Add Rack session with\_lock helper, for Rails 4.0 support [#264] 186 | - Accept connection string in the form of a URL (e.g., memcached://user:pass@hostname:port) [glenngillen] 187 | - Add touch operation [#228, uzzz] 188 | 189 | 2.1.0 190 | ======= 191 | 192 | - Add Railtie to auto-configure Dalli when included in Gemfile [#217, steveklabnik] 193 | 194 | 2.0.5 195 | ======= 196 | 197 | - Create proper keys for arrays of objects passed as keys [twinturbo, #211] 198 | - Handle long key with namespace [#212] 199 | - Add NODELAY to TCP socket options [#206] 200 | 201 | 2.0.4 202 | ======= 203 | 204 | - Dalli no longer needs to be reset after Unicorn/Passenger fork [#208] 205 | - Add option to re-raise errors rescued in the session and cache stores. [pitr, #200] 206 | - DalliStore#fetch called the block if the cached value == false [#205] 207 | - DalliStore should have accessible options [#195] 208 | - Add silence and mute support for DalliStore [#207] 209 | - Tracked down and fixed socket corruption due to Timeout [#146] 210 | 211 | 2.0.3 212 | ======= 213 | 214 | - Allow proper retrieval of stored `false` values [laserlemon, #197] 215 | - Allow non-ascii and whitespace keys, only the text protocol has those restrictions [#145] 216 | - Fix DalliStore#delete error-handling [#196] 217 | 218 | 2.0.2 219 | ======= 220 | 221 | - Fix all dalli\_store operations to handle nil options [#190] 222 | - Increment and decrement with :initial => nil now return nil (lawrencepit, #112) 223 | 224 | 2.0.1 225 | ======= 226 | 227 | - Fix nil option handling in dalli\_store#write [#188] 228 | 229 | 2.0.0 230 | ======= 231 | 232 | - Reimplemented the Rails' dalli\_store to remove use of 233 | ActiveSupport::Cache::Entry which added 109 bytes overhead to every 234 | value stored, was a performance bottleneck and duplicated a lot of 235 | functionality already in Dalli. One benchmark went from 4.0 sec to 3.0 236 | sec with the new dalli\_store. [#173] 237 | - Added reset\_stats operation [#155] 238 | - Added support for configuring keepalive on TCP connections to memcached servers (@bianster, #180) 239 | 240 | Notes: 241 | 242 | * data stored with dalli\_store 2.x is NOT backwards compatible with 1.x. 243 | Upgraders are advised to namespace their keys and roll out the 2.x 244 | upgrade slowly so keys do not clash and caches are warmed. 245 | `config.cache_store = :dalli_store, :expires_in => 24.hours.to_i, :namespace => 'myapp2'` 246 | * data stored with plain Dalli::Client API is unchanged. 247 | * removed support for dalli\_store's race\_condition\_ttl option. 248 | * removed support for em-synchrony and unix socket connection options. 249 | * removed support for Ruby 1.8.6 250 | * removed memcache-client compability layer and upgrade documentation. 251 | 252 | 253 | 1.1.5 254 | ======= 255 | 256 | - Coerce input to incr/decr to integer via #to\_i [#165] 257 | - Convert test suite to minitest/spec (crigor, #166) 258 | - Fix encoding issue with keys [#162] 259 | - Fix double namespacing with Rails and dalli\_store. [#160] 260 | 261 | 1.1.4 262 | ======= 263 | 264 | - Use 127.0.0.1 instead of localhost as default to avoid IPv6 issues 265 | - Extend DalliStore's :expires\_in when :race\_condition\_ttl is also used. 266 | - Fix :expires\_in option not propogating from DalliStore to Client, GH-136 267 | - Added support for native Rack session store. Until now, Dalli's 268 | session store has required Rails. Now you can use Dalli to store 269 | sessions for any Rack application. 270 | 271 | require 'rack/session/dalli' 272 | use Rack::Session::Dalli, :memcache_server => 'localhost:11211', :compression => true 273 | 274 | 1.1.3 275 | ======= 276 | 277 | - Support Rails's autoloading hack for loading sessions with objects 278 | whose classes have not be required yet, GH-129 279 | - Support Unix sockets for connectivity. Shows a 2x performance 280 | increase but keep in mind they only work on localhost. (dfens) 281 | 282 | 1.1.2 283 | ======= 284 | 285 | - Fix incompatibility with latest Rack session API when destroying 286 | sessions, thanks @twinge! 287 | 288 | 1.1.1 289 | ======= 290 | 291 | v1.1.0 was a bad release. Yanked. 292 | 293 | 1.1.0 294 | ======= 295 | 296 | - Remove support for Rails 2.3, add support for Rails 3.1 297 | - Fix socket failure retry logic, now you can restart memcached and Dalli won't complain! 298 | - Add support for fibered operation via em-synchrony (eliaslevy) 299 | - Gracefully handle write timeouts, GH-99 300 | - Only issue bug warning for unexpected StandardErrors, GH-102 301 | - Add travis-ci build support (ryanlecompte) 302 | - Gracefully handle errors in get_multi (michaelfairley) 303 | - Misc fixes from crash2burn, fphilipe, igreg, raggi 304 | 305 | 1.0.5 306 | ======= 307 | 308 | - Fix socket failure retry logic, now you can restart memcached and Dalli won't complain! 309 | 310 | 1.0.4 311 | ======= 312 | 313 | - Handle non-ASCII key content in dalli_store 314 | - Accept key array for read_multi in dalli_store 315 | - Fix multithreaded race condition in creation of mutex 316 | 317 | 1.0.3 318 | ======= 319 | 320 | - Better handling of application marshalling errors 321 | - Work around jruby IO#sysread compatibility issue 322 | 323 | 324 | 1.0.2 325 | ======= 326 | 327 | - Allow browser session cookies (blindsey) 328 | - Compatibility fixes (mwynholds) 329 | - Add backwards compatibility module for memcache-client, require 'dalli/memcache-client'. It makes 330 | Dalli more compatible with memcache-client and prints out a warning any time you do something that 331 | is no longer supported so you can fix your code. 332 | 333 | 1.0.1 334 | ======= 335 | 336 | - Explicitly handle application marshalling bugs, GH-56 337 | - Add support for username/password as options, to allow multiple bucket access 338 | from the same Ruby process, GH-52 339 | - Add support for >1MB values with :value_max_bytes option, GH-54 (r-stu31) 340 | - Add support for default TTL, :expires_in, in Rails 2.3. (Steven Novotny) 341 | config.cache_store = :dalli_store, 'localhost:11211', {:expires_in => 4.hours} 342 | 343 | 344 | 1.0.0 345 | ======= 346 | 347 | Welcome gucki as a Dalli committer! 348 | 349 | - Fix network and namespace issues in get_multi (gucki) 350 | - Better handling of unmarshalling errors (mperham) 351 | 352 | 0.11.2 353 | ======= 354 | 355 | - Major reworking of socket error and failover handling (gucki) 356 | - Add basic JRuby support (mperham) 357 | 358 | 0.11.1 359 | ====== 360 | 361 | - Minor fixes, doc updates. 362 | - Add optional support for kgio sockets, gives a 10-15% performance boost. 363 | 364 | 0.11.0 365 | ====== 366 | 367 | Warning: this release changes how Dalli marshals data. I do not guarantee compatibility until 1.0 but I will increment the minor version every time a release breaks compatibility until 1.0. 368 | 369 | IT IS HIGHLY RECOMMENDED YOU FLUSH YOUR CACHE BEFORE UPGRADING. 370 | 371 | - multi() now works reentrantly. 372 | - Added new Dalli::Client option for default TTLs, :expires_in, defaults to 0 (aka forever). 373 | - Added new Dalli::Client option, :compression, to enable auto-compression of values. 374 | - Refactor how Dalli stores data on the server. Values are now tagged 375 | as "marshalled" or "compressed" so they can be automatically deserialized 376 | without the client having to know how they were stored. 377 | 378 | 0.10.1 379 | ====== 380 | 381 | - Prefer server config from environment, fixes Heroku session store issues (thanks JoshMcKin) 382 | - Better handling of non-ASCII values (size -> bytesize) 383 | - Assert that keys are ASCII only 384 | 385 | 0.10.0 386 | ====== 387 | 388 | Warning: this release changed how Rails marshals data with Dalli. Unfortunately previous versions double marshalled values. It is possible that data stored with previous versions of Dalli will not work with this version. 389 | 390 | IT IS HIGHLY RECOMMENDED YOU FLUSH YOUR CACHE BEFORE UPGRADING. 391 | 392 | - Rework how the Rails cache store does value marshalling. 393 | - Rework old server version detection to avoid a socket read hang. 394 | - Refactor the Rails 2.3 :dalli\_store to be closer to :mem\_cache\_store. 395 | - Better documentation for session store config (plukevdh) 396 | 397 | 0.9.10 398 | ---- 399 | 400 | - Better server retry logic (next2you) 401 | - Rails 3.1 compatibility (gucki) 402 | 403 | 404 | 0.9.9 405 | ---- 406 | 407 | - Add support for *_multi operations for add, set, replace and delete. This implements 408 | pipelined network operations; Dalli disables network replies so we're not limited by 409 | latency, allowing for much higher throughput. 410 | 411 | dc = Dalli::Client.new 412 | dc.multi do 413 | dc.set 'a', 1 414 | dc.set 'b', 2 415 | dc.set 'c', 3 416 | dc.delete 'd' 417 | end 418 | - Minor fix to set the continuum sorted by value (kangster) 419 | - Implement session store with Rails 2.3. Update docs. 420 | 421 | 0.9.8 422 | ----- 423 | 424 | - Implement namespace support 425 | - Misc fixes 426 | 427 | 428 | 0.9.7 429 | ----- 430 | 431 | - Small fix for NewRelic integration. 432 | - Detect and fail on older memcached servers (pre-1.4). 433 | 434 | 0.9.6 435 | ----- 436 | 437 | - Patches for Rails 3.0.1 integration. 438 | 439 | 0.9.5 440 | ----- 441 | 442 | - Major design change - raw support is back to maximize compatibility with Rails 443 | and the increment/decrement operations. You can now pass :raw => true to most methods 444 | to bypass (un)marshalling. 445 | - Support symbols as keys (ddollar) 446 | - Rails 2.3 bug fixes 447 | 448 | 449 | 0.9.4 450 | ----- 451 | 452 | - Dalli support now in rack-bug (http://github.com/brynary/rack-bug), give it a try! 453 | - Namespace support for Rails 2.3 (bpardee) 454 | - Bug fixes 455 | 456 | 457 | 0.9.3 458 | ----- 459 | 460 | - Rails 2.3 support (beanieboi) 461 | - Rails SessionStore support 462 | - Passenger integration 463 | - memcache-client upgrade docs, see Upgrade.md 464 | 465 | 466 | 0.9.2 467 | ---- 468 | 469 | - Verify proper operation in Heroku. 470 | 471 | 472 | 0.9.1 473 | ---- 474 | 475 | - Add fetch and cas operations (mperham) 476 | - Add incr and decr operations (mperham) 477 | - Initial support for SASL authentication via the MEMCACHE_{USERNAME,PASSWORD} environment variables, needed for Heroku (mperham) 478 | 479 | 0.9.0 480 | ----- 481 | 482 | - Initial gem release. 483 | -------------------------------------------------------------------------------- /lib/dalli/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'digest/md5' 3 | require 'set' 4 | 5 | # encoding: ascii 6 | module Dalli 7 | class Client 8 | 9 | ## 10 | # Dalli::Client is the main class which developers will use to interact with 11 | # the memcached server. Usage: 12 | # 13 | # Dalli::Client.new(['localhost:11211:10', 'cache-2.example.com:11211:5', '192.168.0.1:22122:5', '/var/run/memcached/socket'], 14 | # :threadsafe => true, :failover => true, :expires_in => 300) 15 | # 16 | # servers is an Array of "host:port:weight" where weight allows you to distribute cache unevenly. 17 | # Both weight and port are optional. If you pass in nil, Dalli will use the MEMCACHE_SERVERS 18 | # environment variable or default to 'localhost:11211' if it is not present. Dalli also supports 19 | # the ability to connect to Memcached on localhost through a UNIX socket. To use this functionality, 20 | # use a full pathname (beginning with a slash character '/') in place of the "host:port" pair in 21 | # the server configuration. 22 | # 23 | # Options: 24 | # - :namespace - prepend each key with this value to provide simple namespacing. 25 | # - :failover - if a server is down, look for and store values on another server in the ring. Default: true. 26 | # - :threadsafe - ensure that only one thread is actively using a socket at a time. Default: true. 27 | # - :expires_in - default TTL in seconds if you do not pass TTL as a parameter to an individual operation, defaults to 0 or forever 28 | # - :compress - defaults to false, if true Dalli will compress values larger than 1024 bytes before sending them to memcached. 29 | # - :serializer - defaults to Marshal 30 | # - :compressor - defaults to zlib 31 | # - :cache_nils - defaults to false, if true Dalli will not treat cached nil values as 'not found' for #fetch operations. 32 | # - :digest_class - defaults to Digest::MD5, allows you to pass in an object that responds to the hexdigest method, useful for injecting a FIPS compliant hash object. 33 | # 34 | def initialize(servers=nil, options={}) 35 | options[:digest_class] = ::Digest::MD5 unless options[:digest_class] 36 | @servers = normalize_servers(servers || ENV["MEMCACHE_SERVERS"] || '127.0.0.1:11211') 37 | @options = normalize_options(options) 38 | @ring = nil 39 | end 40 | 41 | # 42 | # The standard memcached instruction set 43 | # 44 | 45 | ## 46 | # Turn on quiet aka noreply support. 47 | # All relevant operations within this block will be effectively 48 | # pipelined as Dalli will use 'quiet' operations where possible. 49 | # Currently supports the set, add, replace and delete operations. 50 | def multi 51 | old, Thread.current[:dalli_multi] = Thread.current[:dalli_multi], true 52 | yield 53 | ensure 54 | Thread.current[:dalli_multi] = old 55 | end 56 | 57 | ## 58 | # Get the value associated with the key. 59 | # If a value is not found, then +nil+ is returned. 60 | def get(key, options=nil) 61 | perform(:get, key, options) 62 | end 63 | 64 | ## 65 | # Fetch multiple keys efficiently. 66 | # If a block is given, yields key/value pairs one at a time. 67 | # Otherwise returns a hash of { 'key' => 'value', 'key2' => 'value1' } 68 | def get_multi(*keys) 69 | check_keys = keys.flatten 70 | check_keys.compact! 71 | 72 | return {} if check_keys.empty? 73 | if block_given? 74 | get_multi_yielder(keys) {|k, data| yield k, data.first} 75 | else 76 | Hash.new.tap do |hash| 77 | get_multi_yielder(keys) {|k, data| hash[k] = data.first} 78 | end 79 | end 80 | end 81 | 82 | CACHE_NILS = {cache_nils: true}.freeze 83 | 84 | # Fetch the value associated with the key. 85 | # If a value is found, then it is returned. 86 | # 87 | # If a value is not found and no block is given, then nil is returned. 88 | # 89 | # If a value is not found (or if the found value is nil and :cache_nils is false) 90 | # and a block is given, the block will be invoked and its return value 91 | # written to the cache and returned. 92 | def fetch(key, ttl=nil, options=nil) 93 | options = options.nil? ? CACHE_NILS : options.merge(CACHE_NILS) if @options[:cache_nils] 94 | val = get(key, options) 95 | not_found = @options[:cache_nils] ? 96 | val == Dalli::Server::NOT_FOUND : 97 | val.nil? 98 | if not_found && block_given? 99 | val = yield 100 | add(key, val, ttl_or_default(ttl), options) 101 | end 102 | val 103 | end 104 | 105 | ## 106 | # compare and swap values using optimistic locking. 107 | # Fetch the existing value for key. 108 | # If it exists, yield the value to the block. 109 | # Add the block's return value as the new value for the key. 110 | # Add will fail if someone else changed the value. 111 | # 112 | # Returns: 113 | # - nil if the key did not exist. 114 | # - false if the value was changed by someone else. 115 | # - true if the value was successfully updated. 116 | def cas(key, ttl=nil, options=nil, &block) 117 | cas_core(key, false, ttl, options, &block) 118 | end 119 | 120 | ## 121 | # like #cas, but will yield to the block whether or not the value 122 | # already exists. 123 | # 124 | # Returns: 125 | # - false if the value was changed by someone else. 126 | # - true if the value was successfully updated. 127 | def cas!(key, ttl=nil, options=nil, &block) 128 | cas_core(key, true, ttl, options, &block) 129 | end 130 | 131 | def set(key, value, ttl=nil, options=nil) 132 | perform(:set, key, value, ttl_or_default(ttl), 0, options) 133 | end 134 | 135 | ## 136 | # Conditionally add a key/value pair, if the key does not already exist 137 | # on the server. Returns truthy if the operation succeeded. 138 | def add(key, value, ttl=nil, options=nil) 139 | perform(:add, key, value, ttl_or_default(ttl), options) 140 | end 141 | 142 | ## 143 | # Conditionally add a key/value pair, only if the key already exists 144 | # on the server. Returns truthy if the operation succeeded. 145 | def replace(key, value, ttl=nil, options=nil) 146 | perform(:replace, key, value, ttl_or_default(ttl), 0, options) 147 | end 148 | 149 | def delete(key) 150 | perform(:delete, key, 0) 151 | end 152 | 153 | ## 154 | # Append value to the value already stored on the server for 'key'. 155 | # Appending only works for values stored with :raw => true. 156 | def append(key, value) 157 | perform(:append, key, value.to_s) 158 | end 159 | 160 | ## 161 | # Prepend value to the value already stored on the server for 'key'. 162 | # Prepending only works for values stored with :raw => true. 163 | def prepend(key, value) 164 | perform(:prepend, key, value.to_s) 165 | end 166 | 167 | def flush(delay=0) 168 | time = -delay 169 | ring.servers.map { |s| s.request(:flush, time += delay) } 170 | end 171 | 172 | alias_method :flush_all, :flush 173 | 174 | ## 175 | # Incr adds the given amount to the counter on the memcached server. 176 | # Amt must be a positive integer value. 177 | # 178 | # If default is nil, the counter must already exist or the operation 179 | # will fail and will return nil. Otherwise this method will return 180 | # the new value for the counter. 181 | # 182 | # Note that the ttl will only apply if the counter does not already 183 | # exist. To increase an existing counter and update its TTL, use 184 | # #cas. 185 | def incr(key, amt=1, ttl=nil, default=nil) 186 | raise ArgumentError, "Positive values only: #{amt}" if amt < 0 187 | perform(:incr, key, amt.to_i, ttl_or_default(ttl), default) 188 | end 189 | 190 | ## 191 | # Decr subtracts the given amount from the counter on the memcached server. 192 | # Amt must be a positive integer value. 193 | # 194 | # memcached counters are unsigned and cannot hold negative values. Calling 195 | # decr on a counter which is 0 will just return 0. 196 | # 197 | # If default is nil, the counter must already exist or the operation 198 | # will fail and will return nil. Otherwise this method will return 199 | # the new value for the counter. 200 | # 201 | # Note that the ttl will only apply if the counter does not already 202 | # exist. To decrease an existing counter and update its TTL, use 203 | # #cas. 204 | def decr(key, amt=1, ttl=nil, default=nil) 205 | raise ArgumentError, "Positive values only: #{amt}" if amt < 0 206 | perform(:decr, key, amt.to_i, ttl_or_default(ttl), default) 207 | end 208 | 209 | ## 210 | # Touch updates expiration time for a given key. 211 | # 212 | # Returns true if key exists, otherwise nil. 213 | def touch(key, ttl=nil) 214 | resp = perform(:touch, key, ttl_or_default(ttl)) 215 | resp.nil? ? nil : true 216 | end 217 | 218 | ## 219 | # Collect the stats for each server. 220 | # You can optionally pass a type including :items, :slabs or :settings to get specific stats 221 | # Returns a hash like { 'hostname:port' => { 'stat1' => 'value1', ... }, 'hostname2:port' => { ... } } 222 | def stats(type=nil) 223 | type = nil if ![nil, :items,:slabs,:settings].include? type 224 | values = {} 225 | ring.servers.each do |server| 226 | values["#{server.name}"] = server.alive? ? server.request(:stats,type.to_s) : nil 227 | end 228 | values 229 | end 230 | 231 | ## 232 | # Reset stats for each server. 233 | def reset_stats 234 | ring.servers.map do |server| 235 | server.alive? ? server.request(:reset_stats) : nil 236 | end 237 | end 238 | 239 | ## 240 | ## Make sure memcache servers are alive, or raise an Dalli::RingError 241 | def alive! 242 | ring.server_for_key("") 243 | end 244 | 245 | ## 246 | ## Version of the memcache servers. 247 | def version 248 | values = {} 249 | ring.servers.each do |server| 250 | values["#{server.name}"] = server.alive? ? server.request(:version) : nil 251 | end 252 | values 253 | end 254 | 255 | ## 256 | # Close our connection to each server. 257 | # If you perform another operation after this, the connections will be re-established. 258 | def close 259 | if @ring 260 | @ring.servers.each { |s| s.close } 261 | @ring = nil 262 | end 263 | end 264 | alias_method :reset, :close 265 | 266 | # Stub method so a bare Dalli client can pretend to be a connection pool. 267 | def with 268 | yield self 269 | end 270 | 271 | private 272 | 273 | def cas_core(key, always_set, ttl=nil, options=nil) 274 | (value, cas) = perform(:cas, key) 275 | value = (!value || value == 'Not found') ? nil : value 276 | return if value.nil? && !always_set 277 | newvalue = yield(value) 278 | perform(:set, key, newvalue, ttl_or_default(ttl), cas, options) 279 | end 280 | 281 | def ttl_or_default(ttl) 282 | (ttl || @options[:expires_in]).to_i 283 | rescue NoMethodError 284 | raise ArgumentError, "Cannot convert ttl (#{ttl}) to an integer" 285 | end 286 | 287 | def groups_for_keys(*keys) 288 | groups = mapped_keys(keys).flatten.group_by do |key| 289 | begin 290 | ring.server_for_key(key) 291 | rescue Dalli::RingError 292 | Dalli.logger.debug { "unable to get key #{key}" } 293 | nil 294 | end 295 | end 296 | return groups 297 | end 298 | 299 | def mapped_keys(keys) 300 | keys_array = keys.flatten 301 | keys_array.map! { |a| validate_key(a.to_s) } 302 | keys_array 303 | end 304 | 305 | def make_multi_get_requests(groups) 306 | groups.each do |server, keys_for_server| 307 | begin 308 | # TODO: do this with the perform chokepoint? 309 | # But given the fact that fetching the response doesn't take place 310 | # in that slot it's misleading anyway. Need to move all of this method 311 | # into perform to be meaningful 312 | server.request(:send_multiget, keys_for_server) 313 | rescue DalliError, NetworkError => e 314 | Dalli.logger.debug { e.inspect } 315 | Dalli.logger.debug { "unable to get keys for server #{server.name}" } 316 | end 317 | end 318 | end 319 | 320 | def perform_multi_response_start(servers) 321 | servers.each do |server| 322 | next unless server.alive? 323 | begin 324 | server.multi_response_start 325 | rescue DalliError, NetworkError => e 326 | Dalli.logger.debug { e.inspect } 327 | Dalli.logger.debug { "results from this server will be missing" } 328 | servers.delete(server) 329 | end 330 | end 331 | servers 332 | end 333 | 334 | ## 335 | # Normalizes the argument into an array of servers. 336 | # If the argument is a string, it's expected that the URIs are comma separated e.g. 337 | # "memcache1.example.com:11211,memcache2.example.com:11211,memcache3.example.com:11211" 338 | def normalize_servers(servers) 339 | if servers.is_a? String 340 | return servers.split(",") 341 | else 342 | return servers 343 | end 344 | end 345 | 346 | def ring 347 | @ring ||= Dalli::Ring.new( 348 | @servers.map do |s| 349 | server_options = {} 350 | if s =~ %r{\Amemcached://} 351 | uri = URI.parse(s) 352 | server_options[:username] = uri.user 353 | server_options[:password] = uri.password 354 | s = "#{uri.host}:#{uri.port}" 355 | end 356 | options = @options.dup.merge(server_options) 357 | options.delete(:digest_class) 358 | Dalli::Server.new(s, options) 359 | end, @options 360 | ) 361 | end 362 | 363 | # Chokepoint method for instrumentation 364 | def perform(*all_args) 365 | return yield if block_given? 366 | op, key, *args = *all_args 367 | 368 | key = key.to_s 369 | key = validate_key(key) 370 | begin 371 | server = ring.server_for_key(key) 372 | ret = server.request(op, key, *args) 373 | ret 374 | rescue NetworkError => e 375 | Dalli.logger.debug { e.inspect } 376 | Dalli.logger.debug { "retrying request with new server" } 377 | retry 378 | end 379 | end 380 | 381 | def validate_key(key) 382 | raise ArgumentError, "key cannot be blank" if !key || key.length == 0 383 | key = key_with_namespace(key) 384 | if key.length > 250 385 | max_length_before_namespace = 212 - (namespace || '').size 386 | key = "#{key[0, max_length_before_namespace]}:md5:#{@options[:digest_class].hexdigest(key)}" 387 | end 388 | return key 389 | end 390 | 391 | def key_with_namespace(key) 392 | (ns = namespace) ? "#{ns}:#{key}" : key 393 | end 394 | 395 | def key_without_namespace(key) 396 | (ns = namespace) ? key.sub(%r(\A#{Regexp.escape ns}:), '') : key 397 | end 398 | 399 | def namespace 400 | return nil unless @options[:namespace] 401 | @options[:namespace].is_a?(Proc) ? @options[:namespace].call.to_s : @options[:namespace].to_s 402 | end 403 | 404 | def normalize_options(opts) 405 | if opts[:compression] 406 | Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just :compress => true. Please update your configuration." 407 | opts[:compress] = opts.delete(:compression) 408 | end 409 | begin 410 | opts[:expires_in] = opts[:expires_in].to_i if opts[:expires_in] 411 | rescue NoMethodError 412 | raise ArgumentError, "cannot convert :expires_in => #{opts[:expires_in].inspect} to an integer" 413 | end 414 | unless opts[:digest_class].respond_to? :hexdigest 415 | raise ArgumentError, "The digest_class object must respond to the hexdigest method" 416 | end 417 | opts 418 | end 419 | 420 | ## 421 | # Yields, one at a time, keys and their values+attributes. 422 | def get_multi_yielder(keys) 423 | perform do 424 | return {} if keys.empty? 425 | ring.lock do 426 | begin 427 | groups = groups_for_keys(keys) 428 | if unfound_keys = groups.delete(nil) 429 | Dalli.logger.debug { "unable to get keys for #{unfound_keys.length} keys because no matching server was found" } 430 | end 431 | make_multi_get_requests(groups) 432 | 433 | servers = groups.keys 434 | return if servers.empty? 435 | servers = perform_multi_response_start(servers) 436 | 437 | start = Time.now 438 | while true 439 | # remove any dead servers 440 | servers.delete_if { |s| s.sock.nil? } 441 | break if servers.empty? 442 | 443 | # calculate remaining timeout 444 | elapsed = Time.now - start 445 | timeout = servers.first.options[:socket_timeout] 446 | time_left = (elapsed > timeout) ? 0 : timeout - elapsed 447 | 448 | sockets = servers.map(&:sock) 449 | readable, _ = IO.select(sockets, nil, nil, time_left) 450 | 451 | if readable.nil? 452 | # no response within timeout; abort pending connections 453 | servers.each do |server| 454 | Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" } 455 | server.multi_response_abort 456 | end 457 | break 458 | 459 | else 460 | readable.each do |sock| 461 | server = sock.server 462 | 463 | begin 464 | server.multi_response_nonblock.each_pair do |key, value_list| 465 | yield key_without_namespace(key), value_list 466 | end 467 | 468 | if server.multi_response_completed? 469 | servers.delete(server) 470 | end 471 | rescue NetworkError 472 | servers.delete(server) 473 | end 474 | end 475 | end 476 | end 477 | end 478 | end 479 | end 480 | end 481 | 482 | end 483 | end 484 | -------------------------------------------------------------------------------- /test/test_active_support.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # frozen_string_literal: true 3 | require_relative 'helper' 4 | require 'connection_pool' 5 | require 'openssl' 6 | 7 | class MockUser 8 | def cache_key 9 | "users/1/21348793847982314" 10 | end 11 | end 12 | 13 | class MockUserVersioning 14 | def cache_key_with_version 15 | "users/1/241012793847982434" 16 | end 17 | end 18 | 19 | class ObjectRaisingEquality 20 | def ==(other) 21 | raise "Equality called on fetched object." 22 | end 23 | end 24 | 25 | class MyToParamIsFrozen 26 | def to_param 27 | "frozen".freeze 28 | end 29 | end 30 | 31 | describe 'ActiveSupport::Cache::DalliStore' do 32 | # with and without local cache 33 | def self.it_with_and_without_local_cache(message, &block) 34 | it "#{message} with LocalCache" do 35 | with_cache do 36 | @dalli.with_local_cache do 37 | instance_eval(&block) 38 | end 39 | end 40 | end 41 | 42 | it "#{message} without LocalCache" do 43 | with_cache do 44 | instance_eval(&block) 45 | end 46 | end 47 | end 48 | 49 | describe 'active_support caching' do 50 | it 'has accessible options' do 51 | with_cache :expires_in => 5.minutes, :frob => 'baz' do 52 | assert_equal 'baz', @dalli.options[:frob] 53 | end 54 | 55 | with_cache :expires_in => 5.minutes, :digest_class => OpenSSL::Digest::SHA1 do 56 | assert_equal OpenSSL::Digest::SHA1, @dalli.options[:digest_class] 57 | end 58 | end 59 | 60 | it 'uses valid digest_class option' do 61 | with_cache :expires_in => 5.minutes, :digest_class => OpenSSL::Digest::SHA1 do 62 | dvalue = @dalli.fetch('a_key') { 123 } 63 | assert_equal 123, dvalue 64 | end 65 | end 66 | 67 | it_with_and_without_local_cache 'allow mute and silence' do 68 | @dalli.mute do 69 | assert op_addset_succeeds(@dalli.write('foo', 'bar', nil)) 70 | assert_equal 'bar', @dalli.read('foo', nil) 71 | end 72 | refute @dalli.silence? 73 | @dalli.silence! 74 | assert_equal true, @dalli.silence? 75 | end 76 | 77 | it_with_and_without_local_cache 'handle nil options' do 78 | assert op_addset_succeeds(@dalli.write('foo', 'bar', nil)) 79 | assert_equal 'bar', @dalli.read('foo', nil) 80 | assert_equal 18, @dalli.fetch('lkjsadlfk', nil) { 18 } 81 | assert_equal 18, @dalli.fetch('lkjsadlfk', nil) { 18 } 82 | assert_equal 1, @dalli.increment('lkjsa', 1, nil) 83 | assert_equal 2, @dalli.increment('lkjsa', 1, nil) 84 | assert_equal 1, @dalli.decrement('lkjsa', 1, nil) 85 | assert_equal true, @dalli.delete('lkjsa') 86 | end 87 | 88 | describe 'fetch' do 89 | it_with_and_without_local_cache 'support expires_in' do 90 | dvalue = @dalli.fetch('someotherkeywithoutspaces', :expires_in => 1.second) { 123 } 91 | assert_equal 123, dvalue 92 | end 93 | 94 | it_with_and_without_local_cache 'tests cache misses using correct operand ordering' do 95 | # Some objects customise their equality methods. If you call #== on these objects this can mean your 96 | # returned value from the gem to your application is technically different to what's serialised in the cache. 97 | # 98 | # See https://github.com/petergoldstein/dalli/pull/662 99 | # 100 | obj = ObjectRaisingEquality.new 101 | @dalli.fetch('obj') { obj } 102 | end 103 | 104 | it_with_and_without_local_cache 'fallback block gets a key as a parameter' do 105 | key = rand_key 106 | o = Object.new 107 | o.instance_variable_set :@foo, 'bar' 108 | dvalue = @dalli.fetch(key) { |k| "#{k}-#{o}" } 109 | assert_equal "#{key}-#{o}", dvalue 110 | end 111 | 112 | it_with_and_without_local_cache 'support object' do 113 | o = Object.new 114 | o.instance_variable_set :@foo, 'bar' 115 | dvalue = @dalli.fetch(rand_key) { o } 116 | assert_equal o, dvalue 117 | end 118 | 119 | it_with_and_without_local_cache 'support object with raw' do 120 | o = Object.new 121 | o.instance_variable_set :@foo, 'bar' 122 | dvalue = @dalli.fetch(rand_key, :raw => true) { o } 123 | assert_equal o, dvalue 124 | end 125 | 126 | it_with_and_without_local_cache 'support false value' do 127 | @dalli.write('false', false) 128 | dvalue = @dalli.fetch('false') { flunk } 129 | assert_equal false, dvalue 130 | end 131 | 132 | it 'support nil value when cache_nils: true' do 133 | with_cache cache_nils: true do 134 | @dalli.write('nil', nil) 135 | dvalue = @dalli.fetch('nil') { flunk } 136 | assert_nil dvalue 137 | end 138 | 139 | with_cache cache_nils: false do 140 | @dalli.write('nil', nil) 141 | executed = false 142 | dvalue = @dalli.fetch('nil') { executed = true; 'bar' } 143 | assert_equal true, executed 144 | assert_equal 'bar', dvalue 145 | end 146 | end 147 | 148 | it_with_and_without_local_cache 'support object with cache_key' do 149 | user = MockUser.new 150 | @dalli.write(user.cache_key, false) 151 | dvalue = @dalli.fetch(user) { flunk } 152 | assert_equal false, dvalue 153 | end 154 | 155 | it_with_and_without_local_cache 'support object with cache_key_with_version' do 156 | user = MockUserVersioning.new 157 | @dalli.write(user.cache_key_with_version, false) 158 | dvalue = @dalli.fetch(user) { flunk } 159 | assert_equal false, dvalue 160 | end 161 | end 162 | 163 | it_with_and_without_local_cache 'support keys with spaces' do 164 | dvalue = @dalli.fetch('some key with spaces', :expires_in => 1.second) { 123 } 165 | assert_equal 123, dvalue 166 | end 167 | 168 | it_with_and_without_local_cache 'support read_multi' do 169 | x = rand_key 170 | y = rand_key 171 | assert_equal({}, @dalli.read_multi(x, y)) 172 | @dalli.write(x, '123') 173 | @dalli.write(y, 123) 174 | assert_equal({ x => '123', y => 123 }, @dalli.read_multi(x, y)) 175 | end 176 | 177 | it_with_and_without_local_cache 'support read_multi with an array' do 178 | x = rand_key 179 | y = rand_key 180 | assert_equal({}, @dalli.read_multi([x, y])) 181 | @dalli.write(x, '123') 182 | @dalli.write(y, 123) 183 | assert_equal({}, @dalli.read_multi([x, y])) 184 | @dalli.write([x, y], '123') 185 | assert_equal({ [x, y] => '123' }, @dalli.read_multi([x, y])) 186 | end 187 | 188 | it_with_and_without_local_cache 'support read_multi with an empty array' do 189 | assert_equal({}, @dalli.read_multi([])) 190 | end 191 | 192 | it 'support raw read_multi' do # TODO fails with LocalCache 193 | with_cache do 194 | @dalli.write("abc", 5, :raw => true) 195 | @dalli.write("cba", 5, :raw => true) 196 | assert_equal({'abc' => '5', 'cba' => '5' }, @dalli.read_multi("abc", "cba")) 197 | end 198 | end 199 | 200 | it 'support read_multi with LocalCache' do 201 | with_cache do 202 | x = rand_key 203 | y = rand_key 204 | assert_equal({}, @dalli.read_multi(x, y)) 205 | @dalli.write(x, '123') 206 | @dalli.write(y, 456) 207 | 208 | @dalli.with_local_cache do 209 | assert_equal({ x => '123', y => 456 }, @dalli.read_multi(x, y)) 210 | Dalli::Client.any_instance.expects(:get).with(any_parameters).never 211 | 212 | dres = @dalli.read(x) 213 | assert_equal dres, '123' 214 | end 215 | 216 | Dalli::Client.any_instance.unstub(:get) 217 | 218 | # Fresh LocalStore 219 | @dalli.with_local_cache do 220 | @dalli.read(x) 221 | Dalli::Client.any_instance.expects(:get_multi).with([y.to_s]).returns(y.to_s => 456) 222 | 223 | assert_equal({ x => '123', y => 456}, @dalli.read_multi(x, y)) 224 | end 225 | end 226 | end 227 | 228 | it 'supports frozen strings' do 229 | with_cache do 230 | @dalli.read(["foo".freeze]) 231 | end 232 | end 233 | 234 | it 'supports frozen strings in more contrived scenarios' do 235 | with_cache do 236 | @dalli.read(MyToParamIsFrozen.new) 237 | end 238 | end 239 | 240 | it 'support read_multi with special Regexp characters in namespace' do 241 | # /(?!)/ is a contradictory PCRE and should never be able to match 242 | with_cache :namespace => '(?!)' do 243 | @dalli.write('abc', 123) 244 | @dalli.write('xyz', 456) 245 | 246 | assert_equal({'abc' => 123, 'xyz' => 456}, @dalli.read_multi('abc', 'xyz')) 247 | end 248 | end 249 | 250 | it_with_and_without_local_cache 'supports fetch_multi' do 251 | x = rand_key.to_s 252 | y = rand_key 253 | hash = { x => 'ABC', y => 'DEF' } 254 | 255 | @dalli.write(y, '123') 256 | 257 | results = @dalli.fetch_multi(x, y) { |key| hash[key] } 258 | 259 | assert_equal({ x => 'ABC', y => '123' }, results) 260 | assert_equal('ABC', @dalli.read(x)) 261 | assert_equal('123', @dalli.read(y)) 262 | end 263 | 264 | it_with_and_without_local_cache 'supports fetch_multi with large cache keys' do 265 | x = "x" * 512 266 | y = "y" * 512 267 | hash = { x => 'ABC', y => 'DEF' } 268 | 269 | @dalli.write(y, '123') 270 | 271 | results = @dalli.fetch_multi(x, y) { |key| hash[key] } 272 | 273 | assert_equal({ x => 'ABC', y => '123' }, results) 274 | assert_equal('ABC', @dalli.read(x)) 275 | assert_equal('123', @dalli.read(y)) 276 | end 277 | 278 | it_with_and_without_local_cache 'support read, write and delete' do 279 | y = rand_key 280 | assert_nil @dalli.read(y) 281 | dres = @dalli.write(y, 123) 282 | assert op_addset_succeeds(dres) 283 | 284 | dres = @dalli.read(y) 285 | assert_equal 123, dres 286 | 287 | dres = @dalli.delete(y) 288 | assert_equal true, dres 289 | 290 | user = MockUser.new 291 | dres = @dalli.write(user.cache_key, "foo") 292 | assert op_addset_succeeds(dres) 293 | 294 | dres = @dalli.read(user) 295 | assert_equal "foo", dres 296 | 297 | dres = @dalli.delete(user) 298 | assert_equal true, dres 299 | 300 | dres = @dalli.write(:false_value, false) 301 | assert op_addset_succeeds(dres) 302 | dres = @dalli.read(:false_value) 303 | assert_equal false, dres 304 | 305 | bigkey = '１２３４５６７８９０１２３４５６７８９０１２３４５６７８９０' 306 | @dalli.write(bigkey, 'double width') 307 | assert_equal 'double width', @dalli.read(bigkey) 308 | assert_equal({bigkey => "double width"}, @dalli.read_multi(bigkey)) 309 | end 310 | 311 | it_with_and_without_local_cache 'support read, write and delete with local namespace' do 312 | key = 'key_with_namespace' 313 | namespace_value = @dalli.fetch(key, :namespace => 'namespace') { 123 } 314 | assert_equal 123, namespace_value 315 | 316 | res = @dalli.read(key, :namespace => 'namespace') 317 | assert_equal 123, res 318 | 319 | res = @dalli.delete(key, :namespace => 'namespace') 320 | assert_equal true, res 321 | 322 | res = @dalli.write(key, "foo", :namespace => 'namespace') 323 | assert op_addset_succeeds(res) 324 | 325 | res = @dalli.read(key, :namespace => 'namespace') 326 | assert_equal "foo", res 327 | end 328 | 329 | it_with_and_without_local_cache 'support multi_read and multi_fetch with local namespace' do 330 | x = rand_key.to_s 331 | y = rand_key 332 | namespace = 'namespace' 333 | hash = { x => 'ABC', y => 'DEF' } 334 | 335 | results = @dalli.fetch_multi(x, y, :namespace => namespace) { |key| hash[key] } 336 | 337 | assert_equal({ x => 'ABC', y => 'DEF' }, results) 338 | assert_equal('ABC', @dalli.read(x, :namespace => namespace)) 339 | assert_equal('DEF', @dalli.read(y, :namespace => namespace)) 340 | 341 | @dalli.write("abc", 5, :namespace => 'namespace') 342 | @dalli.write("cba", 5, :namespace => 'namespace') 343 | assert_equal({'abc' => 5, 'cba' => 5 }, @dalli.read_multi("abc", "cba", :namespace => 'namespace')) 344 | end 345 | 346 | it 'support read, write and delete with LocalCache' do 347 | with_cache do 348 | y = rand_key.to_s 349 | @dalli.with_local_cache do 350 | Dalli::Client.any_instance.expects(:get).with(y, {}).once.returns(123) 351 | dres = @dalli.read(y) 352 | assert_equal 123, dres 353 | 354 | Dalli::Client.any_instance.expects(:get).with(y, {}).never 355 | 356 | dres = @dalli.read(y) 357 | assert_equal 123, dres 358 | 359 | @dalli.write(y, 456) 360 | dres = @dalli.read(y) 361 | assert_equal 456, dres 362 | 363 | @dalli.delete(y) 364 | Dalli::Client.any_instance.expects(:get).with(y, {}).once.returns(nil) 365 | dres = @dalli.read(y) 366 | assert_nil dres 367 | end 368 | end 369 | end 370 | 371 | it_with_and_without_local_cache 'support unless_exist' do 372 | y = rand_key.to_s 373 | @dalli.with_local_cache do 374 | Dalli::Client.any_instance.expects(:add).with(y, 123, nil, {:unless_exist => true}).once.returns(true) 375 | dres = @dalli.write(y, 123, :unless_exist => true) 376 | assert_equal true, dres 377 | 378 | Dalli::Client.any_instance.expects(:add).with(y, 321, nil, {:unless_exist => true}).once.returns(false) 379 | 380 | dres = @dalli.write(y, 321, :unless_exist => true) 381 | assert_equal false, dres 382 | 383 | Dalli::Client.any_instance.expects(:get).with(y, {}).once.returns(123) 384 | 385 | dres = @dalli.read(y) 386 | assert_equal 123, dres 387 | end 388 | end 389 | 390 | it_with_and_without_local_cache 'support increment/decrement commands' do 391 | assert op_addset_succeeds(@dalli.write('counter', 0, :raw => true)) 392 | assert_equal 1, @dalli.increment('counter') 393 | assert_equal 2, @dalli.increment('counter') 394 | assert_equal 1, @dalli.decrement('counter') 395 | assert_equal "1", @dalli.read('counter', :raw => true) 396 | 397 | assert_equal 1, @dalli.increment('counterX') 398 | assert_equal 2, @dalli.increment('counterX') 399 | assert_equal 2, @dalli.read('counterX', :raw => true).to_i 400 | 401 | assert_equal 5, @dalli.increment('counterY1', 1, :initial => 5) 402 | assert_equal 6, @dalli.increment('counterY1', 1, :initial => 5) 403 | assert_equal 6, @dalli.read('counterY1', :raw => true).to_i 404 | 405 | assert_nil @dalli.increment('counterZ1', 1, :initial => nil) 406 | assert_nil @dalli.read('counterZ1') 407 | 408 | assert_equal 5, @dalli.decrement('counterY2', 1, :initial => 5) 409 | assert_equal 4, @dalli.decrement('counterY2', 1, :initial => 5) 410 | assert_equal 4, @dalli.read('counterY2', :raw => true).to_i 411 | 412 | assert_nil @dalli.decrement('counterZ2', 1, :initial => nil) 413 | assert_nil @dalli.read('counterZ2') 414 | 415 | user = MockUser.new 416 | assert op_addset_succeeds(@dalli.write(user, 0, :raw => true)) 417 | assert_equal 1, @dalli.increment(user) 418 | assert_equal 2, @dalli.increment(user) 419 | assert_equal 1, @dalli.decrement(user) 420 | assert_equal "1", @dalli.read(user, :raw => true) 421 | end 422 | 423 | it_with_and_without_local_cache 'support exist command' do 424 | @dalli.write(:foo, 'a') 425 | @dalli.write(:false_value, false) 426 | 427 | assert_equal true, @dalli.exist?(:foo) 428 | assert_equal true, @dalli.exist?(:false_value) 429 | 430 | assert_equal false, @dalli.exist?(:bar) 431 | 432 | user = MockUser.new 433 | @dalli.write(user, 'foo') 434 | assert_equal true, @dalli.exist?(user) 435 | end 436 | 437 | it_with_and_without_local_cache 'support other esoteric commands' do 438 | ds = @dalli.stats 439 | assert_equal 1, ds.keys.size 440 | assert ds[ds.keys.first].keys.size > 0 441 | 442 | @dalli.reset 443 | end 444 | 445 | it 'respects "raise_errors" option' do 446 | new_port = 29333 447 | with_cache port: new_port do 448 | @dalli.write 'foo', 'bar' 449 | assert_equal @dalli.read('foo'), 'bar' 450 | 451 | memcached_kill(new_port) 452 | 453 | silence_logger do 454 | assert_nil @dalli.read('foo') 455 | end 456 | end 457 | 458 | with_cache port: new_port, :raise_errors => true do 459 | memcached_kill(new_port) 460 | exception = [Dalli::RingError, { :message => "No server available" }] 461 | 462 | silence_logger do 463 | assert_raises(*exception) { @dalli.read 'foo' } 464 | assert_raises(*exception) { @dalli.read 'foo', :raw => true } 465 | assert_raises(*exception) { @dalli.write 'foo', 'bar' } 466 | assert_raises(*exception) { @dalli.exist? 'foo' } 467 | assert_raises(*exception) { @dalli.increment 'foo' } 468 | assert_raises(*exception) { @dalli.decrement 'foo' } 469 | assert_raises(*exception) { @dalli.delete 'foo' } 470 | assert_equal @dalli.read_multi('foo', 'bar'), {} 471 | assert_raises(*exception) { @dalli.delete 'foo' } 472 | assert_raises(*exception) { @dalli.fetch('foo') { 42 } } 473 | end 474 | end 475 | end 476 | 477 | describe 'instruments' do 478 | it 'notifies errors' do 479 | new_port = 29333 480 | key = 'foo' 481 | with_cache port: new_port, :instrument_errors => true do 482 | memcached_kill(new_port) 483 | payload_proc = Proc.new { |payload| payload } 484 | @dalli.expects(:instrument).with(:read, { :key => key }).yields(&payload_proc).once 485 | @dalli.expects(:instrument).with(:error, { :key => "DalliError", 486 | :message => "No server available" }).once 487 | @dalli.read(key) 488 | end 489 | end 490 | 491 | it 'payload hits' do 492 | with_cache do 493 | payload = {} 494 | assert op_addset_succeeds(@dalli.write('false', false)) 495 | foo = @dalli.fetch('burrito') { 'tacos' } 496 | assert 'tacos', foo 497 | 498 | # NOTE: mocha stubbing for yields 499 | # makes the result of the block nil in all cases 500 | # there was a ticket about this: 501 | # http://floehopper.lighthouseapp.com/projects/22289/tickets/14-8687-blocks-return-value-is-dropped-on-stubbed-yielding-methods 502 | @dalli.stubs(:instrument).yields payload 503 | 504 | @dalli.read('false') 505 | assert_equal true, payload.delete(:hit) 506 | 507 | 508 | @dalli.fetch('unset_key') { 'tacos' } 509 | assert_equal false, payload.delete(:hit) 510 | 511 | @dalli.fetch('burrito') { 'tacos' } 512 | assert_equal true, payload.delete(:hit) 513 | 514 | @dalli.unstub(:instrument) 515 | end 516 | end 517 | end 518 | end 519 | 520 | it_with_and_without_local_cache 'handle crazy characters from far-away lands' do 521 | key = "fooƒ" 522 | value = 'bafƒ' 523 | assert op_addset_succeeds(@dalli.write(key, value)) 524 | assert_equal value, @dalli.read(key) 525 | end 526 | 527 | it 'normalize options as expected' do 528 | with_cache :expires_in => 1, :namespace => 'foo', :compress => true do 529 | assert_equal 1, @dalli.instance_variable_get(:@data).instance_variable_get(:@options)[:expires_in] 530 | assert_equal 'foo', @dalli.instance_variable_get(:@data).instance_variable_get(:@options)[:namespace] 531 | assert_equal ["localhost:19987"], @dalli.instance_variable_get(:@data).instance_variable_get(:@servers) 532 | end 533 | end 534 | 535 | it 'handles nil server with additional options' do 536 | @dalli = ActiveSupport::Cache::DalliStore.new(nil, :expires_in => 1, :namespace => 'foo', :compress => true) 537 | assert_equal 1, @dalli.instance_variable_get(:@data).instance_variable_get(:@options)[:expires_in] 538 | assert_equal 'foo', @dalli.instance_variable_get(:@data).instance_variable_get(:@options)[:namespace] 539 | assert_equal ["127.0.0.1:11211"], @dalli.instance_variable_get(:@data).instance_variable_get(:@servers) 540 | end 541 | 542 | it 'supports connection pooling' do 543 | with_cache :expires_in => 1, :namespace => 'foo', :compress => true, :pool_size => 3 do 544 | assert_nil @dalli.read('foo') 545 | assert @dalli.write('foo', 1) 546 | assert_equal 1, @dalli.fetch('foo') { raise 'boom' } 547 | assert_equal true, @dalli.dalli.is_a?(ConnectionPool) 548 | assert_equal 1, @dalli.increment('bar') 549 | assert_equal 0, @dalli.decrement('bar') 550 | assert_equal true, @dalli.delete('bar') 551 | assert_equal [true], @dalli.clear 552 | assert_equal 1, @dalli.stats.size 553 | end 554 | end 555 | 556 | it_with_and_without_local_cache 'allow keys to be frozen' do 557 | key = "foo" 558 | key.freeze 559 | assert op_addset_succeeds(@dalli.write(key, "value")) 560 | end 561 | 562 | it_with_and_without_local_cache 'allow keys from a hash' do 563 | map = { "one" => "one", "two" => "two" } 564 | map.each_pair do |k, v| 565 | assert op_addset_succeeds(@dalli.write(k, v)) 566 | end 567 | assert_equal map, @dalli.read_multi(*(map.keys)) 568 | end 569 | 570 | def silence_logger 571 | old = Dalli.logger.level 572 | Dalli.logger.level = Logger::ERROR + 1 573 | yield 574 | ensure 575 | Dalli.logger.level = old 576 | end 577 | 578 | def with_cache(options={}) 579 | port = options.delete(:port) || 19987 580 | memcached_persistent(port) do 581 | @dalli = ActiveSupport::Cache.lookup_store(:dalli_store, "localhost:#{port}", options) 582 | @dalli.clear 583 | yield 584 | end 585 | end 586 | 587 | def rand_key 588 | rand(1_000_000_000) 589 | end 590 | end 591 | -------------------------------------------------------------------------------- /test/test_dalli.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require_relative 'helper' 3 | require 'openssl' 4 | 5 | describe 'Dalli' do 6 | describe 'options parsing' do 7 | it 'handle deprecated options' do 8 | dc = Dalli::Client.new('foo', :compression => true) 9 | assert dc.instance_variable_get(:@options)[:compress] 10 | refute dc.instance_variable_get(:@options)[:compression] 11 | end 12 | 13 | it 'not warn about valid options' do 14 | dc = Dalli::Client.new('foo', :compress => true) 15 | # Rails.logger.expects :warn 16 | assert dc.instance_variable_get(:@options)[:compress] 17 | end 18 | 19 | it 'raises error with invalid expires_in' do 20 | bad_data = [{:bad => 'expires in data'}, Hash, [1,2,3]] 21 | bad_data.each do |bad| 22 | assert_raises ArgumentError do 23 | Dalli::Client.new('foo', {:expires_in => bad}) 24 | end 25 | end 26 | end 27 | 28 | it 'return string type for namespace attribute' do 29 | dc = Dalli::Client.new('foo', :namespace => :wunderschoen) 30 | assert_equal "wunderschoen", dc.send(:namespace) 31 | dc.close 32 | 33 | dc = Dalli::Client.new('foo', :namespace => Proc.new{:wunderschoen}) 34 | assert_equal "wunderschoen", dc.send(:namespace) 35 | dc.close 36 | end 37 | 38 | it 'raises error with invalid digest_class' do 39 | assert_raises ArgumentError do 40 | Dalli::Client.new('foo', {:expires_in => 10, :digest_class => Object }) 41 | end 42 | end 43 | end 44 | 45 | describe 'key validation' do 46 | it 'not allow blanks' do 47 | memcached_persistent do |dc| 48 | dc.set ' ', 1 49 | assert_equal 1, dc.get(' ') 50 | dc.set "\t", 1 51 | assert_equal 1, dc.get("\t") 52 | dc.set "\n", 1 53 | assert_equal 1, dc.get("\n") 54 | assert_raises ArgumentError do 55 | dc.set "", 1 56 | end 57 | assert_raises ArgumentError do 58 | dc.set nil, 1 59 | end 60 | end 61 | end 62 | 63 | it 'allow namespace to be a symbol' do 64 | memcached_persistent do |dc, port| 65 | dc = Dalli::Client.new("localhost:#{port}", :namespace => :wunderschoen) 66 | dc.set "x" * 251, 1 67 | assert 1, dc.get("#{'x' * 200}:md5:#{Digest::MD5.hexdigest('x' * 251)}") 68 | end 69 | end 70 | end 71 | 72 | describe 'ttl validation' do 73 | it 'generated an ArgumentError for ttl that does not support to_i' do 74 | memcached_persistent do |dc| 75 | assert_raises ArgumentError do 76 | dc.set('foo', 'bar', []) 77 | end 78 | end 79 | end 80 | end 81 | 82 | it "default to localhost:11211" do 83 | dc = Dalli::Client.new 84 | ring = dc.send(:ring) 85 | s1 = ring.servers.first.hostname 86 | assert_equal 1, ring.servers.size 87 | dc.close 88 | 89 | dc = Dalli::Client.new('localhost:11211') 90 | ring = dc.send(:ring) 91 | s2 = ring.servers.first.hostname 92 | assert_equal 1, ring.servers.size 93 | dc.close 94 | 95 | dc = Dalli::Client.new(['localhost:11211']) 96 | ring = dc.send(:ring) 97 | s3 = ring.servers.first.hostname 98 | assert_equal 1, ring.servers.size 99 | dc.close 100 | 101 | assert_equal '127.0.0.1', s1 102 | assert_equal s2, s3 103 | end 104 | 105 | it "accept comma separated string" do 106 | dc = Dalli::Client.new("server1.example.com:11211,server2.example.com:11211") 107 | ring = dc.send(:ring) 108 | assert_equal 2, ring.servers.size 109 | s1,s2 = ring.servers.map(&:hostname) 110 | assert_equal "server1.example.com", s1 111 | assert_equal "server2.example.com", s2 112 | end 113 | 114 | it "accept array of servers" do 115 | dc = Dalli::Client.new(["server1.example.com:11211","server2.example.com:11211"]) 116 | ring = dc.send(:ring) 117 | assert_equal 2, ring.servers.size 118 | s1,s2 = ring.servers.map(&:hostname) 119 | assert_equal "server1.example.com", s1 120 | assert_equal "server2.example.com", s2 121 | end 122 | 123 | describe 'using a live server' do 124 | 125 | it "support get/set" do 126 | memcached_persistent do |dc| 127 | dc.flush 128 | 129 | val1 = "1234567890"*105000 130 | with_nil_logger do 131 | assert_equal false, dc.set('a', val1) 132 | end 133 | 134 | val1 = "1234567890"*100000 135 | dc.set('a', val1) 136 | val2 = dc.get('a') 137 | assert_equal val1, val2 138 | 139 | assert op_addset_succeeds(dc.set('a', nil)) 140 | assert_nil dc.get('a') 141 | end 142 | end 143 | 144 | it 'supports delete' do 145 | memcached_persistent do |dc| 146 | dc.set('some_key', 'some_value') 147 | assert_equal 'some_value', dc.get('some_key') 148 | 149 | dc.delete('some_key') 150 | assert_nil dc.get('some_key') 151 | end 152 | end 153 | 154 | it 'returns nil for nonexist key' do 155 | memcached_persistent do |dc| 156 | assert_nil dc.get('notexist') 157 | end 158 | end 159 | 160 | it 'allows "Not found" as value' do 161 | memcached_persistent do |dc| 162 | dc.set('key1', 'Not found') 163 | assert_equal 'Not found', dc.get('key1') 164 | end 165 | end 166 | 167 | it "support stats" do 168 | memcached_persistent do |dc| 169 | # make sure that get_hits would not equal 0 170 | dc.set(:a, "1234567890"*100000) 171 | dc.get(:a) 172 | 173 | stats = dc.stats 174 | servers = stats.keys 175 | assert(servers.any? do |s| 176 | stats[s]["get_hits"].to_i != 0 177 | end, "general stats failed") 178 | 179 | stats_items = dc.stats(:items) 180 | servers = stats_items.keys 181 | assert(servers.all? do |s| 182 | stats_items[s].keys.any? do |key| 183 | key =~ /items:[0-9]+:number/ 184 | end 185 | end, "stats items failed") 186 | 187 | stats_slabs = dc.stats(:slabs) 188 | servers = stats_slabs.keys 189 | assert(servers.all? do |s| 190 | stats_slabs[s].keys.any? do |key| 191 | key == "active_slabs" 192 | end 193 | end, "stats slabs failed") 194 | 195 | # reset_stats test 196 | results = dc.reset_stats 197 | assert(results.all? { |x| x }) 198 | stats = dc.stats 199 | servers = stats.keys 200 | 201 | # check if reset was performed 202 | servers.each do |s| 203 | assert_equal 0, dc.stats[s]["get_hits"].to_i 204 | end 205 | end 206 | end 207 | 208 | it "support the fetch operation" do 209 | memcached_persistent do |dc| 210 | dc.flush 211 | 212 | expected = { 'blah' => 'blerg!' } 213 | executed = false 214 | value = dc.fetch('fetch_key') do 215 | executed = true 216 | expected 217 | end 218 | assert_equal expected, value 219 | assert_equal true, executed 220 | 221 | executed = false 222 | value = dc.fetch('fetch_key') do 223 | executed = true 224 | expected 225 | end 226 | assert_equal expected, value 227 | assert_equal false, executed 228 | end 229 | end 230 | 231 | it "support the fetch operation with falsey values" do 232 | memcached_persistent do |dc| 233 | dc.flush 234 | 235 | dc.set("fetch_key", false) 236 | res = dc.fetch("fetch_key") { flunk "fetch block called" } 237 | assert_equal false, res 238 | end 239 | end 240 | 241 | it "support the fetch operation with nil values when cache_nils: true" do 242 | memcached_persistent(21345, cache_nils: true) do |dc| 243 | dc.flush 244 | 245 | dc.set("fetch_key", nil) 246 | res = dc.fetch("fetch_key") { flunk "fetch block called" } 247 | assert_nil res 248 | end 249 | 250 | memcached_persistent(21345, cache_nils: false) do |dc| 251 | dc.flush 252 | dc.set("fetch_key", nil) 253 | executed = false 254 | res = dc.fetch("fetch_key") { executed = true; 'bar' } 255 | assert_equal 'bar', res 256 | assert_equal true, executed 257 | end 258 | end 259 | 260 | it "support the cas operation" do 261 | memcached_persistent do |dc| 262 | dc.flush 263 | 264 | expected = { 'blah' => 'blerg!' } 265 | 266 | resp = dc.cas('cas_key') do |value| 267 | fail('Value it not exist') 268 | end 269 | assert_nil resp 270 | 271 | mutated = { 'blah' => 'foo!' } 272 | dc.set('cas_key', expected) 273 | resp = dc.cas('cas_key') do |value| 274 | assert_equal expected, value 275 | mutated 276 | end 277 | assert op_cas_succeeds(resp) 278 | 279 | resp = dc.get('cas_key') 280 | assert_equal mutated, resp 281 | end 282 | end 283 | 284 | it "support the cas! operation" do 285 | memcached_persistent do |dc| 286 | dc.flush 287 | 288 | mutated = { 'blah' => 'foo!' } 289 | resp = dc.cas!('cas_key') do |value| 290 | assert_nil value 291 | mutated 292 | end 293 | assert op_cas_succeeds(resp) 294 | 295 | resp = dc.get('cas_key') 296 | assert_equal mutated, resp 297 | end 298 | end 299 | 300 | it "support multi-get" do 301 | memcached_persistent do |dc| 302 | dc.close 303 | dc.flush 304 | resp = dc.get_multi(%w(a b c d e f)) 305 | assert_equal({}, resp) 306 | 307 | dc.set('a', 'foo') 308 | dc.set('b', 123) 309 | dc.set('c', %w(a b c)) 310 | # Invocation without block 311 | resp = dc.get_multi(%w(a b c d e f)) 312 | expected_resp = { 'a' => 'foo', 'b' => 123, 'c' => %w(a b c) } 313 | assert_equal(expected_resp, resp) 314 | 315 | # Invocation with block 316 | dc.get_multi(%w(a b c d e f)) do |k, v| 317 | assert(expected_resp.has_key?(k) && expected_resp[k] == v) 318 | expected_resp.delete(k) 319 | end 320 | assert expected_resp.empty? 321 | 322 | # Perform a big multi-get with 1000 elements. 323 | arr = [] 324 | dc.multi do 325 | 1000.times do |idx| 326 | dc.set idx, idx 327 | arr << idx 328 | end 329 | end 330 | 331 | result = dc.get_multi(arr) 332 | assert_equal(1000, result.size) 333 | assert_equal(50, result['50']) 334 | end 335 | end 336 | 337 | it 'support raw incr/decr' do 338 | memcached_persistent do |client| 339 | client.flush 340 | 341 | assert op_addset_succeeds(client.set('fakecounter', 0, 0, :raw => true)) 342 | assert_equal 1, client.incr('fakecounter', 1) 343 | assert_equal 2, client.incr('fakecounter', 1) 344 | assert_equal 3, client.incr('fakecounter', 1) 345 | assert_equal 1, client.decr('fakecounter', 2) 346 | assert_equal "1", client.get('fakecounter', :raw => true) 347 | 348 | resp = client.incr('mycounter', 0) 349 | assert_nil resp 350 | 351 | resp = client.incr('mycounter', 1, 0, 2) 352 | assert_equal 2, resp 353 | resp = client.incr('mycounter', 1) 354 | assert_equal 3, resp 355 | 356 | resp = client.set('rawcounter', 10, 0, :raw => true) 357 | assert op_cas_succeeds(resp) 358 | 359 | resp = client.get('rawcounter', :raw => true) 360 | assert_equal '10', resp 361 | 362 | resp = client.incr('rawcounter', 1) 363 | assert_equal 11, resp 364 | end 365 | end 366 | 367 | it "support incr/decr operations" do 368 | memcached_persistent do |dc| 369 | dc.flush 370 | 371 | resp = dc.decr('counter', 100, 5, 0) 372 | assert_equal 0, resp 373 | 374 | resp = dc.decr('counter', 10) 375 | assert_equal 0, resp 376 | 377 | resp = dc.incr('counter', 10) 378 | assert_equal 10, resp 379 | 380 | current = 10 381 | 100.times do |x| 382 | resp = dc.incr('counter', 10) 383 | assert_equal current + ((x+1)*10), resp 384 | end 385 | 386 | resp = dc.decr('10billion', 0, 5, 10) 387 | # go over the 32-bit mark to verify proper (un)packing 388 | resp = dc.incr('10billion', 10_000_000_000) 389 | assert_equal 10_000_000_010, resp 390 | 391 | resp = dc.decr('10billion', 1) 392 | assert_equal 10_000_000_009, resp 393 | 394 | resp = dc.decr('10billion', 0) 395 | assert_equal 10_000_000_009, resp 396 | 397 | resp = dc.incr('10billion', 0) 398 | assert_equal 10_000_000_009, resp 399 | 400 | assert_nil dc.incr('DNE', 10) 401 | assert_nil dc.decr('DNE', 10) 402 | 403 | resp = dc.incr('big', 100, 5, 0xFFFFFFFFFFFFFFFE) 404 | assert_equal 0xFFFFFFFFFFFFFFFE, resp 405 | resp = dc.incr('big', 1) 406 | assert_equal 0xFFFFFFFFFFFFFFFF, resp 407 | 408 | # rollover the 64-bit value, we'll get something undefined. 409 | resp = dc.incr('big', 1) 410 | refute_equal 0x10000000000000000, resp 411 | dc.reset 412 | end 413 | end 414 | 415 | it 'support the append and prepend operations' do 416 | memcached_persistent do |dc| 417 | dc.flush 418 | assert op_addset_succeeds(dc.set('456', 'xyz', 0, :raw => true)) 419 | assert_equal true, dc.prepend('456', '0') 420 | assert_equal true, dc.append('456', '9') 421 | assert_equal '0xyz9', dc.get('456', :raw => true) 422 | assert_equal '0xyz9', dc.get('456') 423 | 424 | assert_equal false, dc.append('nonexist', 'abc') 425 | assert_equal false, dc.prepend('nonexist', 'abc') 426 | end 427 | end 428 | 429 | it 'supports replace operation' do 430 | memcached_persistent do |dc| 431 | dc.flush 432 | dc.set('key', 'value') 433 | assert op_replace_succeeds(dc.replace('key', 'value2')) 434 | 435 | assert_equal 'value2', dc.get('key') 436 | end 437 | end 438 | 439 | it 'support touch operation' do 440 | memcached_persistent do |dc| 441 | begin 442 | dc.flush 443 | dc.set 'key', 'value' 444 | assert_equal true, dc.touch('key', 10) 445 | assert_equal true, dc.touch('key') 446 | assert_equal 'value', dc.get('key') 447 | assert_nil dc.touch('notexist') 448 | rescue Dalli::DalliError => e 449 | # This will happen when memcached is in lesser version than 1.4.8 450 | assert_equal 'Response error 129: Unknown command', e.message 451 | end 452 | end 453 | end 454 | 455 | it 'support version operation' do 456 | memcached_persistent do |dc| 457 | v = dc.version 458 | servers = v.keys 459 | assert(servers.any? do |s| 460 | v[s] != nil 461 | end, "version failed") 462 | end 463 | end 464 | 465 | it 'allow TCP connections to be configured for keepalive' do 466 | memcached_persistent do |dc, port| 467 | dc = Dalli::Client.new("localhost:#{port}", :keepalive => true) 468 | dc.set(:a, 1) 469 | ring = dc.send(:ring) 470 | server = ring.servers.first 471 | socket = server.instance_variable_get('@sock') 472 | 473 | optval = socket.getsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE) 474 | optval = optval.unpack 'i' 475 | 476 | assert_equal true, (optval[0] != 0) 477 | end 478 | end 479 | 480 | it 'allow TCP connections to configure SO_RCVBUF' do 481 | memcached_persistent do |dc, port| 482 | value = 5000 483 | dc = Dalli::Client.new("localhost:#{port}", :rcvbuf => value) 484 | dc.set(:a, 1) 485 | ring = dc.send(:ring) 486 | server = ring.servers.first 487 | socket = server.instance_variable_get('@sock') 488 | 489 | optval = socket.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF) 490 | expected = jruby? ? value : value * 2 491 | assert_equal expected, optval.unpack('i')[0] 492 | end 493 | end 494 | 495 | it 'allow TCP connections to configure SO_SNDBUF' do 496 | memcached_persistent do |dc, port| 497 | value = 5000 498 | dc = Dalli::Client.new("localhost:#{port}", :sndbuf => value) 499 | dc.set(:a, 1) 500 | ring = dc.send(:ring) 501 | server = ring.servers.first 502 | socket = server.instance_variable_get('@sock') 503 | 504 | optval = socket.getsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF) 505 | expected = jruby? ? value : value * 2 506 | assert_equal expected, optval.unpack('i')[0] 507 | end 508 | end 509 | 510 | it "pass a simple smoke test" do 511 | memcached_persistent do |dc, port| 512 | resp = dc.flush 513 | refute_nil resp 514 | assert_equal [true, true], resp 515 | 516 | assert op_addset_succeeds(dc.set(:foo, 'bar')) 517 | assert_equal 'bar', dc.get(:foo) 518 | 519 | resp = dc.get('123') 520 | assert_nil resp 521 | 522 | assert op_addset_succeeds(dc.set('123', 'xyz')) 523 | 524 | resp = dc.get('123') 525 | assert_equal 'xyz', resp 526 | 527 | assert op_addset_succeeds(dc.set('123', 'abc')) 528 | 529 | dc.prepend('123', '0') 530 | dc.append('123', '0') 531 | 532 | assert_raises Dalli::UnmarshalError do 533 | resp = dc.get('123') 534 | end 535 | 536 | dc.close 537 | dc = nil 538 | 539 | dc = Dalli::Client.new("localhost:#{port}", :digest_class => ::OpenSSL::Digest::SHA1) 540 | 541 | assert op_addset_succeeds(dc.set('456', 'xyz', 0, :raw => true)) 542 | 543 | resp = dc.prepend '456', '0' 544 | assert_equal true, resp 545 | 546 | resp = dc.append '456', '9' 547 | assert_equal true, resp 548 | 549 | resp = dc.get('456', :raw => true) 550 | assert_equal '0xyz9', resp 551 | 552 | assert op_addset_succeeds(dc.set('456', false)) 553 | 554 | resp = dc.get('456') 555 | assert_equal false, resp 556 | 557 | resp = dc.stats 558 | assert_equal Hash, resp.class 559 | 560 | dc.close 561 | end 562 | end 563 | 564 | it "pass a simple smoke test on unix socket" do 565 | memcached_persistent(MemcachedMock::UNIX_SOCKET_PATH) do |dc, path| 566 | resp = dc.flush 567 | refute_nil resp 568 | assert_equal [true], resp 569 | 570 | assert op_addset_succeeds(dc.set(:foo, 'bar')) 571 | assert_equal 'bar', dc.get(:foo) 572 | 573 | resp = dc.get('123') 574 | assert_nil resp 575 | 576 | assert op_addset_succeeds(dc.set('123', 'xyz')) 577 | 578 | resp = dc.get('123') 579 | assert_equal 'xyz', resp 580 | 581 | assert op_addset_succeeds(dc.set('123', 'abc')) 582 | 583 | dc.prepend('123', '0') 584 | dc.append('123', '0') 585 | 586 | assert_raises Dalli::UnmarshalError do 587 | resp = dc.get('123') 588 | end 589 | 590 | dc.close 591 | dc = nil 592 | 593 | dc = Dalli::Client.new(path) 594 | 595 | assert op_addset_succeeds(dc.set('456', 'xyz', 0, :raw => true)) 596 | 597 | resp = dc.prepend '456', '0' 598 | assert_equal true, resp 599 | 600 | resp = dc.append '456', '9' 601 | assert_equal true, resp 602 | 603 | resp = dc.get('456', :raw => true) 604 | assert_equal '0xyz9', resp 605 | 606 | assert op_addset_succeeds(dc.set('456', false)) 607 | 608 | resp = dc.get('456') 609 | assert_equal false, resp 610 | 611 | resp = dc.stats 612 | assert_equal Hash, resp.class 613 | 614 | dc.close 615 | end 616 | end 617 | 618 | it "support multithreaded access" do 619 | memcached_persistent do |cache| 620 | cache.flush 621 | workers = [] 622 | 623 | cache.set('f', 'zzz') 624 | assert op_cas_succeeds((cache.cas('f') do |value| 625 | value << 'z' 626 | end)) 627 | assert_equal 'zzzz', cache.get('f') 628 | 629 | # Have a bunch of threads perform a bunch of operations at the same time. 630 | # Verify the result of each operation to ensure the request and response 631 | # are not intermingled between threads. 632 | 10.times do 633 | workers << Thread.new do 634 | 100.times do 635 | cache.set('a', 9) 636 | cache.set('b', 11) 637 | inc = cache.incr('cat', 10, 0, 10) 638 | cache.set('f', 'zzz') 639 | res = cache.cas('f') do |value| 640 | value << 'z' 641 | end 642 | refute_nil res 643 | assert_equal false, cache.add('a', 11) 644 | assert_equal({ 'a' => 9, 'b' => 11 }, cache.get_multi(['a', 'b'])) 645 | inc = cache.incr('cat', 10) 646 | assert_equal 0, inc % 5 647 | cache.decr('cat', 5) 648 | assert_equal 11, cache.get('b') 649 | 650 | assert_equal %w(a b), cache.get_multi('a', 'b', 'c').keys.sort 651 | 652 | end 653 | end 654 | end 655 | 656 | workers.each { |w| w.join } 657 | cache.flush 658 | end 659 | end 660 | 661 | it "handle namespaced keys" do 662 | memcached_persistent do |dc, port| 663 | dc = Dalli::Client.new("localhost:#{port}", :namespace => 'a') 664 | dc.set('namespaced', 1) 665 | dc2 = Dalli::Client.new("localhost:#{port}", :namespace => 'b') 666 | dc2.set('namespaced', 2) 667 | assert_equal 1, dc.get('namespaced') 668 | assert_equal 2, dc2.get('namespaced') 669 | end 670 | end 671 | 672 | it "handle nil namespace" do 673 | memcached_persistent do |dc, port| 674 | dc = Dalli::Client.new("localhost:#{port}", :namespace => nil) 675 | assert_equal 'key', dc.send(:validate_key, 'key') 676 | end 677 | end 678 | 679 | it 'truncate cache keys that are too long' do 680 | memcached_persistent do |dc, port| 681 | dc = Dalli::Client.new("localhost:#{port}", :namespace => 'some:namspace') 682 | key = "this cache key is far too long so it must be hashed and truncated and stuff" * 10 683 | value = "some value" 684 | assert op_addset_succeeds(dc.set(key, value)) 685 | assert_equal value, dc.get(key) 686 | end 687 | end 688 | 689 | it "handle namespaced keys in multi_get" do 690 | memcached_persistent do |dc, port| 691 | dc = Dalli::Client.new("localhost:#{port}", :namespace => 'a') 692 | dc.set('a', 1) 693 | dc.set('b', 2) 694 | assert_equal({'a' => 1, 'b' => 2}, dc.get_multi('a', 'b')) 695 | end 696 | end 697 | 698 | it 'handle special Regexp characters in namespace with get_multi' do 699 | memcached_persistent do |dc, port| 700 | # /(?!)/ is a contradictory PCRE and should never be able to match 701 | dc = Dalli::Client.new("localhost:#{port}", :namespace => '(?!)') 702 | dc.set('a', 1) 703 | dc.set('b', 2) 704 | assert_equal({'a' => 1, 'b' => 2}, dc.get_multi('a', 'b')) 705 | end 706 | end 707 | 708 | it "handle application marshalling issues" do 709 | memcached_persistent do |dc| 710 | with_nil_logger do 711 | assert_equal false, dc.set('a', Proc.new { true }) 712 | end 713 | end 714 | end 715 | 716 | describe 'with compression' do 717 | it 'allow large values' do 718 | memcached_persistent do |dc| 719 | dalli = Dalli::Client.new(dc.instance_variable_get(:@servers), :compress => true) 720 | 721 | value = "0"*1024*1024 722 | with_nil_logger do 723 | assert_equal false, dc.set('verylarge', value) 724 | end 725 | dalli.set('verylarge', value) 726 | end 727 | end 728 | 729 | it 'allow large values to be set' do 730 | memcached_persistent do |dc| 731 | value = "0"*1024*1024 732 | assert dc.set('verylarge', value, nil, :compress => true) 733 | end 734 | end 735 | end 736 | 737 | describe 'in low memory conditions' do 738 | 739 | it 'handle error response correctly' do 740 | memcached_low_mem_persistent do |dc| 741 | failed = false 742 | value = "1234567890"*100 743 | 1_000.times do |idx| 744 | begin 745 | assert op_addset_succeeds(dc.set(idx, value)) 746 | rescue Dalli::DalliError 747 | failed = true 748 | assert((800..960).include?(idx), "unexpected failure on iteration #{idx}") 749 | break 750 | end 751 | end 752 | assert failed, 'did not fail under low memory conditions' 753 | end 754 | end 755 | 756 | it 'fit more values with compression' do 757 | memcached_low_mem_persistent do |dc, port| 758 | dalli = Dalli::Client.new("localhost:#{port}", :compress => true) 759 | failed = false 760 | value = "1234567890"*1000 761 | 10_000.times do |idx| 762 | begin 763 | assert op_addset_succeeds(dalli.set(idx, value)) 764 | rescue Dalli::DalliError 765 | failed = true 766 | assert((6000..7800).include?(idx), "unexpected failure on iteration #{idx}") 767 | break 768 | end 769 | end 770 | assert failed, 'did not fail under low memory conditions' 771 | end 772 | end 773 | 774 | end 775 | 776 | end 777 | end 778 | --------------------------------------------------------------------------------