├── .gitignore ├── Makefile ├── README.md └── lib └── resty └── memcached └── shdict.lua /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.swp 3 | *.swo 4 | *.bak 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | OPENRESTY_PREFIX=/usr/local/openresty 2 | 3 | #LUA_VERSION := 5.1 4 | PREFIX ?= /usr/local 5 | LUA_INCLUDE_DIR ?= $(PREFIX)/include 6 | LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION) 7 | INSTALL ?= install 8 | 9 | .PHONY: all test install 10 | 11 | all: ; 12 | 13 | install: all 14 | $(INSTALL) -d $(DESTDIR)$(LUA_LIB_DIR)/resty/memcached/ 15 | $(INSTALL) lib/resty/memcached/*.lua $(DESTDIR)$(LUA_LIB_DIR)/resty/memcached/ 16 | 17 | test: all 18 | PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r t 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Name 2 | ==== 3 | 4 | lua-resty-memcached-shdict - Powerful memcached client with a shdict caching layer and many other features 5 | 6 | Table of Contents 7 | ================= 8 | 9 | * [Name](#name) 10 | * [Synopsis](#synopsis) 11 | * [Description](#description) 12 | * [Caveats](#caveats) 13 | * [Dependencies](#dependencies) 14 | * [TODO](#todo) 15 | * [Author](#author) 16 | * [Copyright and License](#copyright-and-license) 17 | * [See Also](#see-also) 18 | 19 | Status 20 | ====== 21 | 22 | Experimental. 23 | 24 | Synopsis 25 | ======== 26 | 27 | ```lua 28 | local shdict_memc = require "resty.memcached.shdict" 29 | 30 | local function dlog(ctx, ...) 31 | ngx.log(ngx.DEBUG, "my app: ", ...) 32 | end 33 | 34 | local function error_log(ctx, ...) 35 | ngx.log(ngx.ERR, "my app: ", ...) 36 | end 37 | 38 | local function warn(ctx, ...) 39 | ngx.log(ngx.WARN, "my app: ", ...) 40 | end 41 | 42 | local memc_fetch, memc_store = 43 | shdict_memc.gen_memc_methods{ 44 | tag = "my memcached server tag", 45 | debug_logger = dlog, 46 | warn_logger = warn, 47 | error_logger = error_log, 48 | 49 | locks_shdict_name = "some_lua_shared_dict_name", 50 | 51 | shdict_set = meta_shdict_set, -- generated by lua-resty-shdict-simple or 52 | -- any other API compatible function factories 53 | 54 | shdict_get = meta_shdict_get, -- ditto 55 | 56 | disable_shdict = false, -- optional, default false 57 | 58 | memc_host = "127.0.0.1", 59 | memc_port = 11211, 60 | memc_timeout = 200, -- in ms 61 | memc_conn_pool_size = 5, 62 | memc_fetch_retries = 2, -- optional, default 1 63 | memc_fetch_retry_delay = 100, -- in ms, optional, default to 100 (ms) 64 | 65 | memc_conn_max_idle_time = 10 * 1000, -- in ms, for in-pool connections, 66 | -- optional, default to nil 67 | 68 | memc_store_retries = 2, -- optional, default to 1 69 | memc_store_retry_delay = 100, -- in ms, optional, default to 100 (ms) 70 | 71 | store_ttl = 1, -- in seconds, optional, default to 0 (i.e., never expires) 72 | } 73 | 74 | -- on hot code paths: 75 | 76 | local ctx = ngx.ctx 77 | 78 | -- in case of failure, can return the stale data with an error message 79 | local data, err = memc_fetch(ctx, key) 80 | 81 | -- using the default ttl specified by "store_ttl" 82 | local ok = memc_store(ctx, key, value) 83 | 84 | ok = memc_store(ctx, key, value, ttl) -- overriding the default ttl 85 | ``` 86 | 87 | Description 88 | =========== 89 | 90 | This library provides a powerful memcached client that deliver the following important features: 91 | 92 | 1. Automatically using shm-based caching layer with OpenResty's [lua_shared_dict](https://github.com/openresty/lua-nginx-module#lua_shared_dict). 93 | This local shm cache layer can also be turned off by setting the `disable_shdict` to `true`. 94 | 1. Automatically using server-level (not worker-level) cache locks (based on 95 | [lua-resty-lock](https://github.com/openresty/lua-resty-lock)) to avoid duplicate and concurrent 96 | queries to the memcached server for the same key when it is a cache miss in the local shm cache. 97 | 1. Automatically handles and logs any errors while accessing the Memcached server or `lua_shared_dict`. 98 | 1. Automatically uses connection pools for all the Memcached queries. 99 | 1. Automatically returns stale (or expired) data item in the local shm cache when a cache miss happens 100 | *and* some other light threads are already querying Memcached and updating the cache for the same key. 101 | 1. Automatically retry querying Memcached when failures happen (like intermittent network issues). 102 | 103 | This library is mostly used for cases that use Memcached or Memcached-compatible servers as the 104 | data storage (like Kyoto Tycoon). 105 | 106 | Atop this library, the user can further add another per-worker caching layer by employing the 107 | [lua-resty-lrucache](https://github.com/openresty/lua-resty-lrucache) 108 | library herself, and/or add asynchronous cache updating mechanism via [ngx.timer.at](https://github.com/openresty/lua-nginx-module#ngxtimerat). 109 | 110 | Caveats 111 | ======= 112 | 113 | * Unlike the [lua-resty-memcached](https://github.com/openresty/lua-resty-memcached) library, 114 | this client does not escape special characters in the key by default. So it is the caller's responsibility 115 | to avoid using special characters in the keys. 116 | 117 | [Back to TOC](#table-of-contents) 118 | 119 | Dependencies 120 | ============ 121 | 122 | This library depends on the following Lua libraries: 123 | 124 | * [lua-resty-memcached](https://github.com/openresty/lua-resty-memcached) 125 | * [lua-resty-lock](https://github.com/openresty/lua-resty-lock) 126 | * [lua-resty-shdict-simple](https://github.com/openresty/lua-resty-shdict-simple) 127 | 128 | [Back to TOC](#table-of-contents) 129 | 130 | TODO 131 | ==== 132 | 133 | * Add a test suite under `t/`. 134 | 135 | [Back to TOC](#table-of-contents) 136 | 137 | Author 138 | ====== 139 | 140 | Yichun "agentzh" Zhang (章亦春) , CloudFlare Inc. 141 | 142 | [Back to TOC](#table-of-contents) 143 | 144 | Copyright and License 145 | ===================== 146 | 147 | This module is licensed under the BSD license. 148 | 149 | Copyright (C) 2016, by CloudFlare Inc. 150 | 151 | All rights reserved. 152 | 153 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 154 | 155 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 156 | 157 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 158 | 159 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 160 | 161 | [Back to TOC](#table-of-contents) 162 | 163 | See Also 164 | ======== 165 | 166 | * [lua-resty-shdict-simple](https://github.com/openresty/lua-resty-shdict-simple) 167 | * [lua-resty-lrucache](https://github.com/openresty/lua-resty-lrucache) 168 | * [lua-resty-memcached](https://github.com/openresty/lua-resty-memcached) 169 | * [lua_shared_dict](https://github.com/openresty/lua-nginx-module#lua_shared_dict) 170 | 171 | [Back to TOC](#table-of-contents) 172 | -------------------------------------------------------------------------------- /lib/resty/memcached/shdict.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) CloudFlare Inc. 2 | -- 3 | -- Generate memcached getter/setter pair from config 4 | 5 | 6 | local _M = { 7 | version = '0.01', 8 | } 9 | 10 | 11 | local memcached = require "resty.memcached" 12 | local resty_lock = require "resty.lock" 13 | local str_format = string.format 14 | local sleep = ngx.sleep 15 | 16 | 17 | local DEBUG = ngx.config.debug 18 | 19 | 20 | local function id(a) 21 | return a 22 | end 23 | 24 | 25 | local memc_config = { 26 | key_transform = { id, id }, 27 | } 28 | 29 | 30 | local default_fetch_retries = 1 31 | local default_store_retries = 1 32 | local default_retry_delay = 100 -- msec 33 | local default_store_ttl = 0 -- no expiration 34 | 35 | 36 | function _M.gen_memc_methods (opts) 37 | local error_log = opts.error_logger 38 | local dlog = opts.debug_logger 39 | local warn = opts.warn_logger 40 | local tag = opts.tag 41 | 42 | local disable_shdict = opts.disable_shdict 43 | local shdict_set = opts.shdict_set 44 | local shdict_get = opts.shdict_get 45 | 46 | local locks_shdict_name = opts.locks_shdict_name 47 | local memc_timeout = opts.memc_timeout 48 | local pool_size = opts.memc_conn_pool_size 49 | local max_idle_time = opts.memc_conn_max_idle_time 50 | local memc_host = opts.memc_host 51 | local memc_port = opts.memc_port 52 | local fetch_retries = opts.memc_fetch_retries or default_fetch_retries 53 | local fetch_retry_delay = opts.memc_fetch_retry_delay or default_retry_delay 54 | fetch_retry_delay = fetch_retry_delay / 1000 -- sec 55 | local store_retries = opts.memc_store_retries or default_store_retries 56 | local store_retry_delay = opts.memc_store_retry_delay or default_retry_delay 57 | store_retry_delay = store_retry_delay / 1000 -- sec 58 | local store_ttl = opts.store_ttl or default_store_ttl 59 | 60 | local lock_opts = { 61 | exptime = memc_timeout * 2 / 1000, -- sec 62 | timeout = memc_timeout * 1.2 / 1000, -- sec 63 | step = 0.001, -- sec 64 | max_step = 0.1, -- sec 65 | } 66 | 67 | local function init_memc (ctx) 68 | local memc = ctx[tag] 69 | 70 | if not memc then 71 | local err 72 | memc, err = memcached:new(memc_config) 73 | if not memc then 74 | return nil, err 75 | end 76 | 77 | ctx[tag] = memc 78 | 79 | memc:set_timeout(memc_timeout) 80 | end 81 | 82 | -- we rely on the user to call release_memc() upon every invocation 83 | -- of init_memc(). 84 | 85 | local ok, err = memc:connect(memc_host, memc_port) 86 | if not ok then 87 | return nil, err 88 | end 89 | 90 | if DEBUG then dlog(ctx, "connected to memc") end 91 | 92 | return memc 93 | end 94 | 95 | local function release_memc(ctx, memc) 96 | local ok, err = memc:set_keepalive(max_idle_time, pool_size) 97 | if not ok then 98 | warn(ctx, "failed to put connection for ", tag, 99 | " into the pool: ", err) 100 | end 101 | end 102 | 103 | local function fetch_key_from_memc(ctx, key) 104 | 105 | local cached, stale = shdict_get(ctx, key) 106 | if cached and not stale then 107 | if cached == "" then 108 | return nil 109 | end 110 | 111 | return cached 112 | end 113 | 114 | -- cache lock 115 | 116 | local lock 117 | if not disable_shdict then 118 | local err 119 | lock, err = resty_lock:new(locks_shdict_name, lock_opts) 120 | if not lock then 121 | error_log(ctx, "failed to create a lock for memc queries: ", 122 | err) 123 | return nil 124 | end 125 | 126 | local elapsed, err = lock:lock(key) 127 | if elapsed then 128 | -- lock is acquired 129 | 130 | -- check if other timers have already insert a fresh result 131 | -- into the shdict. 132 | 133 | local cached, stale = shdict_get(ctx, key) 134 | if cached and not stale then 135 | if DEBUG then 136 | dlog(ctx, "some other timer just inserted a fresh memc " 137 | .. 'result into shdict for key "', key, '"') 138 | end 139 | if lock then 140 | lock:unlock() 141 | end 142 | 143 | if cached == "" then 144 | -- negative hit 145 | return nil 146 | end 147 | 148 | return cached 149 | end 150 | 151 | else 152 | error_log(ctx, 'failed to acquire cache lock on key "', key, 153 | '"; proceed anyway') 154 | end 155 | end 156 | 157 | if cached == "" then 158 | -- negative hit 159 | cached = nil 160 | end 161 | 162 | -- here we intentionally duplicate the code a bit to avoid 163 | -- inner loops with low iteration count on normal code 164 | -- path. 165 | 166 | local memc, err = init_memc(ctx) 167 | if not memc then 168 | if lock then 169 | lock:unlock() 170 | end 171 | return cached, "failed to init " .. tag .. ": " .. (err or "") 172 | end 173 | 174 | local res, flags, err = memc:get(key) 175 | if not res and err then 176 | 177 | for i = 1, fetch_retries do 178 | 179 | if fetch_retry_delay > 0 then 180 | if DEBUG then 181 | dlog(ctx, "waiting for ", fetch_retry_delay, 182 | " sec before the next retry #", i) 183 | end 184 | sleep(fetch_retry_delay) 185 | end 186 | 187 | warn("retrying fetch #", i, " from ", tag, 188 | ' due to error "', err, '"') 189 | 190 | memc, err = init_memc(ctx) 191 | if not memc then 192 | -- "fetch retries" are not "connect retries" anyway: 193 | if lock then 194 | lock:unlock() 195 | end 196 | return cached, "failed to init " .. tag .. ": " 197 | .. (err or "") 198 | end 199 | 200 | res, flags, err = memc:get(key) 201 | if res or not err then 202 | break 203 | end 204 | end 205 | 206 | if err then 207 | if lock then 208 | lock:unlock() 209 | end 210 | 211 | local msg = str_format('failed to get key "%s" from %s: %s', 212 | key, tag, err or "") 213 | return cached, msg 214 | end 215 | end 216 | 217 | release_memc(ctx, memc) 218 | 219 | shdict_set(ctx, key, res or "") 220 | 221 | if lock then 222 | lock:unlock() 223 | end 224 | 225 | return res 226 | end 227 | 228 | local function store_key_to_memc(ctx, key, value, ttl) 229 | shdict_set(ctx, key, value or "", ttl) 230 | 231 | ttl = ttl or store_ttl 232 | 233 | local lock 234 | if not disable_shdict then 235 | local err 236 | lock, err = resty_lock:new(locks_shdict_name, lock_opts) 237 | if not lock then 238 | error_log(ctx, "failed to create a lock for memc queries: ", 239 | err) 240 | return nil 241 | end 242 | 243 | local elapsed, err = lock:lock(key) 244 | if elapsed then 245 | -- lock is acquired 246 | else 247 | error_log(ctx, 'failed to acquire cache lock on key "', key, 248 | '"; proceed anyway') 249 | return nil 250 | end 251 | end 252 | 253 | local memc, err = init_memc(ctx) 254 | if not memc then 255 | if lock then 256 | lock:unlock() 257 | end 258 | 259 | error_log(ctx, "failed to init ", tag, ": ", err, 260 | ', dst: ', memc_host, ':', memc_port) 261 | return nil 262 | end 263 | 264 | local ok, err = memc:set(key, value, ttl) 265 | if not ok and err then 266 | 267 | for i = 1, store_retries do 268 | 269 | if store_retry_delay > 0 then 270 | if DEBUG then 271 | dlog(ctx, "waiting for ", store_retry_delay, 272 | " sec before the next retry #", i) 273 | end 274 | sleep(store_retry_delay) 275 | end 276 | 277 | warn("retrying store #", i, " from ", tag, 278 | ' due to error "', err, '"') 279 | 280 | memc, err = init_memc(ctx) 281 | if memc then 282 | ok, err = memc:set(key, value, ttl) 283 | if ok or not err then 284 | break 285 | end 286 | end 287 | end 288 | 289 | if err then 290 | if lock then 291 | lock:unlock() 292 | end 293 | 294 | error_log(ctx, 'failed to store key "', key, '" to ', tag, 295 | ': ', err) 296 | return nil 297 | end 298 | end 299 | 300 | release_memc(ctx, memc) 301 | 302 | if lock then 303 | lock:unlock() 304 | end 305 | 306 | return true 307 | end 308 | 309 | return fetch_key_from_memc, store_key_to_memc 310 | end 311 | 312 | 313 | return _M 314 | --------------------------------------------------------------------------------