├── .gitignore ├── README.md ├── check-env.sh ├── config_dev └── config.lua ├── ctrl ├── api.lua ├── cconfig_demo.lua ├── home.lua ├── metrics.lua ├── stock.lua └── test_resty_shell.lua ├── gendeb.sh ├── install.deb ├── lib ├── cconfig.lua ├── client.lua ├── client.test.lua ├── cors.lua ├── lup.lua ├── lup.test.lua ├── mongo_client.lua ├── mysql_clinet.lua ├── prom.lua ├── redis_client.lua ├── route.lua ├── sentry.demo.lua ├── sentry.lua ├── stat.lua ├── swoole.lua ├── tlcache.lua ├── tlcache.test.lua ├── zset.lua └── zset.test.lua ├── lualib ├── classic.lua ├── inotify.so ├── lfs.so ├── librestychash.so ├── multipart.lua ├── prometheus.lua ├── raven │ ├── init.lua │ ├── senders │ │ ├── luasocket.lua │ │ ├── ngx.lua │ │ ├── reference.lua │ │ └── test.lua │ └── util.lua ├── redis_slot.so ├── rediscluster.lua ├── resty │ ├── beanstalkd.lua │ ├── chash.lua │ ├── etcd.lua │ ├── http.lua │ ├── http_headers.lua │ ├── kafka │ │ ├── broker.lua │ │ ├── client.lua │ │ ├── errors.lua │ │ ├── producer.lua │ │ ├── request.lua │ │ ├── response.lua │ │ ├── ringbuffer.lua │ │ └── sendbuffer.lua │ ├── mlcache.lua │ ├── mlcache │ │ └── ipc.lua │ ├── moongoo.lua │ ├── moongoo │ │ ├── auth │ │ │ ├── cr.lua │ │ │ └── scram.lua │ │ ├── collection.lua │ │ ├── connection.lua │ │ ├── cursor.lua │ │ ├── database.lua │ │ ├── gridfs.lua │ │ ├── gridfs │ │ │ └── file.lua │ │ └── utils.lua │ ├── redis │ │ ├── connector.lua │ │ └── sentinel.lua │ ├── template.lua │ ├── template │ │ ├── html.lua │ │ └── microbenchmark.lua │ └── uuid.lua ├── serpent.lua ├── skiplist.so ├── typeof.lua └── zlib.so ├── nginx.conf ├── oresty_1.15.8.2-20200224162915_amd64.deb ├── reload.sh ├── restart.sh ├── site.conf ├── site.lua ├── static └── a.js ├── template ├── home.html └── layout.html └── version /.gitignore: -------------------------------------------------------------------------------- 1 | *.service 2 | /config 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | * [Oresty](#oresty) 4 | * [Benchmark - 基准测试](#benchmark---基准测试) 5 | * [Quick Start - 快速开始](#quick-start---快速开始) 6 | * [OpenResty目录结构](#openresty目录结构) 7 | * [Sentry - raven.lua](#sentry---ravenlua) 8 | * [lib/route.lua](#libroutelua) 9 | * [resty命令行](#resty命令行) 10 | * [lib/tlcache.lua Oresty独有的库，基于mlcache和lrucache](#libtlcachelua-oresty独有的库基于mlcache和lrucache) 11 | * [lib/prom.lua](#libpromlua) 12 | * [lib/stat.lua](#libstatlua) 13 | * [lib/swoole.lua](#libswoolelua) 14 | * [错误栈](#错误栈) 15 | * [lib/cors.lua](#libcorslua) 16 | * [lib/redis_clinet.lua](#libredis_clinetlua) 17 | * [lib/mongo_client.lua](#libmongo_clientlua) 18 | * [lib/mysql_clinet.lua](#libmysql_clinetlua) 19 | * [使用prometheus](#使用prometheus) 20 | * [lup.lua](#luplua) 21 | * [client.lua](#clientlua) 22 | * [竞品Alternatives](#竞品alternatives) 23 | * [引用Reference](#引用reference) 24 | * [Changelog](#changelog) 25 | * [20200310](#20200310) 26 | * [20200310](#20200310-1) 27 | * [20200308](#20200308) 28 | * [20190929](#20190929) 29 | * [20190928](#20190928) 30 | * [20190915](#20190915) 31 | 32 | 33 | 34 | # Oresty 35 | 36 | 基于OpenResty的web应用框架，旨在推广OpenResty在web应用领域的应用 37 | - 支持异步任务，异步IO 38 | - 比Go性能好，比PHP入门简单 39 | - Demo: http://tjx.be/ 40 | 41 | ## Benchmark - 基准测试 42 | 43 | - Go 44 | - PHP 45 | - Oresty 46 | 47 | ## Quick Start - 快速开始 48 | 49 | # 50 | # 安装deb包，初始化应用环境 51 | # 52 | dpkg -i install.deb 53 | bash check-env.sh 54 | 55 | # 56 | # 启动程序，检测进程状况 57 | # 58 | systemctl start oresy 59 | systemctl status oresy 60 | 61 | # 62 | # 浏览器打开 63 | # 64 | http://127.0.0.1:2223 65 | 66 | ## OpenResty目录结构 67 | 68 | ./oresty/bin 69 | ./oresty/lualib 70 | ./oresty/luajit 71 | ./oresty/nginx 72 | ./oresty/site 73 | ./oresty/pod 74 | ./oresty/ 75 | 76 | ## Sentry - raven.lua 77 | 78 | ## lib/route.lua 79 | 80 | ## resty命令行 81 | 便捷的执行带参数的resty命令的技巧 82 | 83 | #!/usr/local/bin/env -S /usr/local/oresty/bin/resty --shdict 'prometheus_metrics 2M' -I ./lualib/ 84 | 85 | ## lib/tlcache.lua Oresty独有的库，基于mlcache和lrucache 86 | 87 | - lib/lrucache.lua OpenResty内置 88 | 通过upvalue实现缓存 89 | 90 | - lib/mlcache.lua 优质的第三方库 91 | - 缓存库，设计理念源于lua-resty-mlcache 92 | - 区别在于可以再配置一层mongo或者redis实现多主机共享缓存 93 | - 相对于mlcache的两层缓存（L1-upvalue，L2-shdict），tlcache是三级缓存 94 | - TODO: 第三级缓存支持锁，避免多主机并发场景下的多次初始化 95 | 96 | ``` 97 | -- 98 | -- 现在site.conf 里面添加lua_shared_dict配置 99 | -- 100 | lua_shared_dict tlcache 100m; 101 | 102 | local tlcache = require 'lib.tlcache' 103 | local cache1 = tlcache.new('cache1', 'tlcache', { 104 | ttl = 10, 105 | neg_ttl = 10, 106 | mongo_config = {} 107 | redis_config = {} 108 | }) 109 | while 1 do 110 | cache1:get('key1', {ttl = 10, neg_ttl}, function() 111 | ngx.say(ngx.ERR, ngx.time()) 112 | end) 113 | ngx.sleep(1) 114 | end 115 | ``` 116 | 117 | ## lib/prom.lua 118 | 统计库，支持Prometheus格式输出 119 | 120 | ## lib/stat.lua 121 | 122 | 统计库，支持Prometheus格式输出 123 | 124 | -- 125 | -- 启动stat进程，注意只需要在一个worker里面启动 126 | -- 127 | init_worker_by_lua_bock { 128 | if ngx.worker.id() == 1 then 129 | local stat = require 'lib.stat' 130 | stat.start() 131 | end 132 | } 133 | 134 | -- 135 | -- 在content_by_lua或者log_by_lua等阶段使用，incr是唯一接口 136 | -- 137 | local stat = require 'lib.stat' 138 | local s = stat.new('label_name=label_value') 139 | s:incr('key_name', 123) 140 | 141 | -- 142 | -- 输出Prometheus的方法 143 | -- 144 | local metrics = stat.get_metrics() 145 | for _, m in pairs(metrics) do 146 | ngx.say(m.metric, m.labels or '', ' ', m.value) 147 | end 148 | 149 | -- 150 | -- 本库会统计好count，delta，avg等数值输出，如下 151 | -- 152 | key_name{label_name="label_value"} 123 153 | key_name_count{label_name="label_value"} 1 154 | key_name_delta{label_name="label_value"} 123 155 | key_name_avg{label_name="label_value"} 123 156 | 157 | 158 | ## lib/swoole.lua 159 | 160 | 用lua实现swoole，task-workers和request-workers隔离 161 | 162 | ## 错误栈 163 | 164 | ## lib/cors.lua 165 | 166 | ## lib/redis_clinet.lua 167 | 168 | local redis_client = require 'lib.redis_client' 169 | local red = redis_client.new() 170 | local ret = red:set() 171 | local red:get() 172 | 173 | ## lib/mongo_client.lua 174 | 175 | local mongo_client = require 'lib.mongo_client' 176 | local mdb = mongo_client.new() 177 | mdb:get() 178 | mdb:set() 179 | 180 | ## lib/mysql_clinet.lua 181 | 182 | local mysql_client = require 'lib.mysql_client' 183 | local db = mysql_client.new() 184 | local res = db:query('select * from test.a_table;') 185 | 186 | ## 使用prometheus 187 | knyar/nginx-lua-prometheus: Prometheus metric library for Nginx written in Lua 188 | https://github.com/knyar/nginx-lua-prometheus 189 | 190 | ## lup.lua 191 | 192 | 用lua实现PHP的function 193 | 194 | ## client.lua 195 | 196 | http客户端封装库，依赖lua-resty-http 197 | https://github.com/ledgetech/lua-resty-http 198 | 设计理念是建立和php-guzzle接口完全兼容的lua库 199 | 接口和PHP开源项目Guzzle完全一致 200 | http://docs.guzzlephp.org/ 201 | 202 | 203 | local client = require 'lib.client' 204 | local ret = client.get('http://baidu.com', { 205 | timeout = 1000 -- timeout in ms 206 | retry = 3, -- 网络错误时候，重试次数 207 | retry_interval = 1 -- retry interval in second 208 | proxy = 'http://baidu.com', -- 代理 209 | headers = { -- http-header 指定 210 | ['X-Auth'] = 'nihao' 211 | }, 212 | query = { -- url的参数部分 213 | ni = 'hao' 214 | } 215 | }) 216 | 217 | -- 218 | -- send post request 219 | -- 220 | local ret = client.post('http://baidu.com', { 221 | -- 222 | -- body will be encoded with json 223 | -- content_type in header is 'application/json' 224 | -- 225 | json = { 226 | ni = 'hao' 227 | } 228 | }) 229 | local ret = client.post('http://baidu.com', { 230 | -- 231 | -- body will be encoded with json 232 | -- content_type in header is 'application/x-www-form-urlencoded' 233 | -- 234 | form_params = { 235 | ni = 'hao' 236 | } 237 | }) 238 | local ret = client.post('http://baidu.com', { 239 | body = 'BODY_CONTENT' 240 | }) 241 | 242 | -- 243 | -- send other request 244 | -- 245 | client.delete(url) 246 | client.put(url) 247 | client.options(url) 248 | 249 | -- 250 | -- deal with output 251 | -- 252 | 253 | ## 竞品Alternatives 254 | 255 | - sumory/lor: a fast, minimalist web framework for lua based on OpenResty 256 | https://github.com/sumory/lor 257 | 258 | - leafo/lapis: A web framework for Lua and OpenResty written in MoonScript 259 | https://github.com/leafo/lapis 260 | 261 | 262 | ## 引用Reference 263 | - 序 · OpenResty最佳实践 264 | https://moonbingbing.gitbooks.io/openresty-best-practices/ 265 | 266 | - bungle/awesome-resty: A List of Quality OpenResty Libraries, and Resources. 267 | https://github.com/bungle/awesome-resty 268 | 269 | - apache/incubator-apisix: Cloud-Native Microservices API Gateway 270 | https://github.com/apache/incubator-apisix 271 | 272 | ## Changelog 273 | 274 | ### 20200310 275 | - 升级lib/route.lua 276 | - 弃用init，启用systemd作为进程管理 277 | 278 | ### 20200310 279 | - 添加prom.lua, zset.lua 280 | - 添加toc目录 281 | 282 | ### 20200308 283 | - 优化文档 284 | - 添加prometheus.lua 285 | - 添加sentry.lua 286 | - 增加OpenResty1.17rc版本 287 | - 增加支持lua-resty-shell非阻塞执行本地命令 288 | 289 | ### 20190929 290 | - 更新安装方式，安装文档 291 | - 添加select2.js 292 | - 添加lib/mysql_client 293 | 294 | ### 20190928 295 | - 更新README，添加client,stat,tlcache的使用说明 296 | - 添加tlcache 297 | 298 | ### 20190915 299 | - 升级OpenResty到1.15 300 | - 升级route.lua 301 | - 升级redis_client.lua 302 | - 新增lua-resty-etcd 303 | -------------------------------------------------------------------------------- /check-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | test -e config || ln -s config_${1:-dev} config 5 | 6 | # 7 | # 安装 8 | # 9 | dpkg -i ./install.deb 10 | 11 | # 12 | # 配置systemd 13 | # 14 | APPDIR=$PWD 15 | echo """ 16 | [Service] 17 | User=root 18 | WorkingDirectory=$APPDIR 19 | ExecStart=/usr/local/oresty/nginx/sbin/nginx -c $APPDIR/nginx.conf -g 'daemon off;' 20 | ExecReload=/usr/local/oresty/nginx/sbin/nginx -c $APPDIR/nginx.conf -s reload 21 | ExecStop=/usr/local/oresty/nginx/sbin/nginx -c $APPDIR/nginx.conf -s stop 22 | [Install] 23 | WantedBy=default.target 24 | """ > oresty.service 25 | systemctl enable $(realpath oresty.service) 26 | 27 | test -e .git && { 28 | test -e .git/hooks/pre-commit || { 29 | echo '#!/bin/sh 30 | echo v$(git log master --pretty=oneline | wc -l)-$(date +%Y%m%d) > version 31 | git add version' > .git/hooks/pre-commit 32 | chmod +x .git/hooks/pre-commit 33 | } 34 | } 35 | 36 | echo OK 37 | -------------------------------------------------------------------------------- /config_dev/config.lua: -------------------------------------------------------------------------------- 1 | return { 2 | sentry_dsn = 'https://d7bc348a39024434ac4269469af7a666:cd0883cb985242ee82cd4c0e75065b14@sentry.io/4043287', 3 | redis = { 4 | host = '127.0.0.1', 5 | port = 6379, 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /ctrl/api.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local client = require 'lib.client' 6 | local config = require 'config.config' 7 | local redis_client = require 'lib.redis_client' 8 | local template = require 'resty.template' 9 | 10 | local _M = {} 11 | 12 | function _M.get(params) 13 | 14 | local red = redis_client.new(config.redis) 15 | local ret = '' 16 | 17 | if params.val then 18 | ret = red:set(params.key, params.val) 19 | else 20 | ret = red:get(params.key) 21 | end 22 | 23 | return ret 24 | 25 | end 26 | 27 | return _M 28 | -------------------------------------------------------------------------------- /ctrl/cconfig_demo.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local client = require 'lib.client' 6 | local cconfig = require 'lib.cconfig' 7 | local config = require 'config.config' 8 | local template = require 'resty.template' 9 | 10 | local _M = {} 11 | 12 | function _M.get(params) 13 | 14 | local origin = cconfig.get('origin') 15 | 16 | return origin 17 | 18 | end 19 | 20 | return _M 21 | -------------------------------------------------------------------------------- /ctrl/home.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local client = require 'lib.client' 6 | local lup = require 'lib.lup' 7 | local config = require 'config.config' 8 | 9 | local _M = {} 10 | 11 | function _M.get(params) 12 | 13 | -- local ret = client.get('http://example.com') 14 | -- local body = ret.body 15 | 16 | return { 17 | time = os.date("%Y-%m-%d_%H:%M:%S", ngx.time()), 18 | redis_host = config.redis.host, 19 | message = 'Hello, World!', 20 | arr = { 21 | key1 = 'val1', 22 | key2 = 'val2', 23 | key3 = 'val3', 24 | } 25 | }, 'template/home.html' 26 | 27 | end 28 | 29 | return _M 30 | -------------------------------------------------------------------------------- /ctrl/metrics.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local prometheus = require 'prometheus' 6 | 7 | local _M = {} 8 | 9 | local prom = prometheus.init("prometheus_metrics") 10 | 11 | local metric_requests = prom:counter("nginx_http_requests_total", "Number of HTTP requests", {"host", "status"}) 12 | local metric_latency = prom:histogram("nginx_http_request_duration_seconds", "HTTP request latency", {"host"}) 13 | local metric_connections = prom:gauge("nginx_http_connections", "Number of HTTP connections", {"state"}) 14 | 15 | function _M.get() 16 | 17 | metric_requests:inc(1, {ngx.var.host, ngx.status}) 18 | metric_latency:observe(5, {ngx.var.host}) 19 | 20 | metric_connections:set(ngx.var.connections_reading, {"reading"}) 21 | metric_connections:set(ngx.var.connections_waiting, {"waiting"}) 22 | metric_connections:set(ngx.var.connections_writing, {"writing"}) 23 | 24 | prom:collect() 25 | end 26 | 27 | return _M 28 | -------------------------------------------------------------------------------- /ctrl/stock.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local client = require 'lib.client' 6 | local cjson = require 'cjson.safe' 7 | local serpent = require 'serpent' 8 | local config = require 'config.config' 9 | local template = require 'resty.template' 10 | 11 | local _M = {} 12 | 13 | function _M.get(params) 14 | 15 | local id = params.id or '' 16 | local key = 'e9a53708d6efa874db3134a3da7c4993' 17 | local url = 'http://web.juhe.cn:8080/finance/stock/' 18 | local ret 19 | 20 | if ngx.re.match(id, '^(sh|sz)[0-9]{6}$') then 21 | ret = client.get(url..'hs', { 22 | query = { 23 | key = key, 24 | gid = id, 25 | } 26 | }) 27 | elseif ngx.re.match(id, '^[0-9]{5}$') then 28 | ret = client.get(url..'hk', { 29 | query = { 30 | key = key, 31 | num = id, 32 | } 33 | }) 34 | elseif ngx.re.match(id, '^[a-z]{2,4}$') then 35 | ret = client.get(url..'usa', { 36 | query = { 37 | key = key, 38 | gid = id, 39 | } 40 | }) 41 | else 42 | ngx.status = 404 43 | return '
Usage:\ncurl pri.tjx.be?id=00700\ncurl pri.tjx.be?id=sh600436\ncurl pri.tjx.be?id=baba
' 44 | end 45 | 46 | ngx.log(ngx.ERR, ret.body) 47 | local r = (cjson.decode(ret.body) or {}).result[1].data 48 | return r.lastestpri or r.nowPri 49 | 50 | end 51 | 52 | return _M 53 | -------------------------------------------------------------------------------- /ctrl/test_resty_shell.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local lup = require 'lib.lup' 6 | 7 | local _M = {} 8 | 9 | function _M.get(params) 10 | lup.exec158('sleep 1') 11 | return 'ss'..lup.filesize('/etc/issue') 12 | end 13 | 14 | function _M.get2(params) 15 | lup.exec('sleep 1') 16 | return 'ii' 17 | end 18 | 19 | return _M 20 | -------------------------------------------------------------------------------- /gendeb.sh: -------------------------------------------------------------------------------- 1 | 2 | # apt install -y autoconf libreadline-dev libncurses5-dev libpcre3-dev libssl-dev make 3 | 4 | # OR_VERSION=1.17.8.1rc0 5 | # NGX_VERSION=1.17.8 6 | OR_VERSION=1.15.8.2 7 | NGX_VERSION=1.15.8 8 | wget https://openresty.org/download/openresty-${OR_VERSION}.tar.gz -O /tmp/openresty-${OR_VERSION}.tar.gz 9 | rm -rf /tmp/openresty-${OR_VERSION} 10 | tar xvf /tmp/openresty-${OR_VERSION}.tar.gz -C /tmp/ 11 | cd /tmp/openresty-${OR_VERSION} 12 | 13 | echo ' 14 | 92c92 15 | < p = ngx_cpystrn((u_char *) ngx_os_argv[0], (u_char *) "nginx: ", 16 | --- 17 | > p = ngx_cpystrn((u_char *) ngx_os_argv[0], (u_char *) "oresty: ", 18 | ' |patch ./bundle/nginx-${NGX_VERSION}/src/os/unix/ngx_setproctitle.c 19 | 20 | ./configure \ 21 | --prefix=/usr/local/oresty \ 22 | --with-cc-opt=-O2 \ 23 | --with-luajit 24 | make -j 25 | 26 | checkinstall --pkgrelease=$(date +%Y%m%d%H%M%S) --pkgname=oresty -y 27 | -------------------------------------------------------------------------------- /install.deb: -------------------------------------------------------------------------------- 1 | oresty_1.15.8.2-20200224162915_amd64.deb -------------------------------------------------------------------------------- /lib/cconfig.lua: -------------------------------------------------------------------------------- 1 | local client = require 'lib.client' 2 | local lup = require 'lib.lup' 3 | local cjson = require 'cjson.safe' 4 | local config = require 'config.config' 5 | 6 | local _M = {} 7 | 8 | local cconfig = {} 9 | local mtime = 0 10 | 11 | function _M.get(key) 12 | 13 | local url = config.cconfig_url 14 | 15 | if not url then 16 | return nil 17 | end 18 | 19 | local ret, r 20 | ngx.update_time() 21 | if ngx.time() > mtime + 10 then 22 | ret = lup.a(client.get, url, {timeout=100, retry=2}) 23 | if ret then 24 | cconfig = cjson.decode(ret.body) or cconfig 25 | mtime = ngx.time() 26 | end 27 | end 28 | 29 | return (cconfig or {})[key] 30 | 31 | end 32 | 33 | return _M 34 | -------------------------------------------------------------------------------- /lib/client.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- User: tangjunxing 3 | -- Date : 2017/08/31 4 | -- 5 | 6 | local http = require 'resty.http' 7 | local cjson = require 'cjson.safe' 8 | 9 | local _M = { 10 | version = 171130 11 | } 12 | 13 | _M = setmetatable({version = 171130}, { 14 | __index = function(_, method) 15 | return setmetatable({}, { 16 | __call = function(_, ...) 17 | return _M.request(method, ...) 18 | end 19 | }) 20 | end 21 | }) 22 | 23 | local function get_curl(method, arg_url, args) 24 | local cmds = {} 25 | local query = '' 26 | args = args or {} 27 | for k, v in pairs(args) do 28 | if k == 'query' then 29 | query = '?'..ngx.encode_args(v) 30 | elseif k == 'headers' then 31 | for header, header_val in pairs(v) do 32 | cmds[#cmds+1] = '-H "'..header..': '..header_val..'"' 33 | end 34 | elseif k == 'proxy' then 35 | cmds[#cmds+1] = " -x"..v 36 | elseif k == 'body' then 37 | cmds[#cmds+1] = '-d \''..v..'\'' 38 | elseif k == 'json' then 39 | cmds[#cmds+1] = '-H "Content-Type: application/json"' 40 | cmds[#cmds+1] = '-d \''..cjson.encode(v)..'\'' 41 | elseif k == 'form_params' then 42 | cmds[#cmds+1] = '-H "Content-Type: application/x-www-form-urlencoded"' 43 | cmds[#cmds+1] = '-d '..ngx.encode_args(v) 44 | end 45 | end 46 | return 'curl -X'..string.upper(method)..' "'..arg_url..query..'" '..table.concat(cmds, ' ') 47 | end 48 | 49 | function _M.request(method, url, args) 50 | 51 | method = string.upper(method or '') 52 | if not ({OPTIONS=1, HEAD=1, GET=1, PUT=1, POST=1, DELETE=1})[method] then 53 | return nil, 'bad method' 54 | end 55 | if not url then 56 | return nil, 'bad method' 57 | end 58 | args = args or {} 59 | local headers = {} 60 | local body = nil 61 | local query = '' 62 | for k, v in pairs(args) do 63 | if k == 'query' then 64 | query = '?'..ngx.encode_args(v); 65 | elseif k == 'body' then 66 | body = args.body 67 | elseif k == 'json' then 68 | body = cjson.encode(args.json) 69 | headers['Content-Type'] = 'application/json' 70 | elseif k == 'form_params' then 71 | body = ngx.encode_args(args.form_params) 72 | headers['Content-Type'] = 'application/x-www-form-urlencoded' 73 | elseif k == 'headers' then 74 | for k1, v1 in pairs(v) do 75 | headers[k1] = v1 76 | end 77 | end 78 | end 79 | 80 | local res, err 81 | local httpc = http.new() 82 | httpc:set_timeout(args.timeout or 200) 83 | 84 | -- 85 | -- proxy需要lua-resty-http-0.12支持 86 | -- 87 | if args.proxy then 88 | httpc:set_proxy_options{ 89 | http_proxy = args.proxy 90 | } 91 | end 92 | 93 | local retry = args.retry or 10 94 | while retry > 0 do 95 | res, err = httpc:request_uri(url..query, { 96 | body = body, 97 | method = method, 98 | headers = headers 99 | }) 100 | httpc:close() 101 | ngx.log(ngx.ERR, 'HTTPCERR_'..(err or 'OK')..'_RETRY_'..retry..' '..get_curl(method, url, args)) 102 | retry = retry - 1 103 | if res or retry == 0 then 104 | break 105 | end 106 | ngx.sleep(args.retry_interval or 0.1) 107 | end 108 | if res and ({[301]=1,[302]=1,[200]=1,[204]=1,[206]=1})[res.status] then 109 | return res 110 | end 111 | error({ 112 | err = err, 113 | body = res and res.body, 114 | status = res and res.status 115 | }) 116 | 117 | end 118 | 119 | return _M 120 | 121 | -------------------------------------------------------------------------------- /lib/client.test.lua: -------------------------------------------------------------------------------- 1 | local client = require 'lib.client' 2 | local lup = require 'lib.lup' 3 | 4 | local ret = client.get('http://baidu.com') 5 | 6 | lup.var_dump(ret) 7 | -------------------------------------------------------------------------------- /lib/cors.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- User: tangjunxing 3 | -- Date : 2017/08/31 4 | -- 5 | 6 | local _M = { 7 | version = 190924 8 | } 9 | 10 | --[[ 11 | params = { 12 | cors = { 13 | { 14 | origins = { 15 | 'http://*.baidu.co', 16 | 'https://*.baidu.co', 17 | 'http://*.baidu.com', 18 | 'https://*.baidu.com', 19 | }, 20 | methods = { 21 | 'DELETE', 22 | 'PUT', 23 | }, 24 | headers = { 25 | 'Content-Type', 26 | 'X-Upos-Auth', 27 | 'Range', 28 | }, 29 | expose_headers = { 30 | }, 31 | max_age_seconds = 1800, 32 | } 33 | } 34 | } 35 | ]] 36 | 37 | function _M.apply(params) 38 | 39 | local origin = ngx.var.http_origin 40 | if origin and params.cors then 41 | for _, cors in pairs(params.cors) do 42 | if not cors.origins then 43 | break 44 | end 45 | for _, allowed_origin in pairs(cors.origins) do 46 | if origin:match(allowed_origin:gsub("%.", "%%."):gsub("%*", "%.%*")) then 47 | ngx.header["Access-Control-Allow-Origin"] = origin 48 | if ngx.var.request_method == "OPTIONS" then 49 | if cors.methods then 50 | ngx.header["Access-Control-Allow-Methods"] = table.concat(cors.methods, ", ") 51 | end 52 | if cors.headers then 53 | ngx.header["Access-Control-Allow-Headers"] = table.concat(cors.headers, ", ") 54 | end 55 | if cors.expose_headers then 56 | ngx.header["Access-Control-Expose-Headers"] = table.concat(cors.expose_headers, ", ") 57 | end 58 | ngx.header["Access-Control-Allow-Credentials"] = "true" 59 | else 60 | ngx.header["Access-Control-Allow-Credentials"] = "true" 61 | break 62 | end 63 | return 64 | end 65 | end 66 | end 67 | end 68 | 69 | end 70 | 71 | return _M 72 | -------------------------------------------------------------------------------- /lib/lup.test.lua: -------------------------------------------------------------------------------- 1 | local lup = require 'lib.lup' 2 | 3 | -- 4 | -- in_array 5 | -- 6 | assert(lup.in_array('ni', {'ni', 'hao', 'bu', 'kk'})) 7 | assert(not lup.in_array('xni', {'ni', 'hao', 'bu', 'kk'})) 8 | 9 | -- 10 | -- array_merge 11 | -- 12 | local t1 = {1,2,3} 13 | local t2 = {4,5,6} 14 | lup.var_dump(lup.array_merge(t1, t2)) 15 | 16 | local t1 = {ni={1},2,3} 17 | local t2 = {ni={4},5,6} 18 | lup.var_dump(lup.array_merge(t1, t2)) 19 | lup.var_dump(lup.array_merge({ni={1},2,3}, {ni={4},5,6})) 20 | 21 | -- 22 | -- parse_url 23 | -- 24 | local url = 'http://baidu.com/path/to/file.ext' 25 | lup.var_dump(lup.parse_url(url)) 26 | 27 | -- 28 | -- md5_file 29 | -- 30 | assert('a7eef95d86c3649a192350845d8cb2ae' == lup.md5_file('/etc/issue')) 31 | 32 | 33 | print('PASS') 34 | -------------------------------------------------------------------------------- /lib/mongo_client.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20181018 3 | -- 4 | 5 | local _M = {} 6 | 7 | local moongoo = require 'resty.moongoo' 8 | local config = require 'config.config' 9 | local serpent = require 'serpent' 10 | 11 | 12 | function _M.new(collection, mongo_config) 13 | local self = setmetatable({ 14 | }, { 15 | __index = function(_, method) 16 | return setmetatable({}, { 17 | __call = function(_, self, ...) 18 | local retry = 3 19 | while retry > 0 do 20 | local ok, ret = xpcall(function(...) 21 | local mgobj = moongoo.new(mongo_config or config.mongo) 22 | local coll = mgobj:db(mgobj.default_db):collection(collection) 23 | local ret, err = coll[method](coll, ...) 24 | mgobj:close() 25 | return ret, err 26 | end, 27 | function(err) 28 | return { 29 | err = err, 30 | traceback = debug.traceback() 31 | } 32 | end, 33 | ...) 34 | if ok then 35 | return ret 36 | end 37 | ngx.log(ngx.ERR, 'moongoo_retry: '..serpent.block(ret)) 38 | if retry == 1 then 39 | return nil, ret 40 | end 41 | ngx.sleep(0.1) 42 | retry = retry - 1 43 | end 44 | end 45 | }) 46 | end 47 | }) 48 | return self 49 | end 50 | 51 | return _M 52 | 53 | -- coll.new(name, db) 54 | -- coll._build_write_concern(self) 55 | -- coll._get_last_error(self) 56 | -- coll._check_last_error(self, ...) 57 | -- coll.insert(self, docs) 58 | -- coll.create(self, params) 59 | -- coll.drop(self) 60 | -- coll.drop_index(self, name) 61 | -- coll.ensure_index(self, docs) 62 | -- coll.full_name(self) 63 | -- coll.options(self) 64 | -- coll.remove(self, query, single) 65 | -- coll.stats(self) 66 | -- coll.index_information(self) 67 | -- coll.rename(self, to_name, drop) 68 | -- coll.update(self, query, update, flags) 69 | -- coll.save(self, doc) 70 | -- coll.map_reduce(self, map, reduce, flags) 71 | -- coll.find(self, query, fields) 72 | -- coll.find_one(self, query, fields) 73 | -- coll.find_and_modify(self, query, opts) 74 | -- coll.aggregate(self, pipeline, opts) 75 | -- cur.new(collection, query, fields, explain, id) 76 | -- cur.tailable(self, tailable) 77 | -- cur.await(self, await) 78 | -- cur.comment(self, comment) 79 | -- cur.hint(self, hint) 80 | -- cur.max_scan(self, max_scan) 81 | -- cur.max_time_ms(self, max_time_ms) 82 | -- cur.read_preference(self, read_preference) 83 | -- cur.snapshot(self, snapshot) 84 | -- cur.sort(self, sort) 85 | -- cur.clone(self, explain) 86 | -- cur.skip(self, skip) 87 | -- cur.limit(self, limit) 88 | -- cur._build_query(self) 89 | -- cur.next(self) 90 | -- cur.all(self) 91 | -- cur.rewind(self) 92 | -- cur.count(self) 93 | -- cur.distinct(self, key) 94 | -- cur.explain(self) 95 | -- cur.add_batch(self, docs) 96 | -- cur._finished(self) 97 | 98 | -------------------------------------------------------------------------------- /lib/mysql_clinet.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20190929 3 | -- 4 | -- Usage: 5 | -- 6 | 7 | local _M = { 8 | version = 190924 9 | } 10 | 11 | local mysql = require 'resty.mysql' 12 | local serpent = require 'serpent' 13 | local lup = require 'lib.lup' 14 | 15 | function _M.new(config) 16 | return setmetatable({conf = config}, {__index = _M}) 17 | end 18 | 19 | function _M.query(self, sql) 20 | local conf = self.conf 21 | local db = mysql:new() 22 | db:set_timeout(5000) 23 | local retry = 2 24 | local res, err, errcode, sqlstate 25 | while retry > 0 do 26 | res, err, errcode, sqlstate = db:connect{ 27 | host = conf.host, 28 | port = conf.port, 29 | database = conf.database, 30 | user = conf.user, 31 | password = conf.password, 32 | charset = 'utf8', 33 | max_packet_size = 1024 * 1024, 34 | } 35 | if res then 36 | res, err, errcode, sqlstate = db:query(sql) 37 | if res then 38 | db:set_keepalive(10000, 50) 39 | return res 40 | end 41 | end 42 | db:close() 43 | ngx.log(ngx.ERR, 'QUERY_ERR_'..serpent.line{ 44 | retry = retry, 45 | res = res, 46 | err = err, 47 | errcode = errcode, 48 | sqlstate = sqlstate, 49 | sql = sql, 50 | }) 51 | retry = retry - 1 52 | ngx.sleep(0.1) 53 | end 54 | error('MYSQL_ERROR_RETRIED') 55 | end 56 | 57 | return _M 58 | -------------------------------------------------------------------------------- /lib/prom.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200310 3 | -- 4 | -- 单例模式封装prometheus.lua 5 | -- knyar/nginx-lua-prometheus: Prometheus metric library for Nginx written in Lua 6 | -- https://github.com/knyar/nginx-lua-prometheus 7 | -- 8 | 9 | local prometheus = require 'prometheus' 10 | local serpent = require 'serpent' 11 | local lup = require 'lib.lup' 12 | 13 | local _M = { 14 | version = 20200310 15 | } 16 | 17 | local shm_prom = ngx.shared.prometheus_metrics 18 | local _prom = nil 19 | local _registered = {} 20 | 21 | local function get_metric(_type, name, label_names) 22 | if not _prom then 23 | _prom = prometheus.init('prometheus_metrics') 24 | end 25 | local metric 26 | if not _registered[name] then 27 | metric = _prom[_type](_prom, name, nil, label_names) 28 | _registered[name] = metric 29 | else 30 | metric = _registered[name] 31 | end 32 | return metric 33 | end 34 | 35 | function _M.gauge(...) 36 | return get_metric('gauge', ...) 37 | end 38 | 39 | function _M.histogram(...) 40 | return get_metric('histogram', ...) 41 | end 42 | 43 | function _M.counter(...) 44 | return get_metric('counter', ...) 45 | end 46 | 47 | function _M.collect() 48 | _M.gauge('prom_keys_len'):set(#shm_prom:get_keys(0)) 49 | _prom:collect() 50 | end 51 | 52 | function _M.flush() 53 | shm_prom:flush_all() 54 | return { 55 | flushed = shm_prom:flush_expired() 56 | } 57 | end 58 | 59 | function _M.metric_data() 60 | _M.gauge('prom_keys_len'):set(#shm_prom:get_keys(0)) 61 | return _prom:metric_data() 62 | end 63 | 64 | function _M.parse_metric_data(metric_data) 65 | local rows = {} 66 | for _, line in ipairs(metric_data) do 67 | local m = ngx.re.match(line, '^([^#][^{]*)([^ ]*) (.*)') 68 | if m then 69 | local metric = m[1] 70 | local value = m[3] 71 | local labels = nil 72 | if not lup.empty(m[2]) then 73 | local d = 'do local _='..m[2]..'; return _;end' 74 | local ok, copy = serpent.load(d) 75 | if ok then 76 | labels = copy 77 | end 78 | end 79 | rows[#rows + 1] = { 80 | value = value, 81 | labels = labels, 82 | metric = metric, 83 | } 84 | end 85 | end 86 | return rows 87 | end 88 | 89 | return _M 90 | -------------------------------------------------------------------------------- /lib/redis_client.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- User: tangjunxing 3 | -- Date : 2017/08/31 4 | -- 5 | 6 | local redis = require 'resty.redis' 7 | local redis_cluster = require "rediscluster" 8 | 9 | local _M = {} 10 | 11 | function _M.new(conf) 12 | local instance = setmetatable({}, { 13 | __index = function(_, method) 14 | return setmetatable({method = method}, { 15 | __call = function(_, self, ...) 16 | local retry = conf.max_retry or 1 17 | local ret, err 18 | if conf.serv_list then 19 | local red = redis_cluster:new(conf) 20 | repeat 21 | if ret then 22 | ret, err = red[_.method](red, ...) 23 | if ret then 24 | return ret 25 | end 26 | end 27 | ngx.log(ngx.ERR, 'REDIS_RETRY_', retry, '_ERROR_', err) 28 | retry = retry - 1 29 | ngx.sleep(0.05) 30 | until retry < 0 31 | else 32 | local red = redis.new() 33 | repeat 34 | ret = red:connect(conf.host or '127.0.0.1', conf.port or 6379) 35 | if ret then 36 | ret, err = red[_.method](red, ...) 37 | if ret then 38 | red:set_keepalive(10000, 10) 39 | return ret 40 | end 41 | red:close() 42 | end 43 | ngx.log(ngx.ERR, 'REDIS_RETRY_', retry, '_ERROR_', err) 44 | retry = retry - 1 45 | ngx.sleep(0.05) 46 | until retry < 0 47 | end 48 | return nil, err 49 | end 50 | }) 51 | end 52 | }) 53 | return instance 54 | end 55 | 56 | return _M 57 | -------------------------------------------------------------------------------- /lib/route.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200313 3 | -- 4 | -- Usage: 5 | -- 6 | -- local rt = route.new() 7 | -- rt:dispatch(nil, 'home', 'get', { 8 | -- a = b, 9 | -- c = d 10 | -- }) 11 | -- 12 | -- or self defined route dirs 13 | -- 14 | -- local rt = route.new({'./ctrl/', 'ctrl1'}) 15 | -- rt:dispatch('ctrl', 'home', 'get', { 16 | -- a = b, 17 | -- c = d 18 | -- }) 19 | -- rt:dispatch('ctrl1', 'home1', 'get', { 20 | -- a = b, 21 | -- c = d 22 | -- }) 23 | -- 24 | 25 | local lup = require 'lib.lup' 26 | local serpent = require 'serpent' 27 | local template = require 'resty.template' 28 | local lfs = require 'lfs' 29 | 30 | local _M = { 31 | version = 20200313 32 | } 33 | 34 | local inited = false 35 | local loaded = {} 36 | function _M.new(dirs) 37 | local dirs = dirs or {'./ctrl/'} 38 | for _, dir in ipairs(dirs) do 39 | local dirname = lup.basename(lup.trim(dir, '/')) 40 | if not inited then 41 | for file in lfs.dir(dir) do 42 | local pathinfo = lup.pathinfo(file) 43 | if 'lua' == pathinfo.extension then 44 | local filename = pathinfo.filename 45 | loaded[dirname..'/'..filename] = require(dir..'/'..filename) 46 | ngx.log(ngx.ERR, 47 | dirname..'/'..filename..' => '..dir..'/'..filename 48 | ) 49 | end 50 | end 51 | end 52 | end 53 | inited = true 54 | return setmetatable({}, {__index = _M}) 55 | end 56 | 57 | function _M.dispatch(self, dirname, route, method, params) 58 | 59 | dirname = dirname or 'ctrl' 60 | assert(route, 'ROUTE_NIL') 61 | 62 | -- 63 | -- 调用方法，并且渲染输出 64 | -- 65 | local ok, ret = xpcall(function() 66 | assert(route, 'ROUTE_NIL') 67 | local ctrl = loaded[dirname..'/'..route] 68 | if not ctrl then 69 | ngx.status = 404 70 | error('CTRL_NOT_FOUND_'..dirname..'/'..route) 71 | elseif not ctrl[method] then 72 | ngx.status = 404 73 | error('METHOD_NOT_FOUND') 74 | end 75 | local ret, template_path = ctrl[method](params) 76 | if template_path then 77 | template.render(template_path, ret) 78 | elseif type(ret) == 'table' then 79 | ngx.header.content_type = 'application/json' 80 | ngx.say(lup.json_encode(ret)) 81 | elseif type(ret) == 'string' then 82 | ngx.header.content_type = 'text/plain' 83 | ngx.print(ret) 84 | elseif ret == nil then 85 | else 86 | error('ROUTE_UNKNOWN_RET_TYPE') 87 | end 88 | end, 89 | function(err) 90 | ngx.log(ngx.ERR, serpent.line({ 91 | err = err, 92 | traceback = debug.traceback(), 93 | }, {comment = false})) 94 | return err 95 | end) 96 | 97 | -- 98 | -- 异常处理 99 | -- 100 | if not ok then 101 | if ngx.status == 0 then 102 | ngx.status = 500 103 | end 104 | ngx.say(lup.json_encode({ 105 | params = params, 106 | http_code = ngx.status, 107 | reqid = ngx.var.http_x_upos_reqid, 108 | err = ret, 109 | })) 110 | end 111 | 112 | end 113 | 114 | return _M 115 | -------------------------------------------------------------------------------- /lib/sentry.demo.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local sentry = require 'lib.sentry' 6 | local config = require 'config.config' 7 | 8 | local sen = sentry.new(config.sentry_dsn) 9 | 10 | local function bad_func(n) 11 | return not_defined_func(n) 12 | end 13 | 14 | sen:call(bad_func, 'ni') 15 | -------------------------------------------------------------------------------- /lib/sentry.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local raven = require 'raven' 6 | local sender_luasocket = require 'raven.senders.ngx' 7 | 8 | local _M = {} 9 | 10 | function _M.new(dsn) 11 | local self = {} 12 | local sender, err = sender_luasocket.new({ 13 | dsn = dsn 14 | }) 15 | local rvn = raven.new({ 16 | sender = sender, 17 | tags = { 18 | foo = "bar" 19 | }, 20 | }) 21 | self.rvn = rvn 22 | return setmetatable(self, {__index = _M}) 23 | end 24 | 25 | -- 26 | -- Send a message to sentry 27 | -- 28 | function _M.message(self, content, tags) 29 | local id, err = self.rvn:captureMessage(content, { 30 | tags = tags 31 | }) 32 | if not id then 33 | return nil, err 34 | end 35 | return true 36 | end 37 | 38 | -- 39 | -- Send an exception to sentry 40 | -- 41 | function _M.exception(self, _type, value, module, tags) 42 | local exception = {{ 43 | type = _type, 44 | value = value, 45 | module = module 46 | }} 47 | local id, err = self.rvn:captureException(exception, { 48 | tags = tags 49 | }) 50 | if not id then 51 | return nil, err 52 | end 53 | return true 54 | end 55 | 56 | -- 57 | -- Catch an exception and send it to sentry 58 | -- 59 | function _M.call(self, func, ...) 60 | -- 61 | -- variable 'ok' should be false, and an exception will be sent to sentry 62 | -- 63 | return self.rvn:call(func, ...) 64 | end 65 | 66 | return _M 67 | -------------------------------------------------------------------------------- /lib/stat.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20181016 3 | -- 4 | 5 | local serpent = require 'serpent' 6 | local lup = require 'lib.lup' 7 | require 'resty.core' 8 | 9 | local _M = {} 10 | 11 | local started = false 12 | local _project = nil 13 | 14 | local function _callback(delay) 15 | ngx.update_time() 16 | local time = ngx.time() 17 | local stat_time = os.date("%Y-%m-%dT%H:%M:%S%z", time) -- iso format date 18 | local s = { 19 | project = _project, 20 | stat_time = stat_time, 21 | stat_delay = delay, 22 | hostname = lup.gethostname(), 23 | } 24 | local val, last, count, last_count = 0, 0, 0, 0, 0 25 | for _, key in ipairs(ngx.shared.stat:get_keys()) do 26 | if not ngx.re.match(key, '@', 'jo') then 27 | val = ngx.shared.stat:get(key) or 0 28 | s[key] = val 29 | if type(val) == 'number' then 30 | last = ngx.shared.stat:get(key..'@last') or 0 31 | delta = val - last 32 | s[key..'@delta'] = delta 33 | ngx.shared.stat:set(key..'@last', val) 34 | ngx.shared.stat:set(key..'@delta', delta) 35 | 36 | count = ngx.shared.stat:get(key..'@count') or 0 37 | last_count = ngx.shared.stat:get(key..'@last_count') or 0 38 | count_delta = count - last_count 39 | s[key..'@count_delta'] = count_delta 40 | ngx.shared.stat:set(key..'@last_count', count) 41 | ngx.shared.stat:set(key..'@count_delta', count_delta) 42 | end 43 | end 44 | end 45 | local delta, avg = 0 46 | for k, v in pairs(s) do 47 | if not ngx.re.match(k, '@', 'jo') and type(v) == 'number' then 48 | delta = s[k..'@delta'] 49 | if delta then 50 | count = s[k..'@count_delta'] 51 | if count ~= 0 then 52 | avg = delta / count 53 | s[k..'@avg'] = avg 54 | ngx.shared.stat:set(k..'@avg', avg) 55 | end 56 | end 57 | end 58 | end 59 | ngx.log(ngx.ERR, lup.json_encode(s)) 60 | end 61 | 62 | function _M.get_keys(keys) 63 | local s = {} 64 | for _, key in ipairs(keys) do 65 | for _, suffix in ipairs({'', '@delta', '@last'}) do 66 | s[key..suffix] = ngx.shared.stat:get(key..suffix) 67 | end 68 | end 69 | return s 70 | end 71 | 72 | function _M.start(before_callback, project) 73 | 74 | _project = project or 'UNKNOWN_PROJECT' 75 | assert(started == false, 'ALREADY_STARTED') 76 | local delay = 60 77 | 78 | ngx.timer.every(delay, function() 79 | if before_callback then 80 | before_callback() 81 | end 82 | _callback(delay) 83 | end) 84 | ngx.log(ngx.ERR, 'STAT_STARTED') 85 | 86 | started = true 87 | end 88 | 89 | function _M.set(self, key, value) 90 | assert(ngx.re.match(key, '^[a-z0-9_]+$', 'jo'), 'KEY_FORMAT_ERR') 91 | assert(({number=1, string=1})[type(value)], 'VALUE_FORMAT_ERR') 92 | if self.name then 93 | key = self.name..'.'..key 94 | end 95 | ngx.shared.stat:set(key, value) 96 | end 97 | 98 | function _M.get(self, key) 99 | assert(ngx.re.match(key, '^[a-z0-9_@]+$', 'jo'), 'KEY_FORMAT_ERR') 100 | if self.name then 101 | key = self.name..'.'..key 102 | end 103 | return ngx.shared.stat:get(key) 104 | end 105 | 106 | function _M.incr(self, key, value, default) 107 | assert(ngx.re.match(key, '^[a-z0-9_]+$', 'jo'), 'KEY_FORMAT_ERR') 108 | assert(type(value) == 'number', 'VALUE_FORMAT_ERR') 109 | if self.name then 110 | key = self.name..'.'..key 111 | end 112 | -- 113 | -- 精度控制在小数点后6位 114 | -- 115 | ngx.shared.stat:incr(key, value - value % 0.000001, default or 0) 116 | ngx.shared.stat:incr(key..'@count', 1, 0) 117 | end 118 | 119 | function _M.get_metrics() 120 | local key, metric, name, line, filter = nil, nil, nil, nil, nil 121 | local keys = ngx.shared.stat:get_keys() 122 | local ret = table.new(0, 1024) 123 | for _, key in ipairs(keys) do 124 | local k = lup.explode('.', key) 125 | if k[2] then 126 | metric, name = k[2], k[1] 127 | filter = serpent.line(ngx.decode_args(name), {comment = false, compact = true}) 128 | else 129 | metric = k[1] 130 | end 131 | ret[#ret + 1] = { 132 | metric = metric, 133 | filter = filter, 134 | value = ngx.shared.stat:get(key) or -1, 135 | } 136 | end 137 | return ret 138 | end 139 | 140 | function _M.new(name) 141 | if name then 142 | local pattern = '^[a-z0-9_]+=[a-z0-9_]+(&[a-z0-9_]+=[a-z0-9_]+)*$' 143 | assert(ngx.re.match(name, pattern, 'jo'), 'NAME_FORMAT_ERR') 144 | end 145 | return setmetatable({name = name}, {__index = _M}) 146 | end 147 | 148 | -- "recv": 37432010, 149 | -- "recv@cd": 1164, 150 | -- "recv@delta": 37430944, 151 | -- "sent": 604834877, 152 | -- "sent@cd": 1164, 153 | -- "sent@delta": 604833811, 154 | -- "stat_delay": 60, 155 | -- "stat_time": "2019-09-23T14:24:32+0800", 156 | -- "hostname": "nvm-json-uat-01", 157 | -- "project": "uposgate", 158 | 159 | -- nvme_status{device="/dev/nvme0n1",err="critical_composite_temperature_time"} 0 160 | -- nvme_status{device="/dev/nvme0n1",err="critical_warning"} 0 161 | -- nvme_status{device="/dev/nvme0n1",err="media_errors"} 0 162 | -- nvme_status{device="/dev/nvme0n1",err="num_err_log_entries"} 0 163 | -- nvme_status{device="/dev/nvme0n1",err="percentage_used"} 28 164 | -- nvme_status{device="/dev/nvme0n1",err="warning_temperature_time"} 0 165 | -- nvme_status{device="/dev/nvme1n1",err="critical_composite_temperature_time"} 0 166 | -- nvme_status{device="/dev/nvme1n1",err="critical_warning"} 0 167 | -- nvme_status{device="/dev/nvme1n1",err="media_errors"} 0 168 | -- nvme_status{device="/dev/nvme1n1",err="num_err_log_entries"} 0 169 | -- nvme_status{device="/dev/nvme1n1",err="percentage_used"} 19 170 | -- nvme_status{device="/dev/nvme1n1",err="warning_temperature_time"} 0 171 | 172 | 173 | return _M 174 | -------------------------------------------------------------------------------- /lib/swoole.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- User: tangjunxing 3 | -- Date : 2017/09/14 4 | -- 5 | -- lua-users wiki: Multi Tasking 6 | -- http://lua-users.org/wiki/MultiTasking 7 | -- 8 | 9 | local cjson = require 'cjson.safe' 10 | local _M = {} 11 | local _instance = nil 12 | _M.__index = _M 13 | 14 | function _M.new() 15 | return setmetatable({ 16 | shared = ngx.shared.swoole, 17 | worker_id = ngx.worker.id(), 18 | worker_pid = ngx.worker.pid(), 19 | setting = {}, 20 | events = {}, 21 | }, _M) 22 | end 23 | 24 | function _M.get_instance() 25 | if not _instance then 26 | _instance = _M.new() 27 | end 28 | return _instance 29 | end 30 | 31 | local function handler(premature, self) 32 | if premature then 33 | return 34 | end 35 | while not ngx.worker.exiting() do 36 | local waiting = self.shared:rpop('waiting') 37 | if waiting then 38 | local task = cjson.decode(waiting) or {} 39 | local task_data = task.data 40 | local task_callback = self.events.task 41 | local finish_callback = self.events.finish 42 | local worker_error_callback = self.events.worker_error 43 | self.shared:set('worker_'..self.worker_id, waiting, 600) 44 | 45 | local ok, ret = xpcall(task_callback, function(err) 46 | return { 47 | err = err, 48 | traceback = debug.traceback() 49 | } 50 | end, self, task.id, self.worker_id, task_data) 51 | if not ok then 52 | ret = ret or {} 53 | ngx.log(ngx.ERR, cjson.encode{ 54 | err_type = 'SWOOLE_ERROR', 55 | err = ret.err, 56 | traceback = ret.traceback, 57 | }) 58 | end 59 | 60 | self.shared:delete('worker_'..self.worker_id) 61 | if finish_callback then 62 | finish_callback(self, task.id, self.worker_id, ret) 63 | end 64 | -- self.err = ret.err 65 | -- if worker_error_callback then 66 | -- worker_error_callback(self, task.id, self.worker_id, ret.err) 67 | -- end 68 | else 69 | ngx.sleep(2) 70 | end 71 | end 72 | end 73 | 74 | function _M.start(self) 75 | if self.worker_id and tonumber(self.worker_id) < self.setting.task_worker_num then 76 | ngx.timer.at(0, handler, self) 77 | return true 78 | end 79 | return false 80 | end 81 | 82 | function _M.on(self, name, callback) 83 | if name == 'Task' then 84 | self.events.task = callback 85 | elseif name == 'Finish' then 86 | self.events.finish = callback 87 | elseif name == 'WorkerError' then 88 | self.events.worker_error = callback 89 | else 90 | error('UNKNOWN_EVENT_NAME_'..name) 91 | end 92 | end 93 | 94 | function _M.stats(self) 95 | -- request_count => 1000, Server收到的请求次数 96 | -- start_time 服务器启动的时间 97 | -- connection_num 当前连接的数量 98 | -- accept_count 接受了多少个连接 99 | -- close_count 关闭的连接数量 100 | -- task_queue_num => 10, 101 | -- task_queue_bytes => 65536, 102 | -- worker_request_count => 当前Worker进程收到的请求次数 103 | -- tasking_num 当前正在排队的任务数 104 | -- timer_pending_count = ngx.timer.pending_count(), 105 | -- timer_running_count = ngx.timer.running_count(), 106 | require "resty.core.shdict" 107 | return { 108 | shared_free = self.shared.free_space and self.shared:free_space(), 109 | shared_capacity = self.shared.capacity and self.shared:capacity(), 110 | task_worker_num = self.setting.task_worker_num, 111 | worker_count = ngx.worker.count(), 112 | worker_exiting = ngx.worker.exiting(), 113 | task_count = #self.shared:get_keys(), 114 | waiting = self.shared:llen('waiting'), 115 | } 116 | end 117 | 118 | function _M.get_last_error(self) 119 | return self.err 120 | end 121 | 122 | function _M.task(self, data) 123 | 124 | if self:stats().waiting + 1 > 40480 then 125 | error('TASK_WAITING_40480') 126 | end 127 | 128 | local task_id = ngx.crc32_long(ngx.now() + os.clock()) 129 | local length, err = self.shared:lpush('waiting', cjson.encode({ 130 | id = task_id, 131 | data = data 132 | })) 133 | 134 | if not length then 135 | error(err) 136 | end 137 | 138 | return task_id 139 | end 140 | 141 | function _M.set(self, setting) 142 | for k, v in pairs(setting) do 143 | self.setting[k] = v 144 | end 145 | end 146 | 147 | return _M 148 | -------------------------------------------------------------------------------- /lib/tlcache.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20190926 3 | -- 4 | 5 | local mlcache = require 'resty.mlcache' 6 | -- local mongo = require 'lib.mongo_client' 7 | local lup = require 'lib.lup' 8 | local redis_client = require 'lib.redis_client' 9 | 10 | local _M = {} 11 | 12 | local function cache_by_redis(red, name, key, ttl, callback, ...) 13 | key = name..'-'..key 14 | local val, err = red:get(key) 15 | if val and val ~= ngx.null then 16 | return lup.json_decode(val)[1] 17 | else 18 | val = callback(...) 19 | if not val then 20 | return nil 21 | end 22 | if not err then 23 | local ok, errmsg = red:set(key, lup.json_encode({val})) 24 | if ok then 25 | -- ngx.log(ngx.ERR, 'UPDATE_REDIS_SUCC_', val_to_save) 26 | if ttl then 27 | red:expire(key, ttl) 28 | end 29 | else 30 | ngx.log(ngx.ERR, 'UPDATE_REDIS_ERR_', errmsg) 31 | end 32 | end 33 | end 34 | return val 35 | end 36 | 37 | local function cache_by_mongo(coll, name, key, ttl, callback, ...) 38 | -- 39 | -- 增加一个0-4秒的随机秒数，以防并发更新缓存 40 | -- 41 | local doc = coll:find_one({ 42 | name = name, 43 | key = key, 44 | timestamp = { 45 | ['$gt'] = ngx.time() - ttl - (math.floor(lup.rand()) % 5) 46 | } 47 | }, { 48 | name = 0, 49 | key = 0, 50 | timestamp = 0, 51 | }) 52 | 53 | local val = nil 54 | if doc then 55 | val = doc.val 56 | else 57 | val = callback(...) 58 | local ret, err = coll:find_and_modify({ 59 | name = name, 60 | key = key, 61 | }, { 62 | update = { 63 | name = name, 64 | key = key, 65 | val = val, 66 | timestamp = ngx.time(), 67 | }, 68 | upsert = true 69 | }) 70 | if ret then 71 | ngx.log(ngx.ERR, 'UPDATE_MONGO_SUCC_'..lup.json_encode(val)) 72 | else 73 | ngx.log(ngx.ERR, 'UPDATE_MONGO_ERR_'..err) 74 | end 75 | end 76 | return val 77 | end 78 | 79 | function _M.get(self, key, opts, callback, ...) 80 | opts = opts or {} 81 | local name = self.name 82 | local ttl_l3 = opts.ttl_l3 or self.opts.ttl_l3 or 10 83 | ngx.update_time() 84 | if self.coll then 85 | return self.mlc:get(key, opts, cache_by_mongo, self.coll, name, key, ttl_l3, callback, ...) 86 | elseif self.red then 87 | return self.mlc:get(key, opts, cache_by_redis, self.red, name, key, ttl_l3, callback, ...) 88 | end 89 | end 90 | 91 | function _M.new(name, shared_name, opts) 92 | assert(type(shared_name) == 'string', 'SHARED_NAME_ERROR') 93 | assert(type(opts) == 'table', 'OPTS_ERROR') 94 | 95 | -- 96 | -- 初始化句柄 97 | -- 98 | local mlc, coll, red = assert(mlcache.new(name, shared_name, opts)) 99 | if opts.mongo_config then 100 | coll = mongo.new(shared_name, opts.mongo_config) 101 | coll:ensure_index({{unique = true, key = {key = 1, name = 1}}, {key = {timestamp = 1}}}) 102 | elseif opts.redis_config then 103 | red = redis_client.new(opts.redis_config) 104 | else 105 | return mlc 106 | end 107 | 108 | return setmetatable({ 109 | name = name, 110 | shared_name = shared_name, 111 | opts = opts, 112 | mlc = mlc, 113 | coll = coll, 114 | red = red, 115 | }, {__index = _M}) 116 | 117 | end 118 | 119 | return _M 120 | -------------------------------------------------------------------------------- /lib/tlcache.test.lua: -------------------------------------------------------------------------------- 1 | local config = require 'config.config' 2 | local tlcache = require 'lib.tlcache' 3 | local lup = require 'lib.lup' 4 | 5 | local cache = tlcache.new('cache1', 'tlcache', { 6 | ttl = 10, 7 | neg_ttl = 10, 8 | mongo_config = config.mongo, 9 | -- redis_config = config.redis, 10 | }) 11 | 12 | while true do 13 | local val = cache:get('key1', nil, function() 14 | return { 15 | time = ngx.now(), 16 | ni = 'i', 17 | } 18 | end) 19 | lup.var_dump(val) 20 | ngx.sleep(1) 21 | end 22 | -------------------------------------------------------------------------------- /lib/zset.lua: -------------------------------------------------------------------------------- 1 | local skiplist = require "skiplist.c" 2 | 3 | local mt = {} 4 | mt.__index = mt 5 | 6 | function mt:add(score, member) 7 | local old = self.tbl[member] 8 | if old then 9 | if old == score then 10 | return 11 | end 12 | self.sl:delete(old, member) 13 | end 14 | 15 | self.sl:insert(score, member) 16 | self.tbl[member] = score 17 | end 18 | 19 | function mt:rem(member) 20 | local score = self.tbl[member] 21 | if score then 22 | self.sl:delete(score, member) 23 | self.tbl[member] = nil 24 | end 25 | end 26 | 27 | function mt:count() 28 | return self.sl:get_count() 29 | end 30 | 31 | function mt:_reverse_rank(r) 32 | return self.sl:get_count() - r + 1 33 | end 34 | 35 | function mt:limit(count, delete_handler) 36 | local total = self.sl:get_count() 37 | if total <= count then 38 | return 0 39 | end 40 | 41 | local delete_function = function(member) 42 | self.tbl[member] = nil 43 | if delete_handler then 44 | delete_handler(member) 45 | end 46 | end 47 | 48 | return self.sl:delete_by_rank(count+1, total, delete_function) 49 | end 50 | 51 | function mt:rev_limit(count, delete_handler) 52 | local total = self.sl:get_count() 53 | if total <= count then 54 | return 0 55 | end 56 | local from = self:_reverse_rank(count+1) 57 | local to = self:_reverse_rank(total) 58 | 59 | local delete_function = function(member) 60 | self.tbl[member] = nil 61 | if delete_handler then 62 | delete_handler(member) 63 | end 64 | end 65 | 66 | return self.sl:delete_by_rank(from, to, delete_function) 67 | end 68 | 69 | function mt:rev_range(r1, r2) 70 | local r1 = self:_reverse_rank(r1) 71 | local r2 = self:_reverse_rank(r2) 72 | return self:range(r1, r2) 73 | end 74 | 75 | function mt:range(r1, r2) 76 | if r1 < 1 then 77 | r1 = 1 78 | end 79 | 80 | if r2 < 1 then 81 | r2 = 1 82 | end 83 | return self.sl:get_rank_range(r1, r2) 84 | end 85 | 86 | function mt:rev_rank(member) 87 | local r = self:rank(member) 88 | if r then 89 | return self:_reverse_rank(r) 90 | end 91 | return r 92 | end 93 | 94 | function mt:rank(member) 95 | local score = self.tbl[member] 96 | if not score then 97 | return nil 98 | end 99 | return self.sl:get_rank(score, member) 100 | end 101 | 102 | function mt:range_by_score(s1, s2) 103 | return self.sl:get_score_range(s1, s2) 104 | end 105 | 106 | function mt:score(member) 107 | return self.tbl[member] 108 | end 109 | 110 | function mt:dump() 111 | self.sl:dump() 112 | end 113 | 114 | local M = {} 115 | function M.new() 116 | local obj = {} 117 | obj.sl = skiplist() 118 | obj.tbl = {} 119 | return setmetatable(obj, mt) 120 | end 121 | return M 122 | 123 | -------------------------------------------------------------------------------- /lib/zset.test.lua: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/env -S /usr/local/oresty/bin/resty -I ./lualib/ 2 | -- 3 | -- tjx@20200310 4 | -- 5 | 6 | local lup = require 'lib.lup' 7 | local zset = require 'lib.zset' 8 | 9 | local zs = zset.new() 10 | 11 | for i = 1, 10 do 12 | zs:add(1, 'key_'..math.random(0, 10)) 13 | end 14 | 15 | local arr = zs:range(0, 10) 16 | 17 | lup.var_dump(arr) 18 | -------------------------------------------------------------------------------- /lualib/classic.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- classic 3 | -- 4 | -- Copyright (c) 2014, rxi 5 | -- 6 | -- This module is free software; you can redistribute it and/or modify it under 7 | -- the terms of the MIT license. See LICENSE for details. 8 | -- 9 | 10 | 11 | local Object = {} 12 | Object.__index = Object 13 | 14 | 15 | function Object:new() 16 | end 17 | 18 | 19 | function Object:extend() 20 | local cls = {} 21 | for k, v in pairs(self) do 22 | if k:find("__") == 1 then 23 | cls[k] = v 24 | end 25 | end 26 | cls.__index = cls 27 | cls.super = self 28 | setmetatable(cls, self) 29 | return cls 30 | end 31 | 32 | 33 | function Object:implement(...) 34 | for _, cls in pairs({...}) do 35 | for k, v in pairs(cls) do 36 | if self[k] == nil and type(v) == "function" then 37 | self[k] = v 38 | end 39 | end 40 | end 41 | end 42 | 43 | 44 | function Object:is(T) 45 | local mt = getmetatable(self) 46 | while mt do 47 | if mt == T then 48 | return true 49 | end 50 | mt = getmetatable(mt) 51 | end 52 | return false 53 | end 54 | 55 | 56 | function Object:__tostring() 57 | return "Object" 58 | end 59 | 60 | 61 | function Object:__call(...) 62 | local obj = setmetatable({}, self) 63 | obj:new(...) 64 | return obj 65 | end 66 | 67 | 68 | return Object 69 | -------------------------------------------------------------------------------- /lualib/inotify.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilibili/oresty/1d0b4be38da37468ac6e08d6562f920b42ce3264/lualib/inotify.so -------------------------------------------------------------------------------- /lualib/lfs.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilibili/oresty/1d0b4be38da37468ac6e08d6562f920b42ce3264/lualib/lfs.so -------------------------------------------------------------------------------- /lualib/librestychash.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilibili/oresty/1d0b4be38da37468ac6e08d6562f920b42ce3264/lualib/librestychash.so -------------------------------------------------------------------------------- /lualib/multipart.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- User: tangjunxing 3 | -- Date : 2018/04/19 4 | -- From: https://github.com/Kong/lua-multipart/blob/master/src/multipart.lua 5 | -- 6 | 7 | local setmetatable = setmetatable 8 | local tostring = tostring 9 | local insert = table.insert 10 | local remove = table.remove 11 | local concat = table.concat 12 | local ipairs = ipairs 13 | local pairs = pairs 14 | local match = string.match 15 | local find = string.find 16 | local sub = string.sub 17 | 18 | 19 | local RANDOM_BOUNDARY = sub(tostring({}), 10) 20 | 21 | 22 | local MultipartData = { RANDOM_BOUNDARY = RANDOM_BOUNDARY} 23 | 24 | 25 | MultipartData.__index = MultipartData 26 | 27 | 28 | setmetatable(MultipartData, { 29 | __call = function (cls, ...) 30 | return cls.new(...) 31 | end, 32 | }) 33 | 34 | 35 | local function is_header(value) 36 | return match(value, "(%S+):%s*(%S+)") 37 | end 38 | 39 | 40 | local function table_size(t) 41 | local res = 0 42 | 43 | if t then 44 | for _,_ in pairs(t) do 45 | res = res + 1 46 | end 47 | end 48 | 49 | return res 50 | end 51 | 52 | -- Create a table representation of multipart/data body 53 | -- 54 | -- @param {string} body The multipart/data string body 55 | -- @param {string} boundary The multipart/data boundary 56 | -- @return {table} Lua representation of the body 57 | local function decode(body, boundary) 58 | local result = { 59 | data = {}, 60 | indexes = {}, 61 | } 62 | 63 | if not boundary then 64 | return result 65 | end 66 | 67 | local part_name 68 | local part_index = 1 69 | local part_headers = {} 70 | local part_value = {} 71 | local part_value_ct = 0 72 | 73 | local end_boundary_length = boundary and #boundary + 2 74 | local processing_part_value = false 75 | 76 | local position = 1 77 | local done = false 78 | 79 | repeat 80 | local s = find(body, "[\r\n]", position) 81 | 82 | local line 83 | 84 | if s then 85 | line = sub(body, position, s - 1) 86 | position = s + 1 87 | 88 | else 89 | if position == 1 then 90 | line = body 91 | 92 | else 93 | line = sub(body, position) 94 | end 95 | 96 | done = true 97 | end 98 | 99 | if line == "" then 100 | if s and processing_part_value then 101 | part_value_ct = part_value_ct + 1 102 | part_value[part_value_ct] = sub(body, s, s) 103 | end 104 | 105 | else 106 | if sub(line, 1, 2) == "--" and sub(line, 3, end_boundary_length) == boundary then 107 | processing_part_value = false 108 | 109 | if part_name ~= nil then 110 | if part_value[part_value_ct] == "\n" then 111 | part_value[part_value_ct] = nil 112 | end 113 | 114 | if part_value[part_value_ct - 1] == "\r" then 115 | part_value[part_value_ct - 1] = nil 116 | end 117 | 118 | result.data[part_index] = { 119 | name = part_name, 120 | headers = part_headers, 121 | value = concat(part_value) 122 | } 123 | 124 | result.indexes[part_name] = part_index 125 | 126 | -- Reset fields for the next part 127 | part_headers = {} 128 | part_value = {} 129 | part_value_ct = 0 130 | part_name = nil 131 | part_index = part_index + 1 132 | end 133 | 134 | else 135 | --Beginning of part 136 | if not processing_part_value and line:sub(1, 19):lower() == "content-disposition" then 137 | -- Extract part_name 138 | for v in line:gmatch("[^;]+") do 139 | if not is_header(v) then -- If it's not content disposition part 140 | local pos = v:match("^%s*[Nn][Aa][Mm][Ee]=()") 141 | if pos then 142 | local current_value = v:match("^%s*([^=]*)", pos):gsub("%s*$", "") 143 | part_name = sub(current_value, 2, #current_value - 1) 144 | end 145 | end 146 | end 147 | 148 | insert(part_headers, line) 149 | 150 | if s and sub(body, s, s + 3) == "\r\n\r\n" then 151 | processing_part_value = true 152 | position = s + 4 153 | end 154 | 155 | elseif not processing_part_value and is_header(line) then 156 | insert(part_headers, line) 157 | 158 | if s and sub(body, s, s + 3) == "\r\n\r\n" then 159 | processing_part_value = true 160 | position = s + 4 161 | end 162 | 163 | else 164 | processing_part_value = true 165 | 166 | -- The value part begins 167 | part_value_ct = part_value_ct + 1 168 | part_value[part_value_ct] = line 169 | 170 | if s then 171 | part_value_ct = part_value_ct + 1 172 | part_value[part_value_ct] = sub(body, s, s) 173 | end 174 | end 175 | end 176 | end 177 | 178 | until done 179 | 180 | if part_name ~= nil then 181 | result.data[part_index] = { 182 | name = part_name, 183 | headers = part_headers, 184 | value = concat(part_value) 185 | } 186 | 187 | result.indexes[part_name] = part_index 188 | end 189 | 190 | return result 191 | end 192 | 193 | -- Creates a multipart/data body from a table 194 | -- 195 | -- @param {table} t The table that contains the multipart/data body properties 196 | -- @param {boundary} boundary The multipart/data boundary to use 197 | -- @return {string} The multipart/data string body 198 | local function encode(t, boundary) 199 | if not boundary then 200 | boundary = RANDOM_BOUNDARY 201 | end 202 | 203 | local result = {} 204 | local i = 0 205 | 206 | for _, v in ipairs(t.data) do 207 | if v.value then 208 | result[i + 1] = "--" 209 | result[i + 2] = boundary 210 | result[i + 3] = "\r\n" 211 | 212 | i = i + 3 213 | 214 | for _, header in ipairs(v.headers) do 215 | result[i + 1] = header 216 | result[i + 2] = "\r\n" 217 | 218 | i = i + 2 219 | end 220 | 221 | result[i + 1] = "\r\n" 222 | result[i + 2] = v.value 223 | result[i + 3] = "\r\n" 224 | 225 | i = i + 3 226 | end 227 | end 228 | 229 | if i == 0 then 230 | return "" 231 | end 232 | 233 | result[i + 1] = "--" 234 | result[i + 2] = boundary 235 | result[i + 3] = "--\r\n" 236 | 237 | return concat(result) 238 | end 239 | 240 | 241 | function MultipartData.new(data, content_type) 242 | local instance = setmetatable({}, MultipartData) 243 | 244 | if content_type then 245 | local boundary = match(content_type, ";%s*boundary=(%S+)") 246 | if boundary then 247 | if (sub(boundary, 1, 1) == '"' and sub(boundary, -1) == '"') or 248 | (sub(boundary, 1, 1) == "'" and sub(boundary, -1) == "'") then 249 | boundary = sub(boundary, 2, -2) 250 | end 251 | 252 | if boundary ~= "" then 253 | instance._boundary = boundary 254 | end 255 | end 256 | end 257 | 258 | instance._data = decode(data or "", instance._boundary) 259 | 260 | return instance 261 | end 262 | 263 | 264 | function MultipartData:get(name) 265 | return self._data.data[self._data.indexes[name]] 266 | end 267 | 268 | 269 | function MultipartData:get_all() 270 | local result = {} 271 | 272 | for k, v in pairs(self._data.indexes) do 273 | result[k] = self._data.data[v].value 274 | end 275 | 276 | return result 277 | end 278 | 279 | 280 | function MultipartData:set_simple(name, value) 281 | if self._data.indexes[name] then 282 | self._data.data[self._data.indexes[name]] = { 283 | name = name, 284 | value = value, 285 | headers = { 'Content-Disposition: form-data; name="' .. name .. '"' } 286 | } 287 | 288 | else 289 | local part_index = table_size(self._data.indexes) + 1 290 | self._data.indexes[name] = part_index 291 | self._data.data[part_index] = { 292 | name = name, 293 | value = value, 294 | headers = { 'Content-Disposition: form-data; name="' .. name .. '"' } 295 | } 296 | end 297 | end 298 | 299 | 300 | function MultipartData:delete(name) 301 | local index = self._data.indexes[name] 302 | 303 | if index then 304 | remove(self._data.data, index) 305 | self._data.indexes[name] = nil 306 | end 307 | end 308 | 309 | 310 | function MultipartData:tostring() 311 | return encode(self._data, self._boundary) 312 | end 313 | 314 | 315 | return MultipartData 316 | -------------------------------------------------------------------------------- /lualib/raven/senders/luasocket.lua: -------------------------------------------------------------------------------- 1 | -- vim: st=4 sts=4 sw=4 et: 2 | --- Network backend using [LuaSocket](http://w3.impa.br/~diego/software/luasocket/home.html). 3 | -- This module should be used when the LuaSocket library is available. Note 4 | -- that the HTTPS support depends on the [LuaSec](https://github.com/brunoos/luasec) 5 | -- library. This libary is not required for plain HTTP. 6 | -- 7 | -- @module raven.senders.luasocket 8 | -- @copyright 2014-2017 CloudFlare, Inc. 9 | -- @license BSD 3-clause (see LICENSE file) 10 | 11 | local util = require 'raven.util' 12 | local http = require 'socket.http' 13 | local ltn12 = require 'ltn12' 14 | 15 | -- try to load luassl (not mandatory, so do not hard fail if the module is 16 | -- not there 17 | local _, https = pcall(require, 'ssl.https') 18 | 19 | local assert = assert 20 | local pairs = pairs 21 | local setmetatable = setmetatable 22 | local table_concat = table.concat 23 | local source_string = ltn12.source.string 24 | local table_sink = ltn12.sink.table 25 | local parse_dsn = util.parse_dsn 26 | local generate_auth_header = util.generate_auth_header 27 | local _VERSION = util._VERSION 28 | local _M = {} 29 | 30 | local mt = {} 31 | mt.__index = mt 32 | 33 | function mt:send(json_str) 34 | local resp_buffer = {} 35 | local opts = { 36 | method = "POST", 37 | url = self.server, 38 | headers = { 39 | ['Content-Type'] = 'applicaion/json', 40 | ['User-Agent'] = "raven-lua-socket/" .. _VERSION, 41 | ['X-Sentry-Auth'] = generate_auth_header(self), 42 | ["Content-Length"] = tostring(#json_str), 43 | }, 44 | source = source_string(json_str), 45 | sink = table_sink(resp_buffer), 46 | } 47 | 48 | -- set master opts (if any) 49 | if self.opts then 50 | for h, v in pairs(self.opts) do 51 | opts[h] = v 52 | end 53 | end 54 | 55 | local ok, code = self.factory(opts) 56 | if not ok then 57 | return nil, code 58 | end 59 | if code ~= 200 then 60 | return nil, table_concat(resp_buffer) 61 | end 62 | return true 63 | end 64 | 65 | --- Configuration table for the nginx sender. 66 | -- @field dsn DSN string 67 | -- @field verify_ssl Whether or not the SSL certificate is checked (boolean, 68 | -- defaults to false) 69 | -- @field cafile Path to a CA bundle (see the `cafile` parameter in the 70 | -- [newcontext](https://github.com/brunoos/luasec/wiki/LuaSec-0.6#ssl_newcontext) 71 | -- docs) 72 | -- @table sender_conf 73 | 74 | --- Create a new sender object for the given DSN 75 | -- @param conf Configuration table, see @{sender_conf} 76 | -- @return A sender object 77 | function _M.new(conf) 78 | local obj, err = parse_dsn(conf.dsn) 79 | if not obj then 80 | return nil, err 81 | end 82 | 83 | if obj.protocol == 'https' then 84 | assert(https, 'LuaSec is required to use HTTPS transport') 85 | obj.factory = https.request 86 | obj.opts = { 87 | verify = conf.verify_ssl and 'peer' or 'none', 88 | cafile = conf.verify_ssl and conf.cafile or nil, 89 | protocol = 'tlsv1_2', 90 | } 91 | else 92 | obj.factory = http.request 93 | end 94 | 95 | return setmetatable(obj, mt) 96 | end 97 | 98 | return _M 99 | 100 | -------------------------------------------------------------------------------- /lualib/raven/senders/ngx.lua: -------------------------------------------------------------------------------- 1 | -- vim: st=4 sts=4 sw=4 et: 2 | --- Network backend using the [lua-nginx-module](https://github.com/openresty/lua-nginx-module) cosocket API. 3 | -- This module can be used with the raw Lua module for nginx or the OpenResty 4 | -- bundle. 5 | -- 6 | -- As socket API is not available in all contexts, the messages are queued and 7 | -- processed in a timer when necessary. This can be forced for all messages too 8 | -- in order to not wait for the message to be sent (see the `async` filed). 9 | -- 10 | -- It will require the `lua_ssl_trusted_certificate` to be set correctly when 11 | -- reporting to a HTTPS endpoint. 12 | -- 13 | -- @module raven.senders.ngx 14 | -- @copyright 2014-2017 CloudFlare, Inc. 15 | -- @license BSD 3-clause (see LICENSE file) 16 | 17 | -- luacheck: globals ngx 18 | 19 | local util = require 'raven.util' 20 | 21 | local ngx_socket = ngx.socket 22 | local ngx_get_phase = ngx.get_phase 23 | local string_format = string.format 24 | local table_remove = table.remove 25 | local parse_dsn = util.parse_dsn 26 | local generate_auth_header = util.generate_auth_header 27 | local _VERSION = util._VERSION 28 | local _M = {} 29 | 30 | -- provide a more sensible implementation of the error log function 31 | function util.errlog(...) 32 | ngx.log(ngx.ERR, 'raven-lua failure: ', ...) 33 | end 34 | 35 | -- as we don't want to use an external HTTP library, just send the HTTP request 36 | -- directly using cosocket API 37 | local HTTP_REQUEST = string.gsub([[POST %s HTTP/1.0 38 | Host: %s 39 | Connection: close 40 | Content-Type: application/json 41 | Content-Length: %d 42 | User-Agent: %s 43 | X-Sentry-Auth: %s 44 | 45 | %s 46 | ]], '\r?\n', '\r\n') 47 | 48 | local CALLBACK_DEFAULT_ERRMSG = 49 | "failed to onfigure socket (custom callback did not returned a value)" 50 | 51 | local function send_msg(self, msg) 52 | local ok 53 | local sock, err = ngx_socket.tcp() 54 | if not sock then 55 | return nil, err 56 | end 57 | 58 | if self.configure_socket then 59 | ok, err = self.configure_socket(self, sock) 60 | if not ok then 61 | return nil, err or CALLBACK_DEFAULT_ERRMSG 62 | end 63 | end 64 | 65 | ok, err = sock:connect(self.target, self.port) 66 | if not ok then 67 | return nil, err 68 | end 69 | 70 | if self.protocol == 'https' then 71 | -- TODO: session resumption? 72 | ok, err = sock:sslhandshake(false, self.host, self.verify_ssl) 73 | if not ok then 74 | sock:close() 75 | return nil, err 76 | end 77 | end 78 | 79 | ok, err = sock:send(msg) 80 | if not ok then 81 | sock:close() 82 | return nil, err 83 | end 84 | 85 | local resp 86 | resp, err = sock:receive('*a') 87 | sock:close() 88 | if not resp then 89 | return nil, err 90 | end 91 | 92 | local status = resp:match("HTTP/%d%.%d (%d%d%d) %w+") 93 | if status ~= "200" then 94 | return nil, "Server response status not 200:" .. (status or "nil") 95 | end 96 | 97 | return resp:match("\r\n\r\n(.*)") or "" 98 | end 99 | 100 | -- async task pushing: sometimes the socket API is not available, in this case, 101 | -- messages are pushed into a queue and sent in a timer. Only one timer at most 102 | -- is running per instance (and per worker): timers are a scarce resource, we 103 | -- don't want to exhaust the timer pool during message storms. 104 | 105 | local function send_task(premature, self) 106 | if premature then 107 | return 108 | end 109 | 110 | local ok, err = xpcall(function() 111 | local send_queue = self.send_queue 112 | while #send_queue > 0 do 113 | local msg = send_queue[1] 114 | -- do not remove the message yet as an empty queue is the signal to 115 | -- re-start the task 116 | local ok, err = send_msg(self, msg) 117 | if not ok then 118 | ngx.log(ngx.ERR, 'raven: failed to send message asyncronously: ', 119 | err, '. Drop the message.') 120 | end 121 | table_remove(send_queue, 1) 122 | end 123 | end, debug.traceback) 124 | 125 | if not ok then 126 | ngx.log(ngx.ERR, 'raven: failed to run the async sender task: ', err) 127 | -- TODO: restart the task here? requires a extra counter as we don't want 128 | -- to fail in loop indefinitely. 129 | end 130 | self.task_running = false 131 | end 132 | 133 | 134 | local mt = {} 135 | mt.__index = mt 136 | 137 | function mt:send(json_str) 138 | local auth = generate_auth_header(self) 139 | local msg = string_format(HTTP_REQUEST, self.request_uri, self.host, #json_str, 140 | "raven-lua-ngx/" .. _VERSION, auth, json_str) 141 | local phase = ngx_get_phase() 142 | -- rewrite_by_lua*, access_by_lua*, content_by_lua*, ngx.timer.*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua* 143 | if (not self.async) and ( 144 | phase == 'rewrite' or 145 | phase == 'access' or 146 | phase == 'content' or 147 | phase == 'timer' or 148 | phase == 'ssl_cert' or 149 | phase == 'ssl_session_fetch' 150 | ) then 151 | -- socket is available 152 | return send_msg(self, msg) 153 | else 154 | -- cannot use socket: push the message in the async queue 155 | local send_queue = self.send_queue 156 | local queue_size = #send_queue 157 | if queue_size <= self.queue_limit then 158 | send_queue[queue_size + 1] = msg 159 | if not self.task_running then 160 | local ok, err = ngx.timer.at(0, send_task, self) 161 | if not ok then 162 | return nil, 'failed to schedule async sender task: ' .. err 163 | end 164 | 165 | -- assume the task is already running, as other messages might 166 | -- be reported before it is actually scheduled and run. 167 | self.task_running = true 168 | end 169 | else 170 | return nil, 'failed to send message asyncronously: queue is full' 171 | end 172 | return true 173 | end 174 | end 175 | 176 | --- Configuration table for the nginx sender. 177 | -- @field dsn DSN string 178 | -- @field verify_ssl Whether or not the SSL certificate is checked (boolean, 179 | -- defaults to false) 180 | -- @field configure_socket Callback used to configure the created socket (e.g. 181 | -- adjusting the timeout). Called with the sender object and socket as arguments. 182 | -- Must return `true` or `nil, error_message`. 183 | -- @field queue_limit Maximum number of message in the asynchronous sending queue 184 | -- (default: 50) 185 | -- @field async Always send message asynchronously, even when it can be sent 186 | -- right away. This is to prevent to slow down processing while contacting the 187 | -- Sentry server. (default: false) 188 | -- @table sender_conf 189 | 190 | --- Create a new sender object for the given DSN 191 | -- @param conf Configuration table, see @{sender_conf} 192 | -- @return A sender object 193 | function _M.new(conf) 194 | local obj, err = parse_dsn(conf.dsn) 195 | if not obj then 196 | return nil, err 197 | end 198 | 199 | obj.target = conf.target or obj.host 200 | obj.verify_ssl = conf.verify_ssl 201 | obj.configure_socket = conf.configure_socket 202 | obj.queue_limit = conf.queue_limit or 50 203 | obj.async = conf.async or false 204 | obj.send_queue = {} 205 | obj.task_running = false 206 | 207 | return setmetatable(obj, mt) 208 | end 209 | 210 | --- Returns the value of the `server_name` variable if possible. 211 | -- Otherwise (wrong phase), this will return "undefined". 212 | -- 213 | -- It is intended to be used as a `get_server_name` override on the main raven 214 | -- instance. 215 | -- 216 | -- @usage 217 | -- local raven_ngx = require("raven.senders.ngx") 218 | -- local rvn = raven.new(...) 219 | -- rvn.get_server_name = raven_ngx.get_server_name 220 | function _M.get_server_name() 221 | local phase = ngx.get_phase() 222 | -- the ngx.var.* API is not available in all contexts 223 | if phase == "set" or 224 | phase == "rewrite" or 225 | phase == "access" or 226 | phase == "content" or 227 | phase == "header_filter" or 228 | phase == "body_filter" or 229 | phase == "log" 230 | then 231 | return ngx.var.server_name 232 | end 233 | return "undefined" 234 | end 235 | 236 | return _M 237 | -------------------------------------------------------------------------------- /lualib/raven/senders/reference.lua: -------------------------------------------------------------------------------- 1 | -- vim: st=4 sts=4 sw=4 et: 2 | --- Sender object specification. 3 | -- A Lua has a disparate ecosystem of networking libraries, this makes difficult 4 | -- doing a one-size-fits-all library. Lua-Raven does as much logic as possible 5 | -- in a generic way, relying on a *sender* object to send the actual payload to 6 | -- the Sentry server. 7 | -- 8 | -- This module only contains documentation, it does not implement an actual 9 | -- sender. 10 | -- 11 | -- @module raven.senders.reference 12 | -- @copyright 2014-2017 CloudFlare, Inc. 13 | -- @license BSD 3-clause (see LICENSE file) 14 | 15 | --- A sender object can be a table or a userdata. 16 | -- I must contain all the logic to locate the server and send the data to it. 17 | -- Notably, the core Raven object has no idea of the URL of the server, the 18 | -- sender object must be able to build it by itself. 19 | -- 20 | -- You can use the @{raven.util.parse_dsn} function to process DSN strings. 21 | -- 22 | -- @type Sender 23 | 24 | --- The `send` method is called whenever a message must be sent to the Sentry 25 | -- server. The authentication header can be generated with 26 | -- @{raven.util.generate_auth_header}. 27 | -- 28 | -- @param json_str Request payload as a serialized string (it must be sent 29 | -- verbatim, the sender should **not** try to alter it in any way) 30 | -- @return[1] `true` when the payload has been sent successfully 31 | -- @return[2] `nil` in case of error 32 | -- @return[2] error message (will be logged) 33 | -- @function Sender:send 34 | 35 | return 36 | -------------------------------------------------------------------------------- /lualib/raven/senders/test.lua: -------------------------------------------------------------------------------- 1 | -- vim: st=4 sts=4 sw=4 et: 2 | -- Test sender for running the test suite without spawning an actual server. 3 | -- 4 | -- @module raven.senders.test 5 | -- @copyright 2014-2017 CloudFlare, Inc. 6 | -- @license BSD 3-clause (see LICENSE file) 7 | local cjson = require 'cjson' 8 | 9 | 10 | local function new() 11 | return { 12 | events = {}, 13 | send = function(self, json_str) 14 | table.insert(self.events, cjson.decode(json_str)) 15 | return true 16 | end, 17 | } 18 | end 19 | 20 | return { 21 | new = new, 22 | } 23 | -------------------------------------------------------------------------------- /lualib/raven/util.lua: -------------------------------------------------------------------------------- 1 | -- vim: st=4 sts=4 sw=4 et: 2 | --- Utility functions module. 3 | -- This is mostly used for internal stuff and should not be relevant for end 4 | -- users (except some function that can be overridden). 5 | -- 6 | -- @module raven.util 7 | -- @copyright 2014-2017 CloudFlare, Inc. 8 | -- @license BSD 3-clause (see LICENSE file) 9 | 10 | local string_format = string.format 11 | local string_find = string.find 12 | local string_sub = string.sub 13 | local string_match = string.match 14 | local math_random = math.random 15 | local os_date = os.date 16 | 17 | local _M = {} 18 | 19 | local _VERSION = "0.5.0" 20 | _M._VERSION = _VERSION 21 | 22 | --- Used to log errors during reporting. 23 | -- The default implementation is quite dumb: it will simply use `print` to 24 | -- display the error message. Users are encouraged to override this 25 | -- implementation to something smarter. 26 | -- @param ... Message to log (will be concatenated) 27 | function _M.errlog(...) 28 | print("[ERROR]", ...) 29 | end 30 | 31 | --- Returns a string suitable to be used as `event_id`. 32 | -- @return a new random `event_id` string. 33 | function _M.generate_event_id() 34 | -- Some version of Lua can only generate random integers up to 2^31, so we are limited to 7 35 | -- hex-digits per call 36 | return string_format("%07x%07x%07x%07x%04x", 37 | math_random(0, 0xfffffff), 38 | math_random(0, 0xfffffff), 39 | math_random(0, 0xfffffff), 40 | math_random(0, 0xfffffff), 41 | math_random(0, 0xffff)) 42 | end 43 | 44 | --- Returns the current date/time in ISO8601 format with no timezone indicator 45 | -- but in UTC. 46 | function _M.iso8601() 47 | 48 | -- The ! forces os_date to return UTC. Don't change this to use 49 | -- os.date/os.time to format the date/time because timezone 50 | -- problems occur 51 | 52 | local t = os_date("!*t") 53 | return string_format("%04d-%02d-%02dT%02d:%02d:%02d", 54 | t["year"], t["month"], t["day"], t["hour"], t["min"], t["sec"]) 55 | end 56 | 57 | local iso8601 = _M.iso8601 58 | 59 | -- parse_host_port: parse long host ("127.0.0.1:2222") 60 | -- to host ("127.0.0.1") and port (2222) 61 | local function parse_host_port(protocol, host) 62 | local i = string_find(host, ":") 63 | if not i then 64 | return host, protocol == 'https' and 443 or 80 65 | end 66 | 67 | local port_str = string_sub(host, i + 1) 68 | local port = tonumber(port_str) 69 | if not port then 70 | return nil, nil, "illegal port: " .. port_str 71 | end 72 | 73 | return string_sub(host, 1, i - 1), port 74 | end 75 | 76 | --- Parsed DSN table containing its different fields. 77 | -- @field protocol Connection protocol (`http`, `https`, ...) 78 | -- @field public_key 79 | -- @field secret_key 80 | -- @field host Hostname (without the port part) 81 | -- @field port Port to connect to as a number (always filled, default 82 | -- depend on protocol) 83 | -- @field path 84 | -- @field project_id 85 | -- @field request_uri URI path for storing messages 86 | -- @field server Full URL to the `/store/` Sentry endpoint 87 | -- @table parsed_dsn 88 | 89 | --- Parses a DSN and returns a table with parsed elements. 90 | -- @param dsn DSN to parse (string) 91 | -- @param[opt] obj Table to populate (create a new one if not provided) 92 | -- @return[1] populated table, see @{parsed_dsn} 93 | -- @return[2] nil 94 | -- @return[2] error message 95 | function _M.parse_dsn(dsn, obj) 96 | if not obj then 97 | obj = {} 98 | end 99 | 100 | assert(type(obj) == "table") 101 | 102 | -- '{PROTOCOL}://{PUBLIC_KEY}:{SECRET_KEY}@{HOST}/{PATH}{PROJECT_ID}' 103 | obj.protocol, obj.public_key, obj.secret_key, obj.long_host, 104 | obj.path, obj.project_id = 105 | string_match(dsn, "^([^:]+)://([^:]+):([^@]+)@([^/]+)(.*/)(.+)$") 106 | 107 | if obj.protocol and obj.public_key and obj.secret_key and obj.long_host 108 | and obj.project_id then 109 | 110 | local host, port, err = parse_host_port(obj.protocol, obj.long_host) 111 | 112 | if not host then 113 | return nil, err 114 | end 115 | 116 | obj.host = host 117 | obj.port = port 118 | 119 | obj.request_uri = string_format("%sapi/%s/store/", obj.path, obj.project_id) 120 | obj.server = string_format("%s://%s:%d%s", obj.protocol, obj.host, obj.port, 121 | obj.request_uri) 122 | 123 | return obj 124 | end 125 | 126 | return nil, "failed to parse DSN string" 127 | end 128 | 129 | --- Generate a Sentry compliant `X-Sentry-Auth` header value. 130 | -- @param dsn_object A @{parsed_dsn} table 131 | -- @return A Sentry authentication string 132 | function _M.generate_auth_header(dsn_object) 133 | return string_format( 134 | "Sentry sentry_version=6, sentry_client=%s, sentry_timestamp=%s, sentry_key=%s, sentry_secret=%s", 135 | "raven-lua/" .. _VERSION, 136 | iso8601(), 137 | dsn_object.public_key, 138 | dsn_object.secret_key) 139 | end 140 | 141 | return _M 142 | -------------------------------------------------------------------------------- /lualib/redis_slot.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilibili/oresty/1d0b4be38da37468ac6e08d6562f920b42ce3264/lualib/redis_slot.so -------------------------------------------------------------------------------- /lualib/resty/chash.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- some of them borrow from https://github.com/cloudflare/lua-resty-json 3 | -- 4 | 5 | local bit = require "bit" 6 | local ffi = require 'ffi' 7 | 8 | 9 | local ffi_new = ffi.new 10 | local C = ffi.C 11 | local crc32 = ngx.crc32_short 12 | local setmetatable = setmetatable 13 | local floor = math.floor 14 | local pairs = pairs 15 | local tostring = tostring 16 | local tonumber = tonumber 17 | local bxor = bit.bxor 18 | 19 | 20 | ffi.cdef[[ 21 | typedef unsigned int uint32_t; 22 | 23 | typedef struct { 24 | uint32_t hash; 25 | uint32_t id; 26 | } chash_point_t; 27 | 28 | void chash_point_init(chash_point_t *points, uint32_t base_hash, uint32_t start, 29 | uint32_t num, uint32_t id); 30 | void chash_point_sort(chash_point_t *points, uint32_t size); 31 | 32 | void chash_point_add(chash_point_t *old_points, uint32_t old_length, 33 | uint32_t base_hash, uint32_t from, uint32_t num, uint32_t id, 34 | chash_point_t *new_points); 35 | void chash_point_reduce(chash_point_t *old_points, uint32_t old_length, 36 | uint32_t base_hash, uint32_t from, uint32_t num, uint32_t id); 37 | void chash_point_delete(chash_point_t *old_points, uint32_t old_length, 38 | uint32_t id); 39 | ]] 40 | 41 | 42 | local ok, new_tab = pcall(require, "table.new") 43 | if not ok or type(new_tab) ~= "function" then 44 | new_tab = function (narr, nrec) return {} end 45 | end 46 | 47 | 48 | -- 49 | -- Find shared object file package.cpath, obviating the need of setting 50 | -- LD_LIBRARY_PATH 51 | -- Or we should add a little patch for ffi.load ? 52 | -- 53 | local function load_shared_lib(so_name) 54 | local string_gmatch = string.gmatch 55 | local string_match = string.match 56 | local io_open = io.open 57 | local io_close = io.close 58 | 59 | local cpath = package.cpath 60 | 61 | for k, _ in string_gmatch(cpath, "[^;]+") do 62 | local fpath = string_match(k, "(.*/)") 63 | fpath = fpath .. so_name 64 | 65 | -- Don't get me wrong, the only way to know if a file exist is trying 66 | -- to open it. 67 | local f = io_open(fpath) 68 | if f ~= nil then 69 | io_close(f) 70 | return ffi.load(fpath) 71 | end 72 | end 73 | end 74 | 75 | 76 | local _M = {} 77 | local mt = { __index = _M } 78 | 79 | 80 | local clib = load_shared_lib("librestychash.so") 81 | if not clib then 82 | error("can not load librestychash.so") 83 | end 84 | 85 | local CONSISTENT_POINTS = 160 -- points per server 86 | local pow32 = math.pow(2, 32) 87 | 88 | local chash_point_t = ffi.typeof("chash_point_t[?]") 89 | 90 | 91 | local function _precompute(nodes) 92 | local n, total_weight = 0, 0 93 | for id, weight in pairs(nodes) do 94 | n = n + 1 95 | total_weight = total_weight + weight 96 | end 97 | 98 | local newnodes = new_tab(0, n) 99 | for id, weight in pairs(nodes) do 100 | newnodes[id] = weight 101 | end 102 | 103 | local ids = new_tab(n, 0) 104 | local npoints = total_weight * CONSISTENT_POINTS 105 | local points = ffi_new(chash_point_t, npoints) 106 | 107 | local start, index = 0, 0 108 | for id, weight in pairs(nodes) do 109 | local num = weight * CONSISTENT_POINTS 110 | local base_hash = bxor(crc32(tostring(id)), 0xffffffff) 111 | 112 | index = index + 1 113 | ids[index] = id 114 | 115 | clib.chash_point_init(points, base_hash, start, num, index) 116 | 117 | start = start + num 118 | end 119 | 120 | clib.chash_point_sort(points, npoints) 121 | 122 | return ids, points, npoints, newnodes 123 | end 124 | 125 | 126 | function _M.new(_, nodes) 127 | local ids, points, npoints, newnodes = _precompute(nodes) 128 | 129 | local self = { 130 | nodes = newnodes, -- it's safer to copy one 131 | ids = ids, 132 | points = points, 133 | npoints = npoints, -- points number 134 | size = npoints, 135 | } 136 | return setmetatable(self, mt) 137 | end 138 | 139 | 140 | function _M.reinit(self, nodes) 141 | self.ids, self.points, self.npoints, self.newnodes = _precompute(nodes) 142 | self.size = self.npoints 143 | end 144 | 145 | 146 | local function _delete(self, id) 147 | local nodes = self.nodes 148 | local ids = self.ids 149 | local old_weight = nodes[id] 150 | 151 | if not old_weight then 152 | return 153 | end 154 | 155 | local index = 1 156 | -- find the index: O(n) 157 | while ids[index] ~= id do 158 | index = index + 1 159 | end 160 | 161 | nodes[id] = nil 162 | ids[index] = nil 163 | 164 | clib.chash_point_delete(self.points, self.npoints, index) 165 | 166 | self.npoints = self.npoints - CONSISTENT_POINTS * old_weight 167 | end 168 | _M.delete = _delete 169 | 170 | 171 | local function _incr(self, id, weight) 172 | local weight = tonumber(weight) or 1 173 | local nodes = self.nodes 174 | local ids = self.ids 175 | local old_weight = nodes[id] 176 | 177 | local index = 1 178 | if old_weight then 179 | -- find the index: O(n) 180 | while ids[index] ~= id do 181 | index = index + 1 182 | end 183 | 184 | else 185 | old_weight = 0 186 | 187 | index = #ids + 1 188 | ids[index] = id 189 | end 190 | 191 | nodes[id] = old_weight + weight 192 | 193 | local new_points = self.points 194 | local new_npoints = self.npoints + weight * CONSISTENT_POINTS 195 | if self.size < new_npoints then 196 | new_points = ffi_new(chash_point_t, new_npoints) 197 | self.size = new_npoints 198 | end 199 | 200 | local base_hash = bxor(crc32(tostring(id)), 0xffffffff) 201 | clib.chash_point_add(self.points, self.npoints, base_hash, 202 | old_weight * CONSISTENT_POINTS, 203 | weight * CONSISTENT_POINTS, 204 | index, new_points) 205 | 206 | self.points = new_points 207 | self.npoints = new_npoints 208 | end 209 | _M.incr = _incr 210 | 211 | 212 | local function _decr(self, id, weight) 213 | local weight = tonumber(weight) or 1 214 | local nodes = self.nodes 215 | local ids = self.ids 216 | local old_weight = nodes[id] 217 | 218 | if not old_weight then 219 | return 220 | end 221 | 222 | if old_weight <= weight then 223 | return _delete(self, id) 224 | end 225 | 226 | local index = 1 227 | -- find the index: O(n) 228 | while ids[index] ~= id do 229 | index = index + 1 230 | end 231 | 232 | local base_hash = bxor(crc32(tostring(id)), 0xffffffff) 233 | clib.chash_point_reduce(self.points, self.npoints, base_hash, 234 | (old_weight - weight) * CONSISTENT_POINTS, 235 | CONSISTENT_POINTS * weight, 236 | index) 237 | 238 | nodes[id] = old_weight - weight 239 | self.npoints = self.npoints - CONSISTENT_POINTS * weight 240 | end 241 | _M.decr = _decr 242 | 243 | 244 | function _M.set(self, id, new_weight) 245 | local new_weight = tonumber(new_weight) or 0 246 | local old_weight = self.nodes[id] or 0 247 | 248 | if old_weight == new_weight then 249 | return true 250 | end 251 | 252 | if old_weight < new_weight then 253 | return _incr(self, id, new_weight - old_weight) 254 | end 255 | 256 | return _decr(self, id, old_weight - new_weight) 257 | end 258 | 259 | 260 | local function _find_id(points, npoints, hash) 261 | local step = pow32 / npoints 262 | local index = floor(hash / step) 263 | 264 | local max_index = npoints - 1 265 | 266 | -- it seems safer to do this 267 | if index > max_index then 268 | index = max_index 269 | end 270 | 271 | -- find the first points >= hash 272 | if points[index].hash >= hash then 273 | for i = index, 1, -1 do 274 | if points[i - 1].hash < hash then 275 | return points[i].id, i 276 | end 277 | end 278 | 279 | return points[0].id, 0 280 | end 281 | 282 | for i = index + 1, max_index do 283 | if hash <= points[i].hash then 284 | return points[i].id, i 285 | end 286 | end 287 | 288 | return points[0].id, 0 289 | end 290 | 291 | 292 | function _M.find(self, key) 293 | local hash = crc32(tostring(key)) 294 | 295 | local id, index = _find_id(self.points, self.npoints, hash) 296 | 297 | return self.ids[id], index 298 | end 299 | 300 | 301 | function _M.next(self, index) 302 | local new_index = (index + 1) % self.npoints 303 | local id = self.points[new_index].id 304 | 305 | return self.ids[id], new_index 306 | end 307 | 308 | 309 | return _M 310 | -------------------------------------------------------------------------------- /lualib/resty/http_headers.lua: -------------------------------------------------------------------------------- 1 | local rawget, rawset, setmetatable = 2 | rawget, rawset, setmetatable 3 | 4 | local str_lower = string.lower 5 | 6 | local _M = { 7 | _VERSION = '0.12', 8 | } 9 | 10 | 11 | -- Returns an empty headers table with internalised case normalisation. 12 | function _M.new() 13 | local mt = { 14 | normalised = {}, 15 | } 16 | 17 | mt.__index = function(t, k) 18 | return rawget(t, mt.normalised[str_lower(k)]) 19 | end 20 | 21 | mt.__newindex = function(t, k, v) 22 | local k_normalised = str_lower(k) 23 | 24 | -- First time seeing this header field? 25 | if not mt.normalised[k_normalised] then 26 | -- Create a lowercased entry in the metatable proxy, with the value 27 | -- of the given field case 28 | mt.normalised[k_normalised] = k 29 | 30 | -- Set the header using the given field case 31 | rawset(t, k, v) 32 | else 33 | -- We're being updated just with a different field case. Use the 34 | -- normalised metatable proxy to give us the original key case, and 35 | -- perorm a rawset() to update the value. 36 | rawset(t, mt.normalised[k_normalised], v) 37 | end 38 | end 39 | 40 | return setmetatable({}, mt) 41 | end 42 | 43 | 44 | return _M 45 | -------------------------------------------------------------------------------- /lualib/resty/kafka/broker.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local response = require "resty.kafka.response" 5 | 6 | 7 | local to_int32 = response.to_int32 8 | local setmetatable = setmetatable 9 | local tcp = ngx.socket.tcp 10 | 11 | 12 | local _M = {} 13 | local mt = { __index = _M } 14 | 15 | 16 | function _M.new(self, host, port, socket_config) 17 | return setmetatable({ 18 | host = host, 19 | port = port, 20 | config = socket_config, 21 | }, mt) 22 | end 23 | 24 | 25 | function _M.send_receive(self, request) 26 | local sock, err = tcp() 27 | if not sock then 28 | return nil, err, true 29 | end 30 | 31 | sock:settimeout(self.config.socket_timeout) 32 | 33 | local ok, err = sock:connect(self.host, self.port) 34 | if not ok then 35 | return nil, err, true 36 | end 37 | 38 | local bytes, err = sock:send(request:package()) 39 | if not bytes then 40 | return nil, err, true 41 | end 42 | 43 | local data, err = sock:receive(4) 44 | if not data then 45 | if err == "timeout" then 46 | sock:close() 47 | return nil, err 48 | end 49 | return nil, err, true 50 | end 51 | 52 | local len = to_int32(data) 53 | 54 | local data, err = sock:receive(len) 55 | if not data then 56 | if err == "timeout" then 57 | sock:close() 58 | return nil, err 59 | end 60 | return nil, err, true 61 | end 62 | 63 | sock:setkeepalive(self.config.keepalive_timeout, self.config.keepalive_size) 64 | 65 | return response:new(data), nil, true 66 | end 67 | 68 | 69 | return _M 70 | -------------------------------------------------------------------------------- /lualib/resty/kafka/client.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local broker = require "resty.kafka.broker" 5 | local request = require "resty.kafka.request" 6 | 7 | 8 | local setmetatable = setmetatable 9 | local timer_at = ngx.timer.at 10 | local ngx_log = ngx.log 11 | local ERR = ngx.ERR 12 | local INFO = ngx.INFO 13 | local DEBUG = ngx.DEBUG 14 | local debug = ngx.config.debug 15 | local pid = ngx.worker.pid 16 | local time = ngx.time 17 | local sleep = ngx.sleep 18 | local ceil = math.ceil 19 | local pairs = pairs 20 | 21 | 22 | local ok, new_tab = pcall(require, "table.new") 23 | if not ok then 24 | new_tab = function (narr, nrec) return {} end 25 | end 26 | 27 | 28 | local _M = { _VERSION = "0.06" } 29 | local mt = { __index = _M } 30 | 31 | 32 | local function _metadata_cache(self, topic) 33 | if not topic then 34 | return self.brokers, self.topic_partitions 35 | end 36 | 37 | local partitions = self.topic_partitions[topic] 38 | if partitions and partitions.num and partitions.num > 0 then 39 | return self.brokers, partitions 40 | end 41 | 42 | return nil, "not found topic" 43 | end 44 | 45 | 46 | local function metadata_encode(client_id, topics, num) 47 | local id = 0 -- hard code correlation_id 48 | local req = request:new(request.MetadataRequest, id, client_id) 49 | 50 | req:int32(num) 51 | 52 | for i = 1, num do 53 | req:string(topics[i]) 54 | end 55 | 56 | return req 57 | end 58 | 59 | 60 | local function metadata_decode(resp) 61 | local bk_num = resp:int32() 62 | local brokers = new_tab(0, bk_num) 63 | 64 | for i = 1, bk_num do 65 | local nodeid = resp:int32(); 66 | brokers[nodeid] = { 67 | host = resp:string(), 68 | port = resp:int32(), 69 | } 70 | end 71 | 72 | local topic_num = resp:int32() 73 | local topics = new_tab(0, topic_num) 74 | 75 | for i = 1, topic_num do 76 | local tp_errcode = resp:int16() 77 | local topic = resp:string() 78 | 79 | local partition_num = resp:int32() 80 | local topic_info = new_tab(partition_num - 1, 3) 81 | 82 | topic_info.errcode = tp_errcode 83 | topic_info.num = partition_num 84 | 85 | for j = 1, partition_num do 86 | local partition_info = new_tab(0, 5) 87 | 88 | partition_info.errcode = resp:int16() 89 | partition_info.id = resp:int32() 90 | partition_info.leader = resp:int32() 91 | 92 | local repl_num = resp:int32() 93 | local replicas = new_tab(repl_num, 0) 94 | for m = 1, repl_num do 95 | replicas[m] = resp:int32() 96 | end 97 | partition_info.replicas = replicas 98 | 99 | local isr_num = resp:int32() 100 | local isr = new_tab(isr_num, 0) 101 | for m = 1, isr_num do 102 | isr[m] = resp:int32() 103 | end 104 | partition_info.isr = isr 105 | 106 | topic_info[partition_info.id] = partition_info 107 | end 108 | topics[topic] = topic_info 109 | end 110 | 111 | return brokers, topics 112 | end 113 | 114 | 115 | local function _fetch_metadata(self, new_topic) 116 | local topics, num = {}, 0 117 | for tp, _p in pairs(self.topic_partitions) do 118 | num = num + 1 119 | topics[num] = tp 120 | end 121 | 122 | if new_topic and not self.topic_partitions[new_topic] then 123 | num = num + 1 124 | topics[num] = new_topic 125 | end 126 | 127 | if num == 0 then 128 | return nil, "not topic" 129 | end 130 | 131 | local broker_list = self.broker_list 132 | local sc = self.socket_config 133 | local req = metadata_encode(self.client_id, topics, num) 134 | 135 | for i = 1, #broker_list do 136 | local host, port = broker_list[i].host, broker_list[i].port 137 | local bk = broker:new(host, port, sc) 138 | 139 | local resp, err = bk:send_receive(req) 140 | if not resp then 141 | ngx_log(INFO, "broker fetch metadata failed, err:", err, host, port) 142 | else 143 | local brokers, topic_partitions = metadata_decode(resp) 144 | self.brokers, self.topic_partitions = brokers, topic_partitions 145 | 146 | return brokers, topic_partitions 147 | end 148 | end 149 | 150 | ngx_log(ERR, "all brokers failed in fetch topic metadata") 151 | return nil, "all brokers failed in fetch topic metadata" 152 | end 153 | _M.refresh = _fetch_metadata 154 | 155 | 156 | local function meta_refresh(premature, self, interval) 157 | if premature then 158 | return 159 | end 160 | 161 | _fetch_metadata(self) 162 | 163 | local ok, err = timer_at(interval, meta_refresh, self, interval) 164 | if not ok then 165 | ngx_log(ERR, "failed to create timer at meta_refresh, err: ", err) 166 | end 167 | end 168 | 169 | 170 | function _M.new(self, broker_list, client_config) 171 | local opts = client_config or {} 172 | local socket_config = { 173 | socket_timeout = opts.socket_timeout or 3000, 174 | keepalive_timeout = opts.keepalive_timeout or 600 * 1000, -- 10 min 175 | keepalive_size = opts.keepalive_size or 2, 176 | } 177 | 178 | local cli = setmetatable({ 179 | broker_list = broker_list, 180 | topic_partitions = {}, 181 | brokers = {}, 182 | client_id = "worker" .. pid(), 183 | socket_config = socket_config, 184 | }, mt) 185 | 186 | if opts.refresh_interval then 187 | meta_refresh(nil, cli, opts.refresh_interval / 1000) -- in ms 188 | end 189 | 190 | return cli 191 | end 192 | 193 | 194 | function _M.fetch_metadata(self, topic) 195 | local brokers, partitions = _metadata_cache(self, topic) 196 | if brokers then 197 | return brokers, partitions 198 | end 199 | 200 | _fetch_metadata(self, topic) 201 | 202 | return _metadata_cache(self, topic) 203 | end 204 | 205 | 206 | function _M.choose_broker(self, topic, partition_id) 207 | local brokers, partitions = self:fetch_metadata(topic) 208 | if not brokers then 209 | return nil, partitions 210 | end 211 | 212 | local partition = partitions[partition_id] 213 | if not partition then 214 | return nil, "not found partition" 215 | end 216 | 217 | local config = brokers[partition.leader] 218 | if not config then 219 | return nil, "not found broker" 220 | end 221 | 222 | return config 223 | end 224 | 225 | 226 | return _M 227 | -------------------------------------------------------------------------------- /lualib/resty/kafka/errors.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local _M = { 5 | [0] = 'NoError', 6 | [-1] = 'Unknown', 7 | [1] = 'OffsetOutOfRange', 8 | [2] = 'InvalidMessage', 9 | [3] = 'UnknownTopicOrPartition', 10 | [4] = 'InvalidMessageSize', 11 | [5] = 'LeaderNotAvailable', 12 | [6] = 'NotLeaderForPartition', 13 | [7] = 'RequestTimedOut', 14 | [8] = 'BrokerNotAvailable', 15 | [9] = 'ReplicaNotAvailable', 16 | [10] = 'MessageSizeTooLarge', 17 | [11] = 'StaleControllerEpochCode', 18 | [12] = 'OffsetMetadataTooLargeCode', 19 | [14] = 'OffsetsLoadInProgressCode', 20 | [15] = 'ConsumerCoordinatorNotAvailableCode', 21 | [16] = 'NotCoordinatorForConsumerCode', 22 | } 23 | 24 | 25 | return _M 26 | -------------------------------------------------------------------------------- /lualib/resty/kafka/producer.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local response = require "resty.kafka.response" 5 | local request = require "resty.kafka.request" 6 | local broker = require "resty.kafka.broker" 7 | local client = require "resty.kafka.client" 8 | local Errors = require "resty.kafka.errors" 9 | local sendbuffer = require "resty.kafka.sendbuffer" 10 | local ringbuffer = require "resty.kafka.ringbuffer" 11 | 12 | 13 | local setmetatable = setmetatable 14 | local timer_at = ngx.timer.at 15 | local is_exiting = ngx.worker.exiting 16 | local ngx_sleep = ngx.sleep 17 | local ngx_log = ngx.log 18 | local ERR = ngx.ERR 19 | local INFO = ngx.INFO 20 | local DEBUG = ngx.DEBUG 21 | local debug = ngx.config.debug 22 | local crc32 = ngx.crc32_short 23 | local pcall = pcall 24 | local pairs = pairs 25 | 26 | 27 | local ok, new_tab = pcall(require, "table.new") 28 | if not ok then 29 | new_tab = function (narr, nrec) return {} end 30 | end 31 | 32 | 33 | local _M = { _VERSION = "0.06" } 34 | local mt = { __index = _M } 35 | 36 | 37 | -- weak value table is useless here, cause _timer_flush always ref p 38 | -- so, weak value table won't works 39 | local cluster_inited = {} 40 | local DEFAULT_CLUSTER_NAME = 1 41 | 42 | 43 | local function default_partitioner(key, num, correlation_id) 44 | local id = key and crc32(key) or correlation_id 45 | 46 | -- partition_id is continuous and start from 0 47 | return id % num 48 | end 49 | 50 | 51 | local function correlation_id(self) 52 | local id = (self.correlation_id + 1) % 1073741824 -- 2^30 53 | self.correlation_id = id 54 | 55 | return id 56 | end 57 | 58 | 59 | local function produce_encode(self, topic_partitions) 60 | local req = request:new(request.ProduceRequest, 61 | correlation_id(self), self.client.client_id) 62 | 63 | req:int16(self.required_acks) 64 | req:int32(self.request_timeout) 65 | req:int32(topic_partitions.topic_num) 66 | 67 | for topic, partitions in pairs(topic_partitions.topics) do 68 | req:string(topic) 69 | req:int32(partitions.partition_num) 70 | 71 | for partition_id, buffer in pairs(partitions.partitions) do 72 | req:int32(partition_id) 73 | 74 | -- MessageSetSize and MessageSet 75 | req:message_set(buffer.queue, buffer.index) 76 | end 77 | end 78 | 79 | return req 80 | end 81 | 82 | 83 | local function produce_decode(resp) 84 | local topic_num = resp:int32() 85 | local ret = new_tab(0, topic_num) 86 | 87 | for i = 1, topic_num do 88 | local topic = resp:string() 89 | local partition_num = resp:int32() 90 | 91 | ret[topic] = {} 92 | 93 | for j = 1, partition_num do 94 | local partition = resp:int32() 95 | 96 | ret[topic][partition] = { 97 | errcode = resp:int16(), 98 | offset = resp:int64(), 99 | } 100 | end 101 | end 102 | 103 | return ret 104 | end 105 | 106 | 107 | local function choose_partition(self, topic, key) 108 | local brokers, partitions = self.client:fetch_metadata(topic) 109 | if not brokers then 110 | return nil, partitions 111 | end 112 | 113 | return self.partitioner(key, partitions.num, self.correlation_id) 114 | end 115 | 116 | 117 | local function _flush_lock(self) 118 | if not self.flushing then 119 | if debug then 120 | ngx_log(DEBUG, "flush lock accquired") 121 | end 122 | self.flushing = true 123 | return true 124 | end 125 | return false 126 | end 127 | 128 | 129 | local function _flush_unlock(self) 130 | if debug then 131 | ngx_log(DEBUG, "flush lock released") 132 | end 133 | self.flushing = false 134 | end 135 | 136 | 137 | local function _send(self, broker_conf, topic_partitions) 138 | local sendbuffer = self.sendbuffer 139 | local resp, retryable = nil, true 140 | 141 | local bk, err = broker:new(broker_conf.host, broker_conf.port, self.socket_config) 142 | if bk then 143 | local req = produce_encode(self, topic_partitions) 144 | 145 | resp, err, retryable = bk:send_receive(req) 146 | if resp then 147 | local result = produce_decode(resp) 148 | 149 | for topic, partitions in pairs(result) do 150 | for partition_id, r in pairs(partitions) do 151 | local errcode = r.errcode 152 | 153 | if errcode == 0 then 154 | sendbuffer:offset(topic, partition_id, r.offset) 155 | sendbuffer:clear(topic, partition_id) 156 | else 157 | err = Errors[errcode] 158 | 159 | -- XX: only 3, 5, 6 can retry 160 | local retryable0 = retryable 161 | if errcode ~= 3 and errcode ~= 5 and errcode ~= 6 then 162 | retryable0 = false 163 | end 164 | 165 | local index = sendbuffer:err(topic, partition_id, err, retryable0) 166 | 167 | ngx_log(INFO, "retry to send messages to kafka err: ", err, ", retryable: ", retryable0, 168 | ", topic: ", topic, ", partition_id: ", partition_id, ", length: ", index / 2) 169 | end 170 | end 171 | end 172 | 173 | return 174 | end 175 | end 176 | 177 | -- when broker new failed or send_receive failed 178 | for topic, partitions in pairs(topic_partitions.topics) do 179 | for partition_id, partition in pairs(partitions.partitions) do 180 | sendbuffer:err(topic, partition_id, err, retryable) 181 | end 182 | end 183 | end 184 | 185 | 186 | local function _batch_send(self, sendbuffer) 187 | local try_num = 1 188 | while try_num <= self.max_retry do 189 | -- aggregator 190 | local send_num, sendbroker = sendbuffer:aggregator(self.client) 191 | if send_num == 0 then 192 | break 193 | end 194 | 195 | for i = 1, send_num, 2 do 196 | local broker_conf, topic_partitions = sendbroker[i], sendbroker[i + 1] 197 | 198 | _send(self, broker_conf, topic_partitions) 199 | end 200 | 201 | if sendbuffer:done() then 202 | return true 203 | end 204 | 205 | self.client:refresh() 206 | 207 | try_num = try_num + 1 208 | if try_num < self.max_retry then 209 | ngx_sleep(self.retry_backoff / 1000) -- ms to s 210 | end 211 | end 212 | end 213 | 214 | 215 | local _flush_buffer 216 | 217 | 218 | local function _flush(premature, self) 219 | if not _flush_lock(self) then 220 | if debug then 221 | ngx_log(DEBUG, "previous flush not finished") 222 | end 223 | return 224 | end 225 | 226 | local ringbuffer = self.ringbuffer 227 | local sendbuffer = self.sendbuffer 228 | 229 | while true do 230 | local topic, key, msg = ringbuffer:pop() 231 | if not topic then 232 | break 233 | end 234 | 235 | local partition_id, err = choose_partition(self, topic, key) 236 | if not partition_id then 237 | partition_id = -1 238 | end 239 | 240 | local overflow = sendbuffer:add(topic, partition_id, key, msg) 241 | if overflow then -- reached batch_size in one topic-partition 242 | break 243 | end 244 | end 245 | 246 | local all_done = _batch_send(self, sendbuffer) 247 | 248 | if not all_done then 249 | for topic, partition_id, buffer in sendbuffer:loop() do 250 | local queue, index, err, retryable = buffer.queue, buffer.index, buffer.err, buffer.retryable 251 | 252 | if self.error_handle then 253 | local ok, err = pcall(self.error_handle, topic, partition_id, queue, index, err, retryable) 254 | if not ok then 255 | ngx_log(ERR, "failed to callback error_handle: ", err) 256 | end 257 | else 258 | ngx_log(ERR, "buffered messages send to kafka err: ", err, 259 | ", retryable: ", retryable, ", topic: ", topic, 260 | ", partition_id: ", partition_id, ", length: ", index / 2) 261 | end 262 | 263 | sendbuffer:clear(topic, partition_id) 264 | end 265 | end 266 | 267 | _flush_unlock(self) 268 | 269 | if ringbuffer:need_send() then 270 | _flush_buffer(self) 271 | 272 | elseif is_exiting() and ringbuffer:left_num() > 0 then 273 | -- still can create 0 timer even exiting 274 | _flush_buffer(self) 275 | end 276 | 277 | return true 278 | end 279 | 280 | 281 | _flush_buffer = function (self) 282 | local ok, err = timer_at(0, _flush, self) 283 | if not ok then 284 | ngx_log(ERR, "failed to create timer at _flush_buffer, err: ", err) 285 | end 286 | end 287 | 288 | 289 | local _timer_flush 290 | _timer_flush = function (premature, self, time) 291 | _flush_buffer(self) 292 | 293 | if premature then 294 | return 295 | end 296 | 297 | local ok, err = timer_at(time, _timer_flush, self, time) 298 | if not ok then 299 | ngx_log(ERR, "failed to create timer at _timer_flush, err: ", err) 300 | end 301 | end 302 | 303 | 304 | function _M.new(self, broker_list, producer_config, cluster_name) 305 | local name = cluster_name or DEFAULT_CLUSTER_NAME 306 | local opts = producer_config or {} 307 | local async = opts.producer_type == "async" 308 | if async and cluster_inited[name] then 309 | return cluster_inited[name] 310 | end 311 | 312 | local cli = client:new(broker_list, producer_config) 313 | local p = setmetatable({ 314 | client = cli, 315 | correlation_id = 1, 316 | request_timeout = opts.request_timeout or 2000, 317 | retry_backoff = opts.retry_backoff or 100, -- ms 318 | max_retry = opts.max_retry or 3, 319 | required_acks = opts.required_acks or 1, 320 | partitioner = opts.partitioner or default_partitioner, 321 | error_handle = opts.error_handle, 322 | async = async, 323 | socket_config = cli.socket_config, 324 | ringbuffer = ringbuffer:new(opts.batch_num or 200, opts.max_buffering or 50000), -- 200, 50K 325 | sendbuffer = sendbuffer:new(opts.batch_num or 200, opts.batch_size or 1048576) 326 | -- default: 1K, 1M 327 | -- batch_size should less than (MaxRequestSize / 2 - 10KiB) 328 | -- config in the kafka server, default 100M 329 | }, mt) 330 | 331 | if async then 332 | cluster_inited[name] = p 333 | _timer_flush(nil, p, (opts.flush_time or 1000) / 1000) -- default 1s 334 | end 335 | return p 336 | end 337 | 338 | 339 | -- offset is cdata (LL in luajit) 340 | function _M.send(self, topic, key, message) 341 | if self.async then 342 | local ringbuffer = self.ringbuffer 343 | 344 | local ok, err = ringbuffer:add(topic, key, message) 345 | if not ok then 346 | return nil, err 347 | end 348 | 349 | if not self.flushing and (ringbuffer:need_send() or is_exiting()) then 350 | _flush_buffer(self) 351 | end 352 | 353 | return true 354 | end 355 | 356 | local partition_id, err = choose_partition(self, topic, key) 357 | if not partition_id then 358 | return nil, err 359 | end 360 | 361 | local sendbuffer = self.sendbuffer 362 | sendbuffer:add(topic, partition_id, key, message) 363 | 364 | local ok = _batch_send(self, sendbuffer) 365 | if not ok then 366 | sendbuffer:clear(topic, partition_id) 367 | return nil, sendbuffer:err(topic, partition_id) 368 | end 369 | 370 | return sendbuffer:offset(topic, partition_id) 371 | end 372 | 373 | 374 | function _M.flush(self) 375 | return _flush(nil, self) 376 | end 377 | 378 | 379 | -- offset is cdata (LL in luajit) 380 | function _M.offset(self) 381 | local topics = self.sendbuffer.topics 382 | local sum, details = 0, {} 383 | 384 | for topic, partitions in pairs(topics) do 385 | details[topic] = {} 386 | for partition_id, buffer in pairs(partitions) do 387 | sum = sum + buffer.offset 388 | details[topic][partition_id] = buffer.offset 389 | end 390 | end 391 | 392 | return sum, details 393 | end 394 | 395 | 396 | return _M 397 | -------------------------------------------------------------------------------- /lualib/resty/kafka/request.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local bit = require "bit" 5 | 6 | 7 | local setmetatable = setmetatable 8 | local concat = table.concat 9 | local rshift = bit.rshift 10 | local band = bit.band 11 | local char = string.char 12 | local crc32 = ngx.crc32_long 13 | local tonumber = tonumber 14 | 15 | 16 | local _M = {} 17 | local mt = { __index = _M } 18 | 19 | 20 | local API_VERSION = 0 21 | 22 | _M.ProduceRequest = 0 23 | _M.FetchRequest = 1 24 | _M.OffsetRequest = 2 25 | _M.MetadataRequest = 3 26 | _M.OffsetCommitRequest = 8 27 | _M.OffsetFetchRequest = 9 28 | _M.ConsumerMetadataRequest = 10 29 | 30 | 31 | local function str_int8(int) 32 | return char(band(int, 0xff)) 33 | end 34 | 35 | 36 | local function str_int16(int) 37 | return char(band(rshift(int, 8), 0xff), 38 | band(int, 0xff)) 39 | end 40 | 41 | 42 | local function str_int32(int) 43 | -- ngx.say(debug.traceback()) 44 | return char(band(rshift(int, 24), 0xff), 45 | band(rshift(int, 16), 0xff), 46 | band(rshift(int, 8), 0xff), 47 | band(int, 0xff)) 48 | end 49 | 50 | 51 | -- XX int can be cdata: LL or lua number 52 | local function str_int64(int) 53 | return char(tonumber(band(rshift(int, 56), 0xff)), 54 | tonumber(band(rshift(int, 48), 0xff)), 55 | tonumber(band(rshift(int, 40), 0xff)), 56 | tonumber(band(rshift(int, 32), 0xff)), 57 | tonumber(band(rshift(int, 24), 0xff)), 58 | tonumber(band(rshift(int, 16), 0xff)), 59 | tonumber(band(rshift(int, 8), 0xff)), 60 | tonumber(band(int, 0xff))) 61 | end 62 | 63 | 64 | function _M.new(self, apikey, correlation_id, client_id) 65 | local c_len = #client_id 66 | 67 | local req = { 68 | 0, -- request size: int32 69 | str_int16(apikey), 70 | str_int16(API_VERSION), 71 | str_int32(correlation_id), 72 | str_int16(c_len), 73 | client_id, 74 | } 75 | return setmetatable({ 76 | _req = req, 77 | offset = 7, 78 | len = c_len + 10, 79 | }, mt) 80 | end 81 | 82 | 83 | function _M.int16(self, int) 84 | local req = self._req 85 | local offset = self.offset 86 | 87 | req[offset] = str_int16(int) 88 | 89 | self.offset = offset + 1 90 | self.len = self.len + 2 91 | end 92 | 93 | 94 | function _M.int32(self, int) 95 | local req = self._req 96 | local offset = self.offset 97 | 98 | req[offset] = str_int32(int) 99 | 100 | self.offset = offset + 1 101 | self.len = self.len + 4 102 | end 103 | 104 | 105 | function _M.int64(self, int) 106 | local req = self._req 107 | local offset = self.offset 108 | 109 | req[offset] = str_int64(int) 110 | 111 | self.offset = offset + 1 112 | self.len = self.len + 8 113 | end 114 | 115 | 116 | function _M.string(self, str) 117 | local req = self._req 118 | local offset = self.offset 119 | local str_len = #str 120 | 121 | req[offset] = str_int16(str_len) 122 | req[offset + 1] = str 123 | 124 | self.offset = offset + 2 125 | self.len = self.len + 2 + str_len 126 | end 127 | 128 | 129 | function _M.bytes(self, str) 130 | local req = self._req 131 | local offset = self.offset 132 | local str_len = #str 133 | 134 | req[offset] = str_int32(str_len) 135 | req[offset + 1] = str 136 | 137 | self.offset = offset + 2 138 | self.len = self.len + 4 + str_len 139 | end 140 | 141 | 142 | local function message_package(key, msg) 143 | local key = key or "" 144 | local key_len = #key 145 | local len = #msg 146 | 147 | local req = { 148 | -- MagicByte 149 | str_int8(0), 150 | -- XX hard code no Compression 151 | str_int8(0), 152 | str_int32(key_len), 153 | key, 154 | str_int32(len), 155 | msg, 156 | } 157 | 158 | local str = concat(req) 159 | return crc32(str), str, key_len + len + 14 160 | end 161 | 162 | 163 | function _M.message_set(self, messages, index) 164 | local req = self._req 165 | local off = self.offset 166 | local msg_set_size = 0 167 | local index = index or #messages 168 | 169 | for i = 1, index, 2 do 170 | local crc32, str, msg_len = message_package(messages[i], messages[i + 1]) 171 | 172 | req[off + 1] = str_int64(0) -- offset 173 | req[off + 2] = str_int32(msg_len) -- include the crc32 length 174 | 175 | req[off + 3] = str_int32(crc32) 176 | req[off + 4] = str 177 | 178 | off = off + 4 179 | msg_set_size = msg_set_size + msg_len + 12 180 | end 181 | 182 | req[self.offset] = str_int32(msg_set_size) -- MessageSetSize 183 | 184 | self.offset = off + 1 185 | self.len = self.len + 4 + msg_set_size 186 | end 187 | 188 | 189 | function _M.package(self) 190 | local req = self._req 191 | req[1] = str_int32(self.len) 192 | 193 | return req 194 | end 195 | 196 | 197 | return _M 198 | -------------------------------------------------------------------------------- /lualib/resty/kafka/response.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local bit = require "bit" 5 | 6 | 7 | local setmetatable = setmetatable 8 | local byte = string.byte 9 | local sub = string.sub 10 | local lshift = bit.lshift 11 | local bor = bit.bor 12 | local strbyte = string.byte 13 | 14 | 15 | local _M = {} 16 | local mt = { __index = _M } 17 | 18 | 19 | function _M.new(self, str) 20 | local resp = setmetatable({ 21 | str = str, 22 | offset = 1, 23 | correlation_id = 0, 24 | }, mt) 25 | 26 | resp.correlation_id = resp:int32() 27 | 28 | return resp 29 | end 30 | 31 | 32 | function _M.int16(self) 33 | local str = self.str 34 | local offset = self.offset 35 | self.offset = offset + 2 36 | 37 | local high = byte(str, offset) 38 | -- high padded 39 | return bor((high >= 128) and 0xffff0000 or 0, 40 | lshift(high, 8), 41 | byte(str, offset + 1)) 42 | end 43 | 44 | 45 | local function to_int32(str, offset) 46 | local offset = offset or 1 47 | local a, b, c, d = strbyte(str, offset, offset + 3) 48 | return bor(lshift(a, 24), lshift(b, 16), lshift(c, 8), d) 49 | end 50 | _M.to_int32 = to_int32 51 | 52 | 53 | function _M.int32(self) 54 | local str = self.str 55 | local offset = self.offset 56 | self.offset = offset + 4 57 | 58 | return to_int32(str, offset) 59 | end 60 | 61 | 62 | -- XX return cdata: LL 63 | function _M.int64(self) 64 | local offset = self.offset 65 | self.offset = offset + 8 66 | 67 | local a, b, c, d, e, f, g, h = strbyte(self.str, offset, offset + 7) 68 | 69 | --[[ 70 | -- only 52 bit accuracy 71 | local hi = bor(lshift(a, 24), lshift(b, 16), lshift(c, 8), d) 72 | local lo = bor(lshift(f, 16), lshift(g, 8), h) 73 | return hi * 4294967296 + 16777216 * e + lo 74 | --]] 75 | 76 | return 4294967296LL * bor(lshift(a, 56), lshift(b, 48), lshift(c, 40), lshift(d, 32)) 77 | + 16777216LL * e 78 | + bor(lshift(f, 16), lshift(g, 8), h) 79 | end 80 | 81 | 82 | function _M.string(self) 83 | local len = self:int16() 84 | 85 | local offset = self.offset 86 | self.offset = offset + len 87 | 88 | return sub(self.str, offset, offset + len - 1) 89 | end 90 | 91 | 92 | function _M.bytes(self) 93 | local len = self:int32() 94 | 95 | local offset = self.offset 96 | self.offset = offset + len 97 | 98 | return sub(self.str, offset, offset + len - 1) 99 | end 100 | 101 | 102 | function _M.correlation_id(self) 103 | return self.correlation_id 104 | end 105 | 106 | 107 | return _M 108 | -------------------------------------------------------------------------------- /lualib/resty/kafka/ringbuffer.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local setmetatable = setmetatable 5 | local ngx_null = ngx.null 6 | 7 | local ok, new_tab = pcall(require, "table.new") 8 | if not ok then 9 | new_tab = function (narr, nrec) return {} end 10 | end 11 | 12 | 13 | local _M = {} 14 | local mt = { __index = _M } 15 | 16 | function _M.new(self, batch_num, max_buffering) 17 | local sendbuffer = { 18 | queue = new_tab(max_buffering * 3, 0), 19 | batch_num = batch_num, 20 | size = max_buffering * 3, 21 | start = 1, 22 | num = 0, 23 | } 24 | return setmetatable(sendbuffer, mt) 25 | end 26 | 27 | 28 | function _M.add(self, topic, key, message) 29 | local num = self.num 30 | local size = self.size 31 | 32 | if num >= size then 33 | return nil, "buffer overflow" 34 | end 35 | 36 | local index = (self.start + num) % size 37 | local queue = self.queue 38 | 39 | queue[index] = topic 40 | queue[index + 1] = key 41 | queue[index + 2] = message 42 | 43 | self.num = num + 3 44 | 45 | return true 46 | end 47 | 48 | 49 | function _M.pop(self) 50 | local num = self.num 51 | if num <= 0 then 52 | return nil, "empty buffer" 53 | end 54 | 55 | self.num = num - 3 56 | 57 | local start = self.start 58 | local queue = self.queue 59 | 60 | self.start = (start + 3) % self.size 61 | 62 | local key, topic, message = queue[start], queue[start + 1], queue[start + 2] 63 | 64 | queue[start], queue[start + 1], queue[start + 2] = ngx_null, ngx_null, ngx_null 65 | 66 | return key, topic, message 67 | end 68 | 69 | 70 | function _M.left_num(self) 71 | return self.num / 3 72 | end 73 | 74 | 75 | function _M.need_send(self) 76 | return self.num / 3 >= self.batch_num 77 | end 78 | 79 | 80 | return _M 81 | -------------------------------------------------------------------------------- /lualib/resty/kafka/sendbuffer.lua: -------------------------------------------------------------------------------- 1 | -- Copyright (C) Dejiang Zhu(doujiang24) 2 | 3 | 4 | local setmetatable = setmetatable 5 | local pairs = pairs 6 | local next = next 7 | 8 | 9 | local ok, new_tab = pcall(require, "table.new") 10 | if not ok then 11 | new_tab = function (narr, nrec) return {} end 12 | end 13 | 14 | local MAX_REUSE = 10000 15 | 16 | 17 | local _M = {} 18 | local mt = { __index = _M } 19 | 20 | function _M.new(self, batch_num, batch_size) 21 | local sendbuffer = { 22 | topics = {}, 23 | queue_num = 0, 24 | batch_num = batch_num * 2, 25 | batch_size = batch_size, 26 | } 27 | return setmetatable(sendbuffer, mt) 28 | end 29 | 30 | 31 | function _M.add(self, topic, partition_id, key, msg) 32 | local topics = self.topics 33 | 34 | if not topics[topic] then 35 | topics[topic] = {} 36 | end 37 | 38 | if not topics[topic][partition_id] then 39 | topics[topic][partition_id] = { 40 | queue = new_tab(self.batch_num, 0), 41 | index = 0, 42 | used = 0, 43 | size = 0, 44 | offset = 0, 45 | retryable = true, 46 | err = "", 47 | } 48 | end 49 | 50 | local buffer = topics[topic][partition_id] 51 | local index = buffer.index 52 | local queue = buffer.queue 53 | 54 | if index == 0 then 55 | self.queue_num = self.queue_num + 1 56 | buffer.retryable = true 57 | end 58 | 59 | queue[index + 1] = key 60 | queue[index + 2] = msg 61 | 62 | buffer.index = index + 2 63 | buffer.size = buffer.size + #msg + (key and #key or 0) 64 | 65 | if (buffer.size >= self.batch_size) or (buffer.index >= self.batch_num) then 66 | return true 67 | end 68 | end 69 | 70 | 71 | function _M.offset(self, topic, partition_id, offset) 72 | local buffer = self.topics[topic][partition_id] 73 | 74 | if not offset then 75 | return buffer.offset 76 | end 77 | 78 | buffer.offset = offset + (buffer.index / 2) 79 | end 80 | 81 | 82 | function _M.clear(self, topic, partition_id) 83 | local buffer = self.topics[topic][partition_id] 84 | buffer.index = 0 85 | buffer.size = 0 86 | buffer.used = buffer.used + 1 87 | 88 | if buffer.used >= MAX_REUSE then 89 | buffer.queue = new_tab(self.batch_num, 0) 90 | buffer.used = 0 91 | end 92 | 93 | self.queue_num = self.queue_num - 1 94 | end 95 | 96 | 97 | function _M.done(self) 98 | return self.queue_num == 0 99 | end 100 | 101 | 102 | function _M.err(self, topic, partition_id, err, retryable) 103 | local buffer = self.topics[topic][partition_id] 104 | 105 | if err then 106 | buffer.err = err 107 | buffer.retryable = retryable 108 | return buffer.index 109 | else 110 | return buffer.err, buffer.retryable 111 | end 112 | end 113 | 114 | 115 | function _M.loop(self) 116 | local topics, t, p = self.topics 117 | 118 | return function () 119 | if t then 120 | for partition_id, queue in next, topics[t], p do 121 | p = partition_id 122 | if queue.index > 0 then 123 | return t, partition_id, queue 124 | end 125 | end 126 | end 127 | 128 | 129 | for topic, partitions in next, topics, t do 130 | t = topic 131 | p = nil 132 | for partition_id, queue in next, partitions, p do 133 | p = partition_id 134 | if queue.index > 0 then 135 | return topic, partition_id, queue 136 | end 137 | end 138 | end 139 | 140 | return 141 | end 142 | end 143 | 144 | 145 | function _M.aggregator(self, client) 146 | local num = 0 147 | local sendbroker = {} 148 | local brokers = {} 149 | 150 | local i = 1 151 | for topic, partition_id, queue in self:loop() do 152 | if queue.retryable then 153 | local broker_conf, err = client:choose_broker(topic, partition_id) 154 | if not broker_conf then 155 | self:err(topic, partition_id, err, true) 156 | 157 | else 158 | if not brokers[broker_conf] then 159 | brokers[broker_conf] = { 160 | topics = {}, 161 | topic_num = 0, 162 | size = 0, 163 | } 164 | end 165 | 166 | local broker = brokers[broker_conf] 167 | if not broker.topics[topic] then 168 | brokers[broker_conf].topics[topic] = { 169 | partitions = {}, 170 | partition_num = 0, 171 | } 172 | 173 | broker.topic_num = broker.topic_num + 1 174 | end 175 | 176 | local broker_topic = broker.topics[topic] 177 | 178 | broker_topic.partitions[partition_id] = queue 179 | broker_topic.partition_num = broker_topic.partition_num + 1 180 | 181 | broker.size = broker.size + queue.size 182 | 183 | if broker.size >= self.batch_size then 184 | sendbroker[num + 1] = broker_conf 185 | sendbroker[num + 2] = brokers[broker_conf] 186 | 187 | num = num + 2 188 | brokers[broker_conf] = nil 189 | end 190 | end 191 | end 192 | end 193 | 194 | for broker_conf, topic_partitions in pairs(brokers) do 195 | sendbroker[num + 1] = broker_conf 196 | sendbroker[num + 2] = brokers[broker_conf] 197 | num = num + 2 198 | end 199 | 200 | return num, sendbroker 201 | end 202 | 203 | 204 | return _M 205 | -------------------------------------------------------------------------------- /lualib/resty/mlcache/ipc.lua: -------------------------------------------------------------------------------- 1 | -- vim: st=4 sts=4 sw=4 et: 2 | 3 | local ERR = ngx.ERR 4 | local WARN = ngx.WARN 5 | local INFO = ngx.INFO 6 | local sleep = ngx.sleep 7 | local shared = ngx.shared 8 | local worker_pid = ngx.worker.pid 9 | local ngx_log = ngx.log 10 | local fmt = string.format 11 | local sub = string.sub 12 | local find = string.find 13 | local min = math.min 14 | local type = type 15 | local pcall = pcall 16 | local error = error 17 | local insert = table.insert 18 | local tonumber = tonumber 19 | local setmetatable = setmetatable 20 | 21 | 22 | local INDEX_KEY = "lua-resty-ipc:index" 23 | local POLL_SLEEP_RATIO = 2 24 | 25 | 26 | local function marshall(worker_pid, channel, data) 27 | return fmt("%d:%d:%s%s", worker_pid, #data, channel, data) 28 | end 29 | 30 | 31 | local function unmarshall(str) 32 | local sep_1 = find(str, ":", nil , true) 33 | local sep_2 = find(str, ":", sep_1 + 1, true) 34 | 35 | local pid = tonumber(sub(str, 1 , sep_1 - 1)) 36 | local data_len = tonumber(sub(str, sep_1 + 1, sep_2 - 1)) 37 | 38 | local channel_last_pos = #str - data_len 39 | 40 | local channel = sub(str, sep_2 + 1, channel_last_pos) 41 | local data = sub(str, channel_last_pos + 1) 42 | 43 | return pid, channel, data 44 | end 45 | 46 | 47 | local function log(lvl, ...) 48 | return ngx_log(lvl, "[ipc] ", ...) 49 | end 50 | 51 | 52 | local _M = {} 53 | local mt = { __index = _M } 54 | 55 | 56 | function _M.new(shm, debug) 57 | local dict = shared[shm] 58 | if not dict then 59 | return nil, "no such lua_shared_dict: " .. shm 60 | end 61 | 62 | local self = { 63 | dict = dict, 64 | pid = debug and 0 or worker_pid(), 65 | idx = 0, 66 | callbacks = {}, 67 | } 68 | 69 | return setmetatable(self, mt) 70 | end 71 | 72 | 73 | function _M:subscribe(channel, cb) 74 | if type(channel) ~= "string" then 75 | error("channel must be a string", 2) 76 | end 77 | 78 | if type(cb) ~= "function" then 79 | error("callback must be a function", 2) 80 | end 81 | 82 | if not self.callbacks[channel] then 83 | self.callbacks[channel] = { cb } 84 | 85 | else 86 | insert(self.callbacks[channel], cb) 87 | end 88 | end 89 | 90 | 91 | function _M:broadcast(channel, data) 92 | if type(channel) ~= "string" then 93 | error("channel must be a string", 2) 94 | end 95 | 96 | if type(data) ~= "string" then 97 | error("data must be a string", 2) 98 | end 99 | 100 | local marshalled_event = marshall(worker_pid(), channel, data) 101 | 102 | local idx, err = self.dict:incr(INDEX_KEY, 1, 0) 103 | if not idx then 104 | return nil, "failed to increment index: " .. err 105 | end 106 | 107 | local ok, err = self.dict:set(idx, marshalled_event) 108 | if not ok then 109 | return nil, "failed to insert event in shm: " .. err 110 | end 111 | 112 | return true 113 | end 114 | 115 | 116 | -- Note: if this module were to be used by users (that is, users can implement 117 | -- their own pub/sub events and thus, callbacks), this method would then need 118 | -- to consider the time spent in callbacks to prevent long running callbacks 119 | -- from penalizing the worker. 120 | -- Since this module is currently only used by mlcache, whose callback is an 121 | -- shm operation, we only worry about the time spent waiting for events 122 | -- between the 'incr()' and 'set()' race condition. 123 | function _M:poll(timeout) 124 | if timeout ~= nil and type(timeout) ~= "number" then 125 | error("timeout must be a number", 2) 126 | end 127 | 128 | local idx, err = self.dict:get(INDEX_KEY) 129 | if err then 130 | return nil, "failed to get index: " .. err 131 | end 132 | 133 | if idx == nil then 134 | -- no events to poll yet 135 | return true 136 | end 137 | 138 | if type(idx) ~= "number" then 139 | return nil, "index is not a number, shm tampered with" 140 | end 141 | 142 | if not timeout then 143 | timeout = 0.3 144 | end 145 | 146 | local elapsed = 0 147 | 148 | for _ = self.idx, idx - 1 do 149 | -- fetch event from shm with a retry policy in case 150 | -- we run our :get() in between another worker's 151 | -- :incr() and :set() 152 | 153 | local v 154 | local idx = self.idx + 1 155 | 156 | do 157 | local perr 158 | local pok = true 159 | local sleep_step = 0.001 160 | 161 | while elapsed < timeout do 162 | v, err = self.dict:get(idx) 163 | if v ~= nil or err then 164 | break 165 | end 166 | 167 | if pok then 168 | log(INFO, "no event data at index '", idx, "', ", 169 | "retrying in: ", sleep_step, "s") 170 | 171 | -- sleep is not available in all ngx_lua contexts 172 | -- if we fail once, never retry to sleep 173 | pok, perr = pcall(sleep, sleep_step) 174 | if not pok then 175 | log(WARN, "could not sleep before retry: ", perr, 176 | " (note: it is safer to call this function ", 177 | "in contexts that support the ngx.sleep() ", 178 | "API)") 179 | end 180 | end 181 | 182 | elapsed = elapsed + sleep_step 183 | sleep_step = min(sleep_step * POLL_SLEEP_RATIO, 184 | timeout - elapsed) 185 | end 186 | end 187 | 188 | -- fetch next event on next iteration 189 | -- even if we timeout, we might miss 1 event (we return in timeout and 190 | -- we don't retry that event), but it's better than being stuck forever 191 | -- on an event that might have been evicted from the shm. 192 | self.idx = idx 193 | 194 | if elapsed >= timeout then 195 | return nil, "timeout" 196 | end 197 | 198 | if err then 199 | log(ERR, "could not get event at index '", self.idx, "': ", err) 200 | 201 | elseif type(v) ~= "string" then 202 | log(ERR, "event at index '", self.idx, "' is not a string, ", 203 | "shm tampered with") 204 | 205 | else 206 | local pid, channel, data = unmarshall(v) 207 | 208 | if self.pid ~= pid then 209 | -- coming from another worker 210 | local cbs = self.callbacks[channel] 211 | if cbs then 212 | for j = 1, #cbs do 213 | local pok, perr = pcall(cbs[j], data) 214 | if not pok then 215 | log(ERR, "callback for channel '", channel, 216 | "' threw a Lua error: ", perr) 217 | end 218 | end 219 | end 220 | end 221 | end 222 | end 223 | 224 | return true 225 | end 226 | 227 | 228 | return _M 229 | -------------------------------------------------------------------------------- /lualib/resty/moongoo.lua: -------------------------------------------------------------------------------- 1 | local cbson = require("cbson") 2 | local connection = require("resty.moongoo.connection") 3 | local database = require("resty.moongoo.database") 4 | local parse_uri = require("resty.moongoo.utils").parse_uri 5 | local auth_scram = require("resty.moongoo.auth.scram") 6 | local auth_cr = require("resty.moongoo.auth.cr") 7 | 8 | 9 | local _M = {} 10 | 11 | _M._VERSION = '0.1' 12 | _M.NAME = 'Moongoo' 13 | 14 | 15 | local mt = { __index = _M } 16 | 17 | function _M.new(uri) 18 | local conninfo = parse_uri(uri) 19 | 20 | if not conninfo.scheme or conninfo.scheme ~= "mongodb" then 21 | return nil, "Wrong scheme in connection uri" 22 | end 23 | 24 | local auth_algo = conninfo.query and conninfo.query.authMechanism or "SCRAM-SHA-1" 25 | local w = conninfo.query and conninfo.query.w or 1 26 | local wtimeout = conninfo.query and conninfo.query.wtimeoutMS or 1000 27 | local journal = conninfo.query and conninfo.query.journal or false 28 | local ssl = conninfo.query and conninfo.query.ssl or false 29 | 30 | local stimeout = conninfo.socketTimeoutMS and conninfo.query.socketTimeoutMS or nil 31 | 32 | return setmetatable({ 33 | connection = nil; 34 | w = w; 35 | wtimeout = wtimeout; 36 | journal = tonumber(journal); 37 | stimeout = stimeout; 38 | hosts = conninfo.hosts; 39 | default_db = conninfo.database; 40 | user = conninfo.user or nil; 41 | password = conninfo.password or ""; 42 | auth_algo = auth_algo, 43 | ssl = ssl, 44 | version = nil 45 | }, mt) 46 | end 47 | 48 | function _M._auth(self, protocol) 49 | if not self.user then return 1 end 50 | 51 | if not protocol or protocol < cbson.int(3) or self.auth_algo == "MONGODB-CR" then 52 | return auth_cr(self:db(self.default_db), self.user, self.password) 53 | else 54 | return auth_scram(self:db(self.default_db), self.user, self.password) 55 | end 56 | 57 | end 58 | 59 | function _M.connect(self) 60 | if self.connection then return self end 61 | 62 | -- foreach host 63 | for k, v in ipairs(self.hosts) do 64 | -- connect 65 | self.connection, err = connection.new(v.host, v.port, self.stimeout) 66 | if not self.connection then 67 | return nil, err 68 | end 69 | local status, err = self.connection:connect() 70 | if status then 71 | if self.ssl then 72 | self.connection:handshake() 73 | end 74 | if not self.version then 75 | query = self:db(self.default_db):_cmd({ buildInfo = 1 }) 76 | if query then 77 | self.version = query.version 78 | end 79 | end 80 | 81 | local ismaster = self:db("admin"):_cmd("ismaster") 82 | if ismaster and ismaster.ismaster then 83 | -- auth 84 | local r, err = self:_auth(ismaster.maxWireVersion) 85 | if not r then 86 | return nil, err 87 | end 88 | return self 89 | else 90 | -- try to connect to master 91 | if ismaster.primary then 92 | local mhost, mport 93 | string.gsub(ismaster.primary, "([^:]+):([^:]+)", function(host,port) mhost=host; mport=port end) 94 | self.connection:close() 95 | self.connection = nil 96 | self.connection, err = connection.new(mhost, mport, self.stimeout) 97 | if not self.connection then 98 | return nil, err 99 | end 100 | local status, err = self.connection:connect() 101 | if not status then 102 | return nil, err 103 | end 104 | if self.ssl then 105 | self.connection:handshake() 106 | end 107 | if not self.version then 108 | query = self:db(self.default_db):_cmd({ buildInfo = 1 }) 109 | if query then 110 | self.version = query.version 111 | end 112 | end 113 | local ismaster = self:db("admin"):_cmd("ismaster") 114 | if ismaster and ismaster.ismaster then 115 | -- auth 116 | local r, err = self:_auth(ismaster.maxWireVersion) 117 | if not r then 118 | return nil, err 119 | end 120 | return self 121 | else 122 | return nil, "Can't connect to master server" 123 | end 124 | end 125 | end 126 | end 127 | end 128 | return nil, "Can't connect to any of servers" 129 | end 130 | 131 | function _M.close(self) 132 | if self.connection then 133 | self.connection:close() 134 | self.connection = nil 135 | end 136 | end 137 | 138 | function _M.get_reused_times(self) 139 | return self.connection:get_reused_times() 140 | end 141 | 142 | function _M.db(self, dbname) 143 | return database.new(dbname, self) 144 | end 145 | 146 | return _M 147 | -------------------------------------------------------------------------------- /lualib/resty/moongoo/auth/cr.lua: -------------------------------------------------------------------------------- 1 | local pass_digest = require("resty.moongoo.utils").pass_digest 2 | 3 | local b64 = ngx and ngx.encode_base64 or require("mime").b64 4 | local unb64 = ngx and ngx.decode_base64 or require("mime").unb64 5 | 6 | local md5 = ngx and ngx.md5 or function(str) return require("crypto").digest("md5", str) end 7 | 8 | local cbson = require("cbson") 9 | 10 | 11 | local function auth(db, username, password) 12 | local r, err = db:_cmd("getnonce", {}) 13 | if not r then 14 | return nil, err 15 | end 16 | 17 | local digest = md5( r.nonce .. username .. pass_digest ( username , password ) ) 18 | 19 | r, err = db:_cmd("authenticate", { 20 | user = username ; 21 | nonce = r.nonce ; 22 | key = digest ; 23 | }) 24 | 25 | if not r then 26 | return nil, err 27 | end 28 | 29 | return 1 30 | end 31 | 32 | return auth -------------------------------------------------------------------------------- /lualib/resty/moongoo/auth/scram.lua: -------------------------------------------------------------------------------- 1 | local Hi = require("resty.moongoo.utils").pbkdf2_hmac_sha1 2 | local saslprep = require("resty.moongoo.utils").saslprep 3 | local pass_digest = require("resty.moongoo.utils").pass_digest 4 | local xor_bytestr = require("resty.moongoo.utils").xor_bytestr 5 | 6 | local b64 = ngx and ngx.encode_base64 or require("mime").b64 7 | local unb64 = ngx and ngx.decode_base64 or require("mime").unb64 8 | 9 | local hmac_sha1 = ngx and ngx.hmac_sha1 or function(str, key) return require("crypto").hmac.digest("sha1", key, str, true) end 10 | local sha1_bin = ngx and ngx.sha1_bin or function(str) return require("crypto").digest("sha1", str, true) end 11 | 12 | local cbson = require("cbson") 13 | 14 | 15 | local function auth(db, username, password) 16 | local username = saslprep(username) 17 | local c_nonce = b64(string.sub(tostring(math.random()), 3 , 14)) 18 | 19 | local first_bare = "n=" .. username .. ",r=" .. c_nonce 20 | 21 | local sasl_start_payload = b64("n,," .. first_bare) 22 | 23 | r, err = db:_cmd("saslStart", { 24 | mechanism = "SCRAM-SHA-1" ; 25 | autoAuthorize = 1 ; 26 | payload = cbson.binary(sasl_start_payload); 27 | }) 28 | 29 | if not r then 30 | return nil, err 31 | end 32 | 33 | 34 | local conversationId = r['conversationId'] 35 | local server_first = r['payload']:raw() 36 | 37 | local parsed_t = {} 38 | for k, v in string.gmatch(server_first, "(%w+)=([^,]*)") do 39 | parsed_t[k] = v 40 | end 41 | 42 | local iterations = tonumber(parsed_t['i']) 43 | local salt = parsed_t['s'] 44 | local s_nonce = parsed_t['r'] 45 | 46 | if not string.sub(s_nonce, 1, 12) == c_nonce then 47 | return nil, 'Server returned an invalid nonce.' 48 | end 49 | 50 | local without_proof = "c=biws,r=" .. s_nonce 51 | 52 | local pbkdf2_key = pass_digest ( username , password ) 53 | local salted_pass = Hi(pbkdf2_key, iterations, unb64(salt), 20) 54 | 55 | local client_key = hmac_sha1(salted_pass, "Client Key") 56 | local stored_key = sha1_bin(client_key) 57 | local auth_msg = first_bare .. ',' .. server_first .. ',' .. without_proof 58 | local client_sig = hmac_sha1(stored_key, auth_msg) 59 | local client_key_xor_sig = xor_bytestr(client_key, client_sig) 60 | local client_proof = "p=" .. b64(client_key_xor_sig) 61 | local client_final = b64(without_proof .. ',' .. client_proof) 62 | local server_key = hmac_sha1(salted_pass, "Server Key") 63 | local server_sig = b64(hmac_sha1(server_key, auth_msg)) 64 | 65 | r, err = db:_cmd("saslContinue",{ 66 | conversationId = conversationId ; 67 | payload = cbson.binary(client_final); 68 | }) 69 | 70 | if not r then 71 | return nil, err 72 | end 73 | 74 | local parsed_s = r['payload']:raw() 75 | parsed_t = {} 76 | for k, v in string.gmatch(parsed_s, "(%w+)=([^,]*)") do 77 | parsed_t[k] = v 78 | end 79 | if parsed_t['v'] ~= server_sig then 80 | return nil, "Server returned an invalid signature." 81 | end 82 | 83 | if not r['done'] then 84 | r, err = db:_cmd("saslContinue", { 85 | conversationId = conversationId ; 86 | payload = cbson.binary("") ; 87 | }) 88 | 89 | if not r then 90 | return nil, err 91 | end 92 | 93 | if not r['done'] then 94 | return nil, 'SASL conversation failed to complete.' 95 | end 96 | 97 | return 1 98 | end 99 | 100 | return 1 101 | end 102 | 103 | return auth 104 | -------------------------------------------------------------------------------- /lualib/resty/moongoo/collection.lua: -------------------------------------------------------------------------------- 1 | local cbson = require("cbson") 2 | local generate_oid = require("resty.moongoo.utils").generate_oid 3 | local cursor = require("resty.moongoo.cursor") 4 | 5 | local _M = {} 6 | 7 | local mt = { __index = _M } 8 | 9 | function _M.new(name, db) 10 | return setmetatable({_db = db, name = name}, mt) 11 | end 12 | 13 | function _M._build_write_concern(self) 14 | return { 15 | j = self._db._moongoo.journal; 16 | w = tonumber(self._db._moongoo.w) and cbson.int(self._db._moongoo.w) or self._db._moongoo.w; 17 | wtimeout = cbson.int(self._db._moongoo.wtimeout); 18 | } 19 | end 20 | 21 | local function check_write_concern(doc, ...) 22 | -- even if write concern failed we may still have successful operation 23 | -- so we check for number of affected docs, and only warn if its > 0 24 | -- otherwise, we just return nil and error 25 | 26 | if doc.writeConcernError then 27 | if not doc.n then 28 | return nil, doc.writeConcernError.errmsg 29 | else 30 | print(doc.writeConcernError.errmsg) 31 | end 32 | end 33 | return ... 34 | end 35 | 36 | function _M._get_last_error(self) 37 | local write_concern = self:_build_write_concern() 38 | local cmd = { getLastError = cbson.int(1), j = write_concern.j, w = write_concern.w, wtimeout = write_concern.wtimeout } 39 | 40 | local doc, err = self._db:cmd(cmd) 41 | if not doc then 42 | return nil, err 43 | end 44 | 45 | return doc 46 | end 47 | 48 | function _M._check_last_error(self, ...) 49 | local cmd, err = self:_get_last_error() 50 | 51 | if not cmd then 52 | return nil, err 53 | end 54 | 55 | if tostring(cmd.err) == "null" then 56 | return ... 57 | end 58 | 59 | return nil, tostring(cmd.err) 60 | end 61 | 62 | local function ensure_oids(docs) 63 | local docs = docs 64 | local ids = {} 65 | for k,v in ipairs(docs) do 66 | if not docs[k]._id then 67 | docs[k]._id = cbson.oid(generate_oid()) 68 | end 69 | table.insert(ids, docs[k]._id) 70 | end 71 | return docs, ids 72 | end 73 | 74 | local function build_index_names(docs) 75 | local docs = docs 76 | for k,v in ipairs(docs) do 77 | if not v.name then 78 | local name = {} 79 | for n, d in pairs(v.key) do 80 | table.insert(name, n) 81 | end 82 | name = table.concat(name, '_') 83 | docs[k].name = name 84 | end 85 | end 86 | return docs 87 | end 88 | 89 | function _M.insert(self, docs) 90 | -- ensure we have oids 91 | if #docs == 0 then 92 | local newdocs = {} 93 | newdocs[1] = docs 94 | docs = newdocs 95 | end 96 | local docs, ids = ensure_oids(docs) 97 | 98 | self._db._moongoo:connect() 99 | 100 | local server_version = tonumber(string.sub(string.gsub(self._db._moongoo.version, "(%D)", ""), 1, 3)) 101 | 102 | if server_version < 254 then 103 | self._db:insert(self:full_name(), docs) 104 | return self:_check_last_error(ids) 105 | else 106 | local doc, err = self._db:cmd( 107 | { insert = self.name }, 108 | { 109 | documents = docs, 110 | ordered = true, 111 | writeConcern = self:_build_write_concern() 112 | } 113 | ) 114 | 115 | if not doc then 116 | return nil, err 117 | end 118 | 119 | return check_write_concern(doc, ids, doc.n) 120 | end 121 | end 122 | 123 | function _M.create(self, params) 124 | local params = params or {} 125 | local doc, err = self._db:cmd( 126 | { create = self.name }, 127 | params 128 | ) 129 | if not doc then 130 | return nil, err 131 | end 132 | return true 133 | end 134 | 135 | function _M.drop(self) 136 | local doc, err = self._db:cmd( 137 | { drop = self.name }, 138 | {} 139 | ) 140 | if not doc then 141 | return nil, err 142 | end 143 | return true 144 | end 145 | 146 | function _M.drop_index(self, name) 147 | local doc, err = self._db:cmd( 148 | { dropIndexes = self.name }, 149 | { index = name } 150 | ) 151 | if not doc then 152 | return nil, err 153 | end 154 | return true 155 | end 156 | 157 | function _M.ensure_index(self, docs) 158 | docs = build_index_names(docs) 159 | 160 | local doc, err = self._db:cmd( 161 | { createIndexes = self.name }, 162 | { indexes = docs } 163 | ) 164 | if not doc then 165 | return nil, err 166 | end 167 | return true 168 | end 169 | 170 | function _M.full_name(self) 171 | return self._db.name .. "." .. self.name 172 | end 173 | 174 | function _M.options(self) 175 | local doc, err = self._db:cmd( 176 | "listCollections", 177 | { 178 | filter = { name = self.name } 179 | } 180 | ) 181 | if not doc then 182 | return nil, err 183 | end 184 | return doc.cursor.firstBatch[1] 185 | end 186 | 187 | function _M.remove(self, query, single) 188 | local query = query or {} 189 | 190 | if getmetatable(cbson.oid("000000000000000000000000")) == getmetatable(query) then 191 | query = { _id = query } 192 | end 193 | 194 | local doc, err = self._db:cmd( 195 | { delete = self.name }, 196 | { 197 | deletes = {{q=query, limit = single and 1 or 0}}, 198 | ordered = true, 199 | writeConcern = self:_build_write_concern() 200 | } 201 | ) 202 | if not doc then 203 | return nil, err 204 | end 205 | 206 | return check_write_concern(doc, doc.n) 207 | end 208 | 209 | function _M.stats(self) 210 | local doc, err = self._db:cmd( 211 | {collstats = self.name}, 212 | {} 213 | ) 214 | if not doc then 215 | return nil, err 216 | end 217 | return doc 218 | end 219 | 220 | function _M.index_information(self) 221 | local doc, err = self._db:cmd( 222 | { listIndexes = self.name }, 223 | { } 224 | ) 225 | if not doc then 226 | return nil, err 227 | end 228 | return doc.cursor.firstBatch 229 | end 230 | 231 | function _M.rename(self, to_name, drop) 232 | local drop = drop or false 233 | -- rename 234 | local doc, err = self._db._moongoo:db("admin"):cmd( 235 | { renameCollection = self:full_name() }, 236 | { 237 | to = to_name, 238 | dropTarget = drop 239 | } 240 | ) 241 | if not doc then 242 | return nil, err 243 | end 244 | 245 | return self.new(to_name, self._db) 246 | end 247 | 248 | function _M.update(self, query, update, flags) 249 | local flags = flags or {} 250 | local query = query or {} 251 | 252 | if getmetatable(cbson.oid("000000000000000000000000")) == getmetatable(query) then 253 | query = { _id = query } 254 | end 255 | 256 | local update = { 257 | q = query, 258 | u = update, 259 | upsert = flags.upsert or false, 260 | multi = flags.multi or false 261 | } 262 | 263 | local doc, err = self._db:cmd( 264 | { update = self.name }, 265 | { 266 | updates = { update }, 267 | ordered = true, 268 | writeConcern = self:_build_write_concern() 269 | } 270 | ) 271 | if not doc then 272 | return nil, err 273 | end 274 | 275 | return doc 276 | end 277 | 278 | function _M.save(self, doc) 279 | if not doc._id then 280 | doc._id = cbson.oid(generate_oid()) 281 | end 282 | local r, err = self:update(doc._id, doc, {upsert = true}); 283 | if not r then 284 | return nil, err 285 | end 286 | 287 | return doc._id 288 | end 289 | 290 | function _M.map_reduce(self, map, reduce, flags) 291 | local flags = flags or {} 292 | flags.map = cbson.code(map) 293 | flags.reduce = cbson.code(reduce) 294 | flags.out = flags.out or { inline = true } 295 | 296 | local doc, err = self._db:cmd( 297 | { mapReduce = self.name }, 298 | flags 299 | ) 300 | if not doc then 301 | return nil, err 302 | end 303 | 304 | if doc.results then 305 | return doc.results 306 | end 307 | 308 | return self.new(doc.result, self._db) 309 | end 310 | 311 | function _M.find(self, query, fields) 312 | local query = query or {} 313 | if getmetatable(cbson.oid("000000000000000000000000")) == getmetatable(query) then 314 | query = { _id = query } 315 | end 316 | return cursor.new(self, query, fields) 317 | end 318 | 319 | function _M.find_one(self, query, fields) 320 | local query = query or {} 321 | if getmetatable(cbson.oid("000000000000000000000000")) == getmetatable(query) then 322 | query = { _id = query } 323 | end 324 | 325 | return self:find(query, fields):limit(-1):next() 326 | end 327 | 328 | function _M.find_and_modify(self, query, opts) 329 | local query = query or {} 330 | if getmetatable(cbson.oid("000000000000000000000000")) == getmetatable(query) then 331 | query = { _id = query } 332 | end 333 | 334 | local opts = opts or {} 335 | opts.query = query 336 | 337 | local doc, err = self._db:cmd( 338 | { findAndModify = self.name }, 339 | opts 340 | ) 341 | if not doc then 342 | return nil, err 343 | end 344 | return doc.value 345 | end 346 | 347 | function _M.aggregate(self, pipeline, opts) 348 | local opts = opts or {} 349 | opts.pipeline = pipeline 350 | if not opts.explain then 351 | opts.cursor = {} 352 | end 353 | 354 | local doc, err = self._db:cmd( 355 | { aggregate = self.name }, 356 | opts 357 | ) 358 | if not doc then 359 | return nil, err 360 | end 361 | 362 | if opts.explain then 363 | return doc 364 | end 365 | 366 | -- collection 367 | if opts.pipeline[#opts.pipeline]['$out'] then 368 | return self.new(opts.pipeline[#opts.pipeline]['$out'], self._db) 369 | end 370 | 371 | -- cursor 372 | return cursor.new(self, {}, {}, false, doc.cursor.id):add_batch(doc.cursor.firstBatch) 373 | end 374 | 375 | 376 | 377 | return _M 378 | -------------------------------------------------------------------------------- /lualib/resty/moongoo/connection.lua: -------------------------------------------------------------------------------- 1 | local socket = ngx and ngx.socket.tcp or require("socket").tcp 2 | local cbson = require("cbson") 3 | 4 | local opcodes = { 5 | OP_REPLY = 1; 6 | OP_MSG = 1000; 7 | OP_UPDATE = 2001; 8 | OP_INSERT = 2002; 9 | RESERVED = 2003; 10 | OP_QUERY = 2004; 11 | OP_GET_MORE = 2005; 12 | OP_DELETE = 2006; 13 | OP_KILL_CURSORS = 2007; 14 | } 15 | 16 | local _M = {} 17 | 18 | local mt = { __index = _M } 19 | 20 | function _M.new(host, port, timeout) 21 | local sock = socket() 22 | if timeout then 23 | sock:settimeout(timeout) 24 | end 25 | 26 | return setmetatable({ 27 | sock = sock; 28 | host = host; 29 | port = port; 30 | _id = 0; 31 | }, mt) 32 | end 33 | 34 | function _M.connect(self, host, port) 35 | self.host = host or self.host 36 | self.port = port or self.port 37 | return self.sock:connect(self.host, self.port) 38 | end 39 | 40 | function _M.handshake(self) 41 | if ngx then 42 | self.sock:sslhandshake() 43 | else 44 | local ssl = require("ssl") 45 | self.sock = ssl.wrap(self.sock, {mode = "client", protocol = "tlsv1_2"}) 46 | assert(self.sock) 47 | self.sock:dohandshake() 48 | end 49 | end 50 | 51 | function _M.close(self) 52 | if ngx then 53 | self.sock:setkeepalive() 54 | else 55 | self.sock:close() 56 | end 57 | end 58 | 59 | function _M.get_reused_times(self) 60 | if not self.sock then 61 | return nil, "not initialized" 62 | end 63 | 64 | return self.sock:getreusedtimes() 65 | end 66 | 67 | function _M.settimeout(self, ms) 68 | self.sock:settimeout(ms) 69 | end 70 | 71 | function _M.send(self, data) 72 | return self.sock:send(data) 73 | end 74 | 75 | function _M.receive(self, pat) 76 | return self.sock:receive(pat) 77 | end 78 | 79 | function _M._handle_reply(self) 80 | local header = assert ( self.sock:receive ( 16 ) ) 81 | 82 | local length = cbson.raw_to_uint( string.sub(header , 1 , 4 )) 83 | local r_id = cbson.raw_to_uint( string.sub(header , 5 , 8 )) 84 | local r_to = cbson.raw_to_uint( string.sub(header , 9 , 12 )) 85 | local opcode = cbson.raw_to_uint( string.sub(header , 13 , 16 )) 86 | 87 | assert ( opcode == cbson.uint(opcodes.OP_REPLY ) ) 88 | assert ( r_to == cbson.uint(self._id) ) 89 | 90 | local data = assert ( self.sock:receive ( tostring(length-16 ) ) ) 91 | 92 | local flags = cbson.raw_to_uint( string.sub(data , 1 , 4 )) 93 | local cursor_id = cbson.raw_to_uint( string.sub(data , 5 , 12 )) 94 | local from = cbson.raw_to_uint( string.sub(data , 13 , 16 )) 95 | local number = tonumber(tostring(cbson.raw_to_uint( string.sub(data , 17 , 20 )))) 96 | 97 | local docs = string.sub(data , 21) 98 | 99 | local pos = 1 100 | local index = 0 101 | local r_docs = {} 102 | while index < number do 103 | local bson_size = tonumber(tostring(cbson.raw_to_uint(docs:sub(pos, pos+3)))) 104 | 105 | local dt = docs:sub(pos,pos+bson_size-1) -- get bson data according to size 106 | 107 | table.insert(r_docs, cbson.decode(dt)) 108 | 109 | pos = pos + bson_size 110 | index = index + 1 111 | end 112 | 113 | return flags, cursor_id, from, number, r_docs 114 | end 115 | 116 | function _M._build_header(self, op, payload_size) 117 | local size = cbson.uint_to_raw(cbson.uint(payload_size+16), 4) 118 | local op = cbson.uint_to_raw(cbson.uint(op), 4) 119 | self._id = self._id+1 120 | local id = cbson.uint_to_raw(cbson.uint(self._id), 4) 121 | local reply_to = "\0\0\0\0" 122 | return size .. id .. reply_to .. op 123 | end 124 | 125 | function _M._query(self, collection, query, to_skip, to_return, selector, flags) 126 | local flags = { 127 | tailable = flags and flags.tailable and 1 or 0, 128 | slaveok = flags and flags.slaveok and 1 or 0, 129 | notimeout = flags and flags.notimeout and 1 or 0, 130 | await = flags and flags.await and 1 or 0, 131 | exhaust = flags and flags.exhaust and 1 or 0, 132 | partial = flags and flags.partial and 1 or 0 133 | } 134 | 135 | local flagset = cbson.int_to_raw( 136 | cbson.int( 137 | 2 * flags["tailable"] + 138 | 2^2 * flags["slaveok"] + 139 | 2^4 * flags["notimeout"] + 140 | 2^5 * flags["await"] + 141 | 2^6 * flags["exhaust"] + 142 | 2^7 * flags["partial"] 143 | ), 144 | 4) 145 | 146 | local selector = selector and #selector and cbson.encode(selector) or "" 147 | 148 | local to_skip = cbson.int_to_raw(cbson.int(to_skip), 4) 149 | local to_return = cbson.int_to_raw(cbson.int(to_return), 4) 150 | 151 | local size = 4 + #collection + 1 + 4 + 4 + #query + #selector 152 | 153 | local header = self:_build_header(opcodes["OP_QUERY"], size) 154 | 155 | local data = header .. flagset .. collection .. "\0" .. to_skip .. to_return .. query .. selector 156 | 157 | assert(self:send(data)) 158 | return self:_handle_reply() 159 | end 160 | 161 | function _M._insert(self, collection, docs, flags) 162 | local encoded_docs = {} 163 | for k, doc in ipairs(docs) do 164 | encoded_docs[k] = cbson.encode(doc) 165 | end 166 | string_docs = table.concat(encoded_docs) 167 | 168 | local flags = { 169 | continue_on_error = flags and flags.continue_on_error and 1 or 0 170 | } 171 | 172 | local flagset = cbson.int_to_raw( 173 | cbson.int( 174 | 2 * flags["continue_on_error"] 175 | ), 176 | 4) 177 | 178 | local size = 4 + 1 + #collection + #string_docs 179 | local header = self:_build_header(opcodes["OP_INSERT"], size) 180 | 181 | local data = header .. flagset .. collection .. "\0" .. string_docs 182 | 183 | assert(self:send(data)) 184 | 185 | return true -- Mongo doesn't send a reply 186 | end 187 | 188 | function _M._kill_cursors(self, id) 189 | local id = cbson.uint_to_raw(id, 8) 190 | local num = cbson.int_to_raw(cbson.int(1), 4) 191 | local zero = cbson.int_to_raw(cbson.int(0), 4) 192 | local size = 8+4+4 193 | local header = self:_build_header(opcodes["OP_KILL_CURSORS"], size) 194 | local data = header .. zero .. num .. id 195 | assert(self:send(data)) 196 | return true -- Mongo doesn't send a reply 197 | end 198 | 199 | function _M._get_more(self, collection, number, cursor) 200 | local num = cbson.int_to_raw(cbson.int(number), 4) 201 | local zero = cbson.int_to_raw(cbson.int(0), 4) 202 | local cursor = cbson.uint_to_raw(cursor, 8) 203 | local size = 4+#collection+1+4+8 204 | local header = self:_build_header(opcodes["OP_GET_MORE"], size) 205 | local data = header .. zero .. collection .. '\0' .. num .. cursor 206 | assert(self:send(data)) 207 | return self:_handle_reply() 208 | end 209 | 210 | return _M -------------------------------------------------------------------------------- /lualib/resty/moongoo/cursor.lua: -------------------------------------------------------------------------------- 1 | local cbson = require("cbson") 2 | local bit = require("bit") 3 | 4 | 5 | local function check_bit(num, bitnum) 6 | return bit.band(num,math.pow(2,bitnum)) ~= 0 -- and true or false 7 | end 8 | 9 | local _M = {} 10 | 11 | local mt = { __index = _M } 12 | 13 | function _M.new(collection, query, fields, explain, id) 14 | return setmetatable( 15 | { 16 | _collection = collection, 17 | _query = query, 18 | _fields = fields, 19 | _id = id or cbson.uint(0), 20 | _skip = 0, 21 | _limit = 0, 22 | _docs = {}, 23 | _started = false, 24 | _cnt = 0, 25 | _comment = nil, 26 | _hint = nil, 27 | _max_scan = nil , 28 | _max_time_ms = nil, 29 | _read_preference = nil, 30 | _snapshot = nil, 31 | _sort = nil, 32 | _await = false, 33 | _tailable = false, 34 | _explain = explain or false 35 | }, 36 | mt) 37 | end 38 | 39 | function _M.tailable(self, tailable) 40 | self._tailable = tailable 41 | return self 42 | end 43 | 44 | function _M.await(self, await) 45 | self._await = await 46 | return self 47 | end 48 | 49 | 50 | function _M.comment(self, comment) 51 | self._comment = comment 52 | return self 53 | end 54 | 55 | function _M.hint(self, hint) 56 | self._hint = hint 57 | return self 58 | end 59 | 60 | function _M.max_scan(self, max_scan) 61 | self._max_scan = max_scan 62 | return self 63 | end 64 | 65 | function _M.max_time_ms(self, max_time_ms) 66 | self._max_time_ms = max_time_ms 67 | return self 68 | end 69 | 70 | function _M.read_preference(self, read_preference) 71 | self._read_preference = read_preference 72 | return self 73 | end 74 | 75 | function _M.snapshot(self, snapshot) 76 | self._snapshot = snapshot 77 | return self 78 | end 79 | 80 | function _M.sort(self, sort) 81 | self._sort = sort 82 | return self 83 | end 84 | 85 | 86 | function _M.clone(self, explain) 87 | local clone = self.new(self._collection, self._query, self._fields, explain) 88 | clone:limit(self._limit) 89 | clone:skip(self._skip) 90 | 91 | clone:comment(self._comment) 92 | clone:hint(self._hint) 93 | clone:max_scan(self._max_scan) 94 | clone:max_time_ms(self._max_time_ms) 95 | clone:read_preference(self._read_preference) 96 | clone:snapshot(self._snapshot) 97 | clone:sort(self._sort) 98 | 99 | return clone 100 | end 101 | 102 | function _M.skip(self, skip) 103 | if self._started then 104 | print("Can's set skip after starting cursor") 105 | else 106 | self._skip = skip 107 | end 108 | return self 109 | end 110 | 111 | function _M.limit(self, limit) 112 | if self._started then 113 | print("Can's set limit after starting cursor") 114 | else 115 | self._limit = limit 116 | end 117 | return self 118 | end 119 | 120 | function _M._build_query(self) 121 | local ext = {} 122 | if self._comment then ext['$comment'] = self._comment end 123 | if self._explain then ext['$explain'] = true end 124 | 125 | 126 | if self._hint then ext['$hint'] = self._hint end 127 | if self._max_scan then ext['$maxScan'] = self._max_scan end 128 | if self._max_time_ms then ext['$maxTimeMS'] = self._max_time_ms end 129 | if self._read_preference then ext['$readPreference'] = self._read_preference end 130 | if self._snapshot then ext['$snapshot'] = true end 131 | if self._sort then ext['$orderby'] = self._sort end 132 | 133 | ext['$query'] = self._query 134 | 135 | return cbson.encode(ext) 136 | end 137 | 138 | function _M.next(self) 139 | local moongoo, err = self._collection._db._moongoo:connect() 140 | if not moongoo then 141 | return nil, err 142 | end 143 | local ret, err = _next(self) 144 | if not ret then 145 | moongoo:close() 146 | end 147 | return ret, err 148 | end 149 | 150 | function _next(self) 151 | 152 | if self:_finished() then 153 | if self._id ~= cbson.uint(0) then 154 | self._collection._db._moongoo.connection:_kill_cursors(self._id) 155 | self._id = cbson.uint(0) 156 | end 157 | return nil, "no more data" 158 | end 159 | 160 | if (not self._started) and (self._id == cbson.uint(0)) then 161 | 162 | -- query and add id and batch 163 | local flags, id, from, number, docs = self._collection._db._moongoo.connection:_query(self._collection:full_name(), self:_build_query(), self._skip, self._limit, self._fields, {tailable = self._tailable, await = self._await}) 164 | 165 | flags = tonumber(tostring(flags)) -- bitop can't work with cbson.int, so... 166 | 167 | if check_bit(flags, 1) then -- QueryFailure 168 | return nil, docs[1]['$err'] -- why is this $err and not errmsg, like others?? 169 | end 170 | self._id = id 171 | self:add_batch(docs) 172 | elseif #self._docs == 0 and self._id ~= cbson.uint(0) then 173 | -- we have something to fetch - get_more and add_batch 174 | local flags, id, from, number, docs = self._collection._db._moongoo.connection:_get_more(self._collection:full_name(), self._limit, self._id) 175 | 176 | flags = tonumber(tostring(flags)) -- bitop can't work with cbson.int, so... 177 | 178 | if check_bit(flags, 0) then -- QueryFailure 179 | return nil, "wrong cursor id" 180 | end 181 | self:add_batch(docs) 182 | self._id = id 183 | 184 | elseif #self._docs == 0 then--or self._id == cbson.uint(0) then 185 | return nil, "no more data" 186 | end 187 | self._cnt = self._cnt+1 188 | return table.remove(self._docs, 1) or nil, 'No more data' 189 | end 190 | 191 | function _M.all(self) 192 | local docs = {} 193 | while true do 194 | local doc = self:next() 195 | if doc == nil then break end 196 | table.insert(docs, doc) 197 | end 198 | return docs 199 | end 200 | 201 | function _M.rewind(self) 202 | self._started = false 203 | self._docs = {} 204 | self._collection._db._moongoo.connection:_kill_cursors(self._id) 205 | self._id = cbson.uint(0) 206 | return self 207 | end 208 | 209 | function _M.count(self) 210 | local doc, err = self._collection._db:cmd( 211 | { count = self._collection.name }, 212 | { 213 | query = self._query, 214 | skip = self._skip, 215 | limit = self._limit 216 | } 217 | ) 218 | if not doc then 219 | return nil, err 220 | end 221 | 222 | return doc and doc.n or 0 223 | end 224 | 225 | function _M.distinct(self, key) 226 | local doc, err = self._collection._db:cmd( 227 | { distinct = self._collection.name }, 228 | { 229 | query = self._query, 230 | key = key 231 | } 232 | ) 233 | if not doc then 234 | return nil, err 235 | end 236 | 237 | return doc and doc.values or {} 238 | end 239 | 240 | function _M.explain(self) 241 | return self:clone(true):sort(nil):next() 242 | end 243 | 244 | function _M.add_batch(self, docs) 245 | self._started = true 246 | for k,v in ipairs(docs) do 247 | table.insert(self._docs, v) 248 | end 249 | return self 250 | end 251 | 252 | function _M._finished(self) 253 | if self._limit == 0 then 254 | return false 255 | else 256 | if self._cnt >= math.abs(self._limit) then 257 | return true 258 | else 259 | return false 260 | end 261 | end 262 | end 263 | 264 | return _M 265 | -------------------------------------------------------------------------------- /lualib/resty/moongoo/database.lua: -------------------------------------------------------------------------------- 1 | local cbson = require("cbson") 2 | local collection = require("resty.moongoo.collection") 3 | local gridfs = require("resty.moongoo.gridfs") 4 | 5 | local _M = {} 6 | 7 | local mt = { __index = _M } 8 | 9 | function _M.new(name, moongoo) 10 | return setmetatable({name = name, _moongoo = moongoo}, mt) 11 | end 12 | 13 | function _M.collection(self, name) 14 | return collection.new(name, self) 15 | end 16 | 17 | function _M.gridfs(self, name) 18 | return gridfs.new(self,name) 19 | end 20 | 21 | function _M.cmd(self, cmd, params) 22 | local r, err = self._moongoo:connect() 23 | if not r then 24 | return nil, err 25 | end 26 | return self:_cmd(cmd, params) 27 | end 28 | 29 | function _M._cmd(self, cmd, params) 30 | local params = params or {} 31 | if type(cmd) == "table" then 32 | local tmpcmd = '' 33 | for k,v in pairs(cmd) do 34 | params[k] = v 35 | tmpcmd = k 36 | end 37 | cmd = tmpcmd 38 | else 39 | params[cmd] = true 40 | end 41 | local cmd = cbson.encode_first(cmd, params) 42 | 43 | local _,_,_,_,docs = self._moongoo.connection:_query(self.name..".$cmd", cmd, 0, 1) 44 | 45 | if not docs[1] then 46 | return nil, "Empty reply from mongodb" 47 | end 48 | 49 | if not docs[1].ok or docs[1].ok == 0 then 50 | return nil, docs[1].errmsg 51 | end 52 | 53 | return docs[1] 54 | end 55 | 56 | function _M.insert(self, collection, docs) 57 | local r, err = self._moongoo:connect() 58 | if not r then 59 | return nil, err 60 | end 61 | return self:_insert(collection, docs) 62 | end 63 | 64 | function _M._insert(self, collection, docs) 65 | self._moongoo.connection:_insert(collection, docs) 66 | return 67 | end 68 | 69 | 70 | return _M -------------------------------------------------------------------------------- /lualib/resty/moongoo/gridfs.lua: -------------------------------------------------------------------------------- 1 | local cbson = require("cbson") 2 | local gfsfile = require("resty.moongoo.gridfs.file") 3 | 4 | local _M = {} 5 | 6 | local mt = { __index = _M } 7 | 8 | function _M.new(db, name) 9 | local name = name or 'fs' 10 | return setmetatable( 11 | { 12 | _db = db, 13 | _name = name, 14 | _files = db:collection(name .. '.files'), 15 | _chunks = db:collection(name .. '.chunks') 16 | }, 17 | mt) 18 | end 19 | 20 | function _M.list(self) 21 | return self._files:find({}):distinct('filename') 22 | end 23 | 24 | function _M.remove(self, id) 25 | local r,err = self._files:remove({_id = cbson.oid(id)}) 26 | if not r then 27 | return nil, "Failed to remove file metadata: "..err 28 | end 29 | r,err = self._chunks:remove({files_id = cbson.oid(id)}); 30 | if not r then 31 | return nil, "Failed to remove file chunks: "..err 32 | end 33 | return r 34 | end 35 | 36 | function _M.find_version(self, name, version) 37 | -- Positive numbers are absolute and negative ones relative 38 | local cursor = self._files:find({filename = name}, {_id = 1}):limit(-1) 39 | cursor:sort({uploadDate = (version < 0) and cbson.int(-1) or cbson.int(1)}):skip(version < 0 and (math.abs(version) - 1) or version) 40 | local doc, err = cursor:next() 41 | if not doc then 42 | return nil, "No such file/version" 43 | end 44 | return doc._id 45 | end 46 | 47 | function _M.open(self, id) 48 | return gfsfile.open(self, id) 49 | end 50 | 51 | 52 | function _M.create(self, name, opts, safe) 53 | return gfsfile.new(self, name, opts, safe) 54 | end 55 | 56 | return _M 57 | 58 | -------------------------------------------------------------------------------- /lualib/resty/moongoo/gridfs/file.lua: -------------------------------------------------------------------------------- 1 | local cbson = require("cbson") 2 | local generate_oid = require("resty.moongoo.utils").generate_oid 3 | 4 | 5 | local _M = {} 6 | 7 | local mt = { __index = _M } 8 | 9 | function _M.new(gridfs, name, opts, safe, read_only) 10 | local read_only = read_only or false 11 | local safe = safe == nil and true or safe 12 | 13 | opts = opts or {} 14 | opts.filename = name 15 | opts.length = opts.length or cbson.uint(0) 16 | opts.chunkSize = opts.chunkSize or cbson.uint(261120) 17 | 18 | local write_only = true 19 | if not safe then 20 | write_only = false 21 | end 22 | 23 | return setmetatable( 24 | { 25 | _gridfs = gridfs, 26 | _meta = opts, 27 | _write_only = write_only, 28 | _read_only = read_only, 29 | _pos = 0, 30 | _chunks = {}, 31 | _closed = false, 32 | _buffer = '', 33 | _n = 0 34 | }, 35 | mt) 36 | end 37 | 38 | function _M.open(gridfs, id) 39 | -- try to fetch 40 | local doc, err = gridfs._files:find_one({ _id = id}) 41 | if not doc then 42 | return nil, "No such file" 43 | else 44 | return _M.new(gridfs, doc.filename, doc, false, true) 45 | end 46 | end 47 | 48 | -- props 49 | 50 | function _M.content_type(self) return self._meta.contentType end 51 | function _M.filename(self) return self._meta.filename end 52 | function _M.md5(self) return self._meta.md5 end 53 | function _M.metadata(self) return self._meta.metadata end 54 | function _M.raw_length(self) return self._meta.length end 55 | function _M.raw_chunk_size(self) return self._meta.chunkSize end 56 | function _M.date(self) return self._meta.uploadDate end 57 | 58 | function _M.length(self) return tonumber(tostring(self._meta.length)) end 59 | function _M.chunk_size(self) return tonumber(tostring(self._meta.chunkSize)) end 60 | 61 | -- reading 62 | 63 | function _M.read(self) 64 | if self._write_only then 65 | return nil, "Can't read from write-only file" 66 | end 67 | 68 | if self._pos >= (self:length() or 0) then 69 | return nil, "EOF" 70 | end 71 | 72 | local n = math.modf(self._pos / self:chunk_size()) 73 | local query = {files_id = self._meta._id, n = n} 74 | local fields = {_id = false, data = true} 75 | 76 | local chunk = self._gridfs._chunks:find_one(query, fields) 77 | if not chunk then 78 | return nil, "EOF?" 79 | end 80 | 81 | return self:_slice(n, chunk.data) 82 | end 83 | 84 | function _M.seek(self, pos) 85 | self._pos = pos 86 | return self 87 | end 88 | 89 | function _M.tell(self) 90 | return self._pos 91 | end 92 | 93 | function _M.slurp(self) 94 | local data = {} 95 | local pos = self._pos 96 | self:seek(0) 97 | while true do 98 | local chunk = self:read() 99 | if not chunk then break end 100 | table.insert(data, chunk) 101 | end 102 | self:seek(pos) 103 | return table.concat(data) 104 | end 105 | 106 | -- writing 107 | 108 | function _M.write(self, data) 109 | if self._read_only then 110 | return nil, "Can't write to read-only file" 111 | end 112 | 113 | if self._closed then 114 | return nil, "Can't write to closed file" 115 | end 116 | 117 | self._buffer = self._buffer .. data 118 | self._meta.length = self._meta.length + data:len() 119 | 120 | while self._buffer:len() >= self:chunk_size() do 121 | local r, res = self:_chunk() 122 | if not r then 123 | return nil, err 124 | end 125 | end 126 | end 127 | 128 | function _M.close(self) 129 | 130 | if self._closed then 131 | return nil, "File already closed" 132 | end 133 | self._closed = true 134 | 135 | self:_chunk() -- enqueue/write last chunk of data 136 | 137 | if self._write_only then 138 | -- insert all collected chunks 139 | for k, v in ipairs(self._chunks) do 140 | local r, err = self._gridfs._chunks:insert(v) 141 | if not r then 142 | return nil, err 143 | end 144 | end 145 | end 146 | 147 | -- ensure indexes 148 | self._gridfs._files:ensure_index({{ key = {filename = true}}}) 149 | self._gridfs._chunks:ensure_index({ { key = {files_id = 1, n = 1}, unique = true } }); 150 | -- compute md5 151 | local file_md5 = self._gridfs._db:cmd({ filemd5 = self:_files_id(), root = self._gridfs._name }).md5 152 | -- insert metadata 153 | local ids, n = self._gridfs._files:insert(self:_metadata(file_md5)) 154 | 155 | if not ids then 156 | return nil, n 157 | end 158 | -- return metadata 159 | return ids[1] 160 | end 161 | 162 | -- private 163 | 164 | function _M._files_id(self) 165 | if not self._meta._id then 166 | self._meta._id = cbson.oid(generate_oid()) 167 | end 168 | return self._meta._id 169 | end 170 | 171 | function _M._metadata(self, file_md5) 172 | local doc = { 173 | _id = self:_files_id(), 174 | length = self:raw_length(), 175 | chunkSize = self:raw_chunk_size(), 176 | uploadDate = cbson.date(os.time(os.date('!*t'))*1000), 177 | md5 = file_md5, 178 | filename = self:filename() or nil, 179 | content_type = self:content_type() or nil, 180 | metadata = self:metadata() or nil 181 | } 182 | 183 | return doc 184 | end 185 | 186 | function _M._slice(self, n, chunk) 187 | local offset = self._pos - (n * self:chunk_size()) 188 | local chunk = chunk:raw() 189 | self._pos = self._pos + chunk:len() 190 | return chunk:sub(offset+1); 191 | end 192 | 193 | function _M._chunk(self) 194 | local chunk = self._buffer:sub(1,self:chunk_size()) 195 | if not chunk then 196 | return 197 | end 198 | self._buffer = self._buffer:sub(self:chunk_size()+1) 199 | local n = self._n 200 | self._n = self._n+1 201 | local data = cbson.binary("") 202 | data:raw(chunk, chunk:len()) 203 | if self._write_only then 204 | -- collect chunks for insert 205 | table.insert(self._chunks, {files_id = self:_files_id(), n = cbson.uint(n), data = data}) 206 | return true 207 | else 208 | -- insert immidiately, so we can read back (ugh) 209 | return self._gridfs._chunks:insert({{files_id = self:_files_id(), n = cbson.uint(n), data = data}}) 210 | end 211 | end 212 | 213 | 214 | 215 | return _M -------------------------------------------------------------------------------- /lualib/resty/moongoo/utils.lua: -------------------------------------------------------------------------------- 1 | local bit = require("bit") 2 | local cbson = require("cbson") 3 | 4 | local md5 = ngx and ngx.md5 or function(str) return require("crypto").digest("md5", str) end 5 | local hmac_sha1 = ngx and ngx.hmac_sha1 or function(str, key) return require("crypto").hmac.digest("sha1", key, str, true) end 6 | local hasposix , posix = pcall(require, "posix") 7 | 8 | local machineid 9 | if hasposix then 10 | machineid = posix.uname("%n") 11 | else 12 | machineid = assert(io.popen("uname -n")):read("*l") 13 | end 14 | machineid = md5(machineid):sub(1, 6) 15 | 16 | local function uint_to_hex(num, len, be) 17 | local len = len or 4 18 | local be = be or 0 19 | local num = cbson.uint(num) 20 | local raw = cbson.uint_to_raw(num, len, be) 21 | local out = '' 22 | for i = 1, #raw do 23 | out = out .. string.format("%02x", raw:byte(i,i)) 24 | end 25 | return out 26 | end 27 | 28 | local counter = 0 29 | 30 | if not ngx then 31 | math.randomseed(os.time()) 32 | counter = math.random(100) 33 | else 34 | local resty_random = require "resty.random" 35 | local resty_string = require "resty.string" 36 | local strong_random = resty_random.bytes(4,true) 37 | while strong_random == nil do 38 | strong_random = resty_random.bytes(4,true) 39 | end 40 | counter = tonumber(resty_string.to_hex(strong_random), 16) 41 | end 42 | 43 | local function generate_oid() 44 | local pid = ngx and ngx.worker.pid() or nil 45 | if not pid then 46 | if hasposix then 47 | pid = posix.getpid("pid") 48 | else 49 | pid = 1 50 | end 51 | end 52 | 53 | pid = uint_to_hex(pid,2) 54 | 55 | counter = counter + 1 56 | local time = os.time() 57 | 58 | return uint_to_hex(time, 4, 1) .. machineid .. pid .. uint_to_hex(counter, 4, 1):sub(3,8) 59 | end 60 | 61 | local function print_r(t, indent) 62 | local indent=indent or '' 63 | if #indent > 5 then return end 64 | if type(t) ~= "table" then 65 | print(t) 66 | return 67 | end 68 | for key,value in pairs(t) do 69 | io.write(indent,'[',tostring(key),']') 70 | if type(value)=="table" then io.write(':\n') print_r(value,indent..'\t') 71 | else io.write(' = ',tostring(value),'\n') end 72 | end 73 | end 74 | 75 | local function parse_uri(url) 76 | -- initialize default parameters 77 | local parsed = {} 78 | -- empty url is parsed to nil 79 | if not url or url == "" then return nil, "invalid url" end 80 | -- remove whitespace 81 | url = string.gsub(url, "%s", "") 82 | -- get fragment 83 | url = string.gsub(url, "#(.*)$", function(f) 84 | parsed.fragment = f 85 | return "" 86 | end) 87 | -- get scheme 88 | url = string.gsub(url, "^([%w][%w%+%-%.]*)%:", 89 | function(s) parsed.scheme = s; return "" end) 90 | 91 | -- get authority 92 | local location 93 | url = string.gsub(url, "^//([^/]*)", function(n) 94 | location = n 95 | return "" 96 | end) 97 | 98 | -- get query stringing 99 | url = string.gsub(url, "%?(.*)", function(q) 100 | parsed.query_string = q 101 | return "" 102 | end) 103 | -- get params 104 | url = string.gsub(url, "%;(.*)", function(p) 105 | parsed.params = p 106 | return "" 107 | end) 108 | -- path is whatever was left 109 | if url ~= "" then parsed.database = string.gsub(url,"^/([^/]*).*","%1") end 110 | if not parsed.database or #parsed.database == 0 then parsed.database = "admin" end 111 | 112 | if not location then return parsed end 113 | 114 | location = string.gsub(location,"^([^@]*)@", 115 | function(u) parsed.userinfo = u; return "" end) 116 | 117 | parsed.hosts = {} 118 | string.gsub(location, "([^,]+)", function(u) 119 | local pr = { host = "localhost", port = 27017 } 120 | u = string.gsub(u, ":([^:]*)$", 121 | function(p) pr.port = p; return "" end) 122 | if u ~= "" then pr.host = u end 123 | table.insert(parsed.hosts, pr) 124 | end) 125 | if #parsed.hosts == 0 then parsed.hosts = {{ host = "localhost", port = 27017 }} end 126 | 127 | parsed.query = {} 128 | if parsed.query_string then 129 | string.gsub(parsed.query_string, "([^&]+)", function(u) 130 | u = string.gsub(u, "([^=]*)=([^=]*)$", 131 | function(k,v) parsed.query[k] = v; return "" end) 132 | end) 133 | end 134 | 135 | local userinfo = parsed.userinfo 136 | if not userinfo then return parsed end 137 | userinfo = string.gsub(userinfo, ":([^:]*)$", 138 | function(p) parsed.password = p; return "" end) 139 | parsed.user = userinfo 140 | return parsed 141 | end 142 | 143 | local function xor_bytestr( a, b ) 144 | local res = "" 145 | for i=1,#a do 146 | res = res .. string.char(bit.bxor(string.byte(a,i,i), string.byte(b, i, i))) 147 | end 148 | return res 149 | end 150 | 151 | local function xor_bytestr( a, b ) 152 | local res = "" 153 | for i=1,#a do 154 | res = res .. string.char(bit.bxor(string.byte(a,i,i), string.byte(b, i, i))) 155 | end 156 | return res 157 | end 158 | 159 | -- A simple implementation of PBKDF2_HMAC_SHA1 160 | local function pbkdf2_hmac_sha1( pbkdf2_key, iterations, salt, len ) 161 | local u1 = hmac_sha1(pbkdf2_key, salt .. "\0\0\0\1") 162 | local ui = u1 163 | for i=1,iterations-1 do 164 | u1 = hmac_sha1(pbkdf2_key, u1) 165 | ui = xor_bytestr(ui, u1) 166 | end 167 | if #ui < len then 168 | for i=1,len-(#ui) do 169 | ui = string.char(0) .. ui 170 | end 171 | end 172 | return ui 173 | end 174 | 175 | -- not full implementation, but oh well 176 | local function saslprep(username) 177 | return string.gsub(string.gsub(username, '=', '=3D'), ',' , '=2C') 178 | end 179 | 180 | local function pass_digest ( username , password ) 181 | return md5(username .. ":mongo:" .. password) 182 | end 183 | 184 | return { 185 | parse_uri = parse_uri; 186 | print_r = print_r; 187 | pbkdf2_hmac_sha1 = pbkdf2_hmac_sha1; 188 | saslprep = saslprep; 189 | pass_digest = pass_digest; 190 | xor_bytestr = xor_bytestr; 191 | generate_oid = generate_oid; 192 | } 193 | -------------------------------------------------------------------------------- /lualib/resty/redis/sentinel.lua: -------------------------------------------------------------------------------- 1 | local ipairs, type = ipairs, type 2 | 3 | local ngx_null = ngx.null 4 | 5 | local tbl_insert = table.insert 6 | local ok, tbl_new = pcall(require, "table.new") 7 | if not ok then 8 | tbl_new = function (narr, nrec) return {} end -- luacheck: ignore 212 9 | end 10 | 11 | 12 | local _M = { 13 | _VERSION = '0.08' 14 | } 15 | 16 | 17 | function _M.get_master(sentinel, master_name) 18 | local res, err = sentinel:sentinel( 19 | "get-master-addr-by-name", 20 | master_name 21 | ) 22 | if res and res ~= ngx_null and res[1] and res[2] then 23 | return { host = res[1], port = res[2] } 24 | else 25 | return nil, err 26 | end 27 | end 28 | 29 | 30 | function _M.get_slaves(sentinel, master_name) 31 | local res, err = sentinel:sentinel("slaves", master_name) 32 | 33 | if res and type(res) == "table" then 34 | local hosts = tbl_new(#res, 0) 35 | for _,slave in ipairs(res) do 36 | local num_recs = #slave 37 | local host = tbl_new(0, num_recs + 1) 38 | for i = 1, num_recs, 2 do 39 | host[slave[i]] = slave[i + 1] 40 | end 41 | 42 | if host["master-link-status"] == "ok" then 43 | host.host = host.ip -- for parity with other functions 44 | tbl_insert(hosts, host) 45 | end 46 | end 47 | if hosts[1] ~= nil then 48 | return hosts 49 | else 50 | return nil, "no slaves available" 51 | end 52 | else 53 | return nil, err 54 | end 55 | end 56 | 57 | 58 | return _M 59 | -------------------------------------------------------------------------------- /lualib/resty/template/html.lua: -------------------------------------------------------------------------------- 1 | local template = require "resty.template" 2 | local setmetatable = setmetatable 3 | local escape = template.escape 4 | local concat = table.concat 5 | local pairs = pairs 6 | local type = type 7 | 8 | local function tag(name, content, attr) 9 | local r, a, content = {}, {}, content or attr 10 | r[#r + 1] = "<" 11 | r[#r + 1] = name 12 | if attr then 13 | for k, v in pairs(attr) do 14 | if type(k) == "number" then 15 | a[#a + 1] = escape(v) 16 | else 17 | a[#a + 1] = k .. '="' .. escape(v) .. '"' 18 | end 19 | end 20 | if #a > 0 then 21 | r[#r + 1] = " " 22 | r[#r + 1] = concat(a, " ") 23 | end 24 | end 25 | if type(content) == "string" then 26 | r[#r + 1] = ">" 27 | r[#r + 1] = escape(content) 28 | r[#r + 1] = "" 31 | else 32 | r[#r + 1] = " />" 33 | end 34 | return concat(r) 35 | end 36 | 37 | local html = { __index = function(_, name) 38 | return function(attr) 39 | if type(attr) == "table" then 40 | return function(content) 41 | return tag(name, content, attr) 42 | end 43 | else 44 | return tag(name, attr) 45 | end 46 | end 47 | end } 48 | 49 | template.html = setmetatable(html, html) 50 | 51 | return template.html 52 | -------------------------------------------------------------------------------- /lualib/resty/template/microbenchmark.lua: -------------------------------------------------------------------------------- 1 | local template = require "resty.template" 2 | 3 | local ok, new_tab = pcall(require, "table.new") 4 | if not ok then 5 | new_tab = function() return {} end 6 | end 7 | 8 | local function run(iterations) 9 | local gc, total, print, parse, compile, iterations, clock, format = collectgarbage, 0, ngx and ngx.say or print, template.parse, template.compile, iterations or 1000, os.clock, string.format 10 | local view = [[ 11 | ]] 16 | 17 | print(format("Running %d iterations in each test", iterations)) 18 | 19 | gc() 20 | gc() 21 | 22 | local x = clock() 23 | for _ = 1, iterations do 24 | parse(view, true) 25 | end 26 | local z = clock() - x 27 | print(format(" Parsing Time: %.6f", z)) 28 | total = total + z 29 | 30 | gc() 31 | gc() 32 | 33 | x = clock() 34 | for _ = 1, iterations do 35 | compile(view, nil, true) 36 | template.cache = {} 37 | end 38 | z = clock() - x 39 | print(format("Compilation Time: %.6f (template)", z)) 40 | total = total + z 41 | 42 | compile(view, nil, true) 43 | 44 | gc() 45 | gc() 46 | 47 | x = clock() 48 | for _ = 1, iterations do 49 | compile(view, 1, true) 50 | end 51 | z = clock() - x 52 | print(format("Compilation Time: %.6f (template, cached)", z)) 53 | total = total + z 54 | 55 | local context = { "Emma", "James", "Nicholas", "Mary" } 56 | 57 | template.cache = {} 58 | 59 | gc() 60 | gc() 61 | 62 | x = clock() 63 | for _ = 1, iterations do 64 | compile(view, 1, true)(context) 65 | template.cache = {} 66 | end 67 | z = clock() - x 68 | print(format(" Execution Time: %.6f (same template)", z)) 69 | total = total + z 70 | 71 | template.cache = {} 72 | compile(view, 1, true) 73 | 74 | gc() 75 | gc() 76 | 77 | x = clock() 78 | for _ = 1, iterations do 79 | compile(view, 1, true)(context) 80 | end 81 | z = clock() - x 82 | print(format(" Execution Time: %.6f (same template, cached)", z)) 83 | total = total + z 84 | 85 | template.cache = {} 86 | 87 | local views = new_tab(iterations, 0) 88 | for i = 1, iterations do 89 | views[i] = "

Iteration " .. i .. "

\n" .. view 90 | end 91 | 92 | gc() 93 | gc() 94 | 95 | x = clock() 96 | for i = 1, iterations do 97 | compile(views[i], i, true)(context) 98 | end 99 | z = clock() - x 100 | print(format(" Execution Time: %.6f (different template)", z)) 101 | total = total + z 102 | 103 | gc() 104 | gc() 105 | 106 | x = clock() 107 | for i = 1, iterations do 108 | compile(views[i], i, true)(context) 109 | end 110 | z = clock() - x 111 | print(format(" Execution Time: %.6f (different template, cached)", z)) 112 | total = total + z 113 | 114 | local contexts = new_tab(iterations, 0) 115 | 116 | for i = 1, iterations do 117 | contexts[i] = { "Emma", "James", "Nicholas", "Mary" } 118 | end 119 | 120 | template.cache = {} 121 | 122 | gc() 123 | gc() 124 | 125 | x = clock() 126 | for i = 1, iterations do 127 | compile(views[i], i, true)(contexts[i]) 128 | end 129 | z = clock() - x 130 | print(format(" Execution Time: %.6f (different template, different context)", z)) 131 | total = total + z 132 | 133 | gc() 134 | gc() 135 | 136 | x = clock() 137 | for i = 1, iterations do 138 | compile(views[i], i, true)(contexts[i]) 139 | end 140 | z = clock() - x 141 | print(format(" Execution Time: %.6f (different template, different context, cached)", z)) 142 | total = total + z 143 | print(format(" Total Time: %.6f", total)) 144 | end 145 | 146 | return { 147 | run = run 148 | } -------------------------------------------------------------------------------- /lualib/resty/uuid.lua: -------------------------------------------------------------------------------- 1 | local ffi = require "ffi" 2 | local ffi_new = ffi.new 3 | local ffi_str = ffi.string 4 | local ffi_load = ffi.load 5 | local ffi_cdef = ffi.cdef 6 | local C = ffi.C 7 | local OSX = ffi.os == "OSX" 8 | local pcall = pcall 9 | local assert = assert 10 | local tonumber = tonumber 11 | local setmetatable = setmetatable 12 | 13 | ffi_cdef[[ 14 | typedef unsigned char uuid_t[16]; 15 | typedef long time_t; 16 | typedef struct timeval { 17 | time_t tv_sec; 18 | time_t tv_usec; 19 | } timeval; 20 | void uuid_generate(uuid_t out); 21 | void uuid_generate_random(uuid_t out); 22 | void uuid_generate_time(uuid_t out); 23 | int uuid_generate_time_safe(uuid_t out); 24 | int uuid_parse(const char *in, uuid_t uu); 25 | void uuid_unparse(const uuid_t uu, char *out); 26 | int uuid_type(const uuid_t uu); 27 | int uuid_variant(const uuid_t uu); 28 | time_t uuid_time(const uuid_t uu, struct timeval *ret_tv); 29 | ]] 30 | 31 | local function L(n) 32 | local ok, lib = pcall(ffi_load, n) 33 | if ok then return lib end 34 | ok, lib = pcall(ffi_load, n .. '.so.1') 35 | assert(ok, lib) 36 | return lib 37 | end 38 | 39 | local lib = OSX and C or L "uuid" 40 | local uid = ffi_new "uuid_t" 41 | local tvl = ffi_new "timeval" 42 | local buf = ffi_new("char[?]", 36) 43 | 44 | local uuid = {} 45 | local mt = {} 46 | 47 | local function unparse(id) 48 | lib.uuid_unparse(id, buf) 49 | return ffi_str(buf, 36) 50 | end 51 | 52 | local function parse(id) 53 | return lib.uuid_parse(id, uid) == 0 and uid or nil 54 | end 55 | 56 | function uuid.generate() 57 | lib.uuid_generate(uid) 58 | return unparse(uid) 59 | end 60 | 61 | function uuid.generate_random() 62 | lib.uuid_generate_random(uid) 63 | return unparse(uid) 64 | end 65 | 66 | function uuid.generate_time() 67 | lib.uuid_generate_time(uid) 68 | return unparse(uid) 69 | end 70 | 71 | function uuid.generate_time_safe() 72 | assert(not OSX, "uuid_generate_time_safe is not supported on OS X.") 73 | local safe = lib.uuid_generate_time_safe(uid) == 0 74 | return unparse(uid), safe 75 | end 76 | 77 | function uuid.type(id) 78 | assert(not OSX, "uuid_type is not supported on OS X.") 79 | local parsed = parse(id) 80 | return parsed and lib.uuid_type(parsed) 81 | end 82 | 83 | function uuid.variant(id) 84 | assert(not OSX, "uuid_variant is not supported on OS X.") 85 | local parsed = parse(id) 86 | return parsed and lib.uuid_variant(parsed) 87 | end 88 | 89 | function uuid.time(id) 90 | local parsed = parse(id) 91 | if parsed then 92 | local secs = lib.uuid_time(parsed, tvl) 93 | return tonumber(secs), tonumber(tvl.tv_usec) 94 | end 95 | end 96 | 97 | function uuid.is_valid(id) 98 | return not not parse(id) 99 | end 100 | 101 | mt.__call = uuid.generate 102 | 103 | return setmetatable(uuid, mt) 104 | -------------------------------------------------------------------------------- /lualib/serpent.lua: -------------------------------------------------------------------------------- 1 | local n, v = "serpent", 0.288 -- (C) 2012-17 Paul Kulchenko; MIT License 2 | local c, d = "Paul Kulchenko", "Lua serializer and pretty printer" 3 | local snum = {[tostring(1/0)]='1/0 --[[math.huge]]',[tostring(-1/0)]='-1/0 --[[-math.huge]]',[tostring(0/0)]='0/0'} 4 | local badtype = {thread = true, userdata = true, cdata = true} 5 | local getmetatable = debug and debug.getmetatable or getmetatable 6 | local keyword, globals, G = {}, {}, (_G or _ENV) 7 | for _,k in ipairs({'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 8 | 'for', 'function', 'goto', 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 9 | 'return', 'then', 'true', 'until', 'while'}) do keyword[k] = true end 10 | for k,v in pairs(G) do globals[v] = k end -- build func to name mapping 11 | for _,g in ipairs({'coroutine', 'debug', 'io', 'math', 'string', 'table', 'os'}) do 12 | for k,v in pairs(type(G[g]) == 'table' and G[g] or {}) do globals[v] = g..'.'..k end end 13 | 14 | local function s(t, opts) 15 | local name, indent, fatal, maxnum = opts.name, opts.indent, opts.fatal, opts.maxnum 16 | local sparse, custom, huge = opts.sparse, opts.custom, not opts.nohuge 17 | local space, maxl = (opts.compact and '' or ' '), (opts.maxlevel or math.huge) 18 | local maxlen = tonumber(opts.maxlength) 19 | local iname, comm = '_'..(name or ''), opts.comment and (tonumber(opts.comment) or math.huge) 20 | local numformat = opts.numformat or "%.17g" 21 | local seen, sref, syms, symn = {}, {'local '..iname..'={}'}, {}, 0 22 | local function gensym(val) return '_'..(tostring(tostring(val)):gsub("[^%w]",""):gsub("(%d%w+)", 23 | -- tostring(val) is needed because __tostring may return a non-string value 24 | function(s) if not syms[s] then symn = symn+1; syms[s] = symn end return tostring(syms[s]) end)) end 25 | local function safestr(s) return type(s) == "number" and tostring(huge and snum[tostring(s)] or numformat:format(s)) 26 | or type(s) ~= "string" and tostring(s) -- escape NEWLINE/010 and EOF/026 27 | or ("%q"):format(s):gsub("\010","n"):gsub("\026","\\026") end 28 | local function comment(s,l) return comm and (l or 0) < comm and ' --[['..select(2, pcall(tostring, s))..']]' or '' end 29 | local function globerr(s,l) return globals[s] and globals[s]..comment(s,l) or not fatal 30 | and safestr(select(2, pcall(tostring, s))) or error("Can't serialize "..tostring(s)) end 31 | local function safename(path, name) -- generates foo.bar, foo[3], or foo['b a r'] 32 | local n = name == nil and '' or name 33 | local plain = type(n) == "string" and n:match("^[%l%u_][%w_]*$") and not keyword[n] 34 | local safe = plain and n or '['..safestr(n)..']' 35 | return (path or '')..(plain and path and '.' or '')..safe, safe end 36 | local alphanumsort = type(opts.sortkeys) == 'function' and opts.sortkeys or function(k, o, n) -- k=keys, o=originaltable, n=padding 37 | local maxn, to = tonumber(n) or 12, {number = 'a', string = 'b'} 38 | local function padnum(d) return ("%0"..tostring(maxn).."d"):format(tonumber(d)) end 39 | table.sort(k, function(a,b) 40 | -- sort numeric keys first: k[key] is not nil for numerical keys 41 | return (k[a] ~= nil and 0 or to[type(a)] or 'z')..(tostring(a):gsub("%d+",padnum)) 42 | < (k[b] ~= nil and 0 or to[type(b)] or 'z')..(tostring(b):gsub("%d+",padnum)) end) end 43 | local function val2str(t, name, indent, insref, path, plainindex, level) 44 | local ttype, level, mt = type(t), (level or 0), getmetatable(t) 45 | local spath, sname = safename(path, name) 46 | local tag = plainindex and 47 | ((type(name) == "number") and '' or name..space..'='..space) or 48 | (name ~= nil and sname..space..'='..space or '') 49 | if seen[t] then -- already seen this element 50 | sref[#sref+1] = spath..space..'='..space..seen[t] 51 | return tag..'nil'..comment('ref', level) end 52 | -- protect from those cases where __tostring may fail 53 | if type(mt) == 'table' then 54 | local to, tr = pcall(function() return mt.__tostring(t) end) 55 | local so, sr = pcall(function() return mt.__serialize(t) end) 56 | if (to or so) then -- knows how to serialize itself 57 | seen[t] = insref or spath 58 | if so then t = sr else t = tostring(t) end 59 | ttype = type(t) 60 | end -- new value falls through to be serialized 61 | end 62 | if ttype == "table" then 63 | if level >= maxl then return tag..'{}'..comment('maxlvl', level) end 64 | seen[t] = insref or spath 65 | if next(t) == nil then return tag..'{}'..comment(t, level) end -- table empty 66 | if maxlen and maxlen < 0 then return tag..'{}'..comment('maxlen', level) end 67 | local maxn, o, out = math.min(#t, maxnum or #t), {}, {} 68 | for key = 1, maxn do o[key] = key end 69 | if not maxnum or #o < maxnum then 70 | local n = #o -- n = n + 1; o[n] is much faster than o[#o+1] on large tables 71 | for key in pairs(t) do if o[key] ~= key then n = n + 1; o[n] = key end end end 72 | if maxnum and #o > maxnum then o[maxnum+1] = nil end 73 | if opts.sortkeys and #o > maxn then alphanumsort(o, t, opts.sortkeys) end 74 | local sparse = sparse and #o > maxn -- disable sparsness if only numeric keys (shorter output) 75 | for n, key in ipairs(o) do 76 | local value, ktype, plainindex = t[key], type(key), n <= maxn and not sparse 77 | if opts.valignore and opts.valignore[value] -- skip ignored values; do nothing 78 | or opts.keyallow and not opts.keyallow[key] 79 | or opts.keyignore and opts.keyignore[key] 80 | or opts.valtypeignore and opts.valtypeignore[type(value)] -- skipping ignored value types 81 | or sparse and value == nil then -- skipping nils; do nothing 82 | elseif ktype == 'table' or ktype == 'function' or badtype[ktype] then 83 | if not seen[key] and not globals[key] then 84 | sref[#sref+1] = 'placeholder' 85 | local sname = safename(iname, gensym(key)) -- iname is table for local variables 86 | sref[#sref] = val2str(key,sname,indent,sname,iname,true) end 87 | sref[#sref+1] = 'placeholder' 88 | local path = seen[t]..'['..tostring(seen[key] or globals[key] or gensym(key))..']' 89 | sref[#sref] = path..space..'='..space..tostring(seen[value] or val2str(value,nil,indent,path)) 90 | else 91 | out[#out+1] = val2str(value,key,indent,insref,seen[t],plainindex,level+1) 92 | if maxlen then 93 | maxlen = maxlen - #out[#out] 94 | if maxlen < 0 then break end 95 | end 96 | end 97 | end 98 | local prefix = string.rep(indent or '', level) 99 | local head = indent and '{\n'..prefix..indent or '{' 100 | local body = table.concat(out, ','..(indent and '\n'..prefix..indent or space)) 101 | local tail = indent and "\n"..prefix..'}' or '}' 102 | return (custom and custom(tag,head,body,tail) or tag..head..body..tail)..comment(t, level) 103 | elseif badtype[ttype] then 104 | seen[t] = insref or spath 105 | return tag..globerr(t, level) 106 | elseif ttype == 'function' then 107 | seen[t] = insref or spath 108 | if opts.nocode then return tag.."function() --[[..skipped..]] end"..comment(t, level) end 109 | local ok, res = pcall(string.dump, t) 110 | local func = ok and "((loadstring or load)("..safestr(res)..",'@serialized'))"..comment(t, level) 111 | return tag..(func or globerr(t, level)) 112 | else return tag..safestr(t) end -- handle all other types 113 | end 114 | local sepr = indent and "\n" or ";"..space 115 | local body = val2str(t, name, indent) -- this call also populates sref 116 | local tail = #sref>1 and table.concat(sref, sepr)..sepr or '' 117 | local warn = opts.comment and #sref>1 and space.."--[[incomplete output with shared/self-references skipped]]" or '' 118 | return not name and body..warn or "do local "..body..sepr..tail.."return "..name..sepr.."end" 119 | end 120 | 121 | local function deserialize(data, opts) 122 | local env = (opts and opts.safe == false) and G 123 | or setmetatable({}, { 124 | __index = function(t,k) return t end, 125 | __call = function(t,...) error("cannot call functions") end 126 | }) 127 | local f, res = (loadstring or load)('return '..data, nil, nil, env) 128 | if not f then f, res = (loadstring or load)(data, nil, nil, env) end 129 | if not f then return f, res end 130 | if setfenv then setfenv(f, env) end 131 | return pcall(f) 132 | end 133 | 134 | local function merge(a, b) if b then for k,v in pairs(b) do a[k] = v end end; return a; end 135 | return { _NAME = n, _COPYRIGHT = c, _DESCRIPTION = d, _VERSION = v, serialize = s, 136 | load = deserialize, 137 | dump = function(a, opts) return s(a, merge({name = '_', compact = true, sparse = true}, opts)) end, 138 | line = function(a, opts) return s(a, merge({sortkeys = true, comment = true}, opts)) end, 139 | block = function(a, opts) return s(a, merge({indent = ' ', sortkeys = true, comment = true}, opts)) end } 140 | -------------------------------------------------------------------------------- /lualib/skiplist.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilibili/oresty/1d0b4be38da37468ac6e08d6562f920b42ce3264/lualib/skiplist.so -------------------------------------------------------------------------------- /lualib/typeof.lua: -------------------------------------------------------------------------------- 1 | local INFINITE_POS = math.huge 2 | local INFINITE_NEG = -INFINITE_POS 3 | local type = type 4 | local floor = math.floor 5 | local rawequal = rawequal 6 | 7 | local function typeof(cmp, arg) 8 | return cmp == type(arg) 9 | end 10 | 11 | local function typeof_nil(...) 12 | return typeof('nil', ...) 13 | end 14 | 15 | local function typeof_bool(...) 16 | return typeof('boolean', ...) 17 | end 18 | 19 | local function typeof_str(...) 20 | return typeof('string', ...) 21 | end 22 | 23 | local function typeof_num(...) 24 | return typeof('number', ...) 25 | end 26 | 27 | local function typeof_fun(...) 28 | return typeof('function', ...) 29 | end 30 | 31 | local function typeof_table(...) 32 | return typeof('table', ...) 33 | end 34 | 35 | local function typeof_thread(...) 36 | return typeof('thread', ...) 37 | end 38 | 39 | local function typeof_userdata(...) 40 | return typeof('userdata', ...) 41 | end 42 | 43 | local function typeof_finite(arg) 44 | return type(arg) == 'number' and (arg < INFINITE_POS and arg > INFINITE_NEG) 45 | end 46 | 47 | local function typeof_unsigned(arg) 48 | return type(arg) == 'number' and (arg < INFINITE_POS and arg >= 0) 49 | end 50 | 51 | local function typeof_int(arg) 52 | return typeof_finite(arg) and rawequal(floor(arg), arg) 53 | end 54 | 55 | local function typeof_int8(arg) 56 | return typeof_int(arg) and arg >= -128 and arg <= 127 57 | end 58 | 59 | local function typeof_int16(arg) 60 | return typeof_int(arg) and arg >= -32768 and arg <= 32767 61 | end 62 | 63 | local function typeof_int32(arg) 64 | return typeof_int(arg) and arg >= -2147483648 and arg <= 2147483647 65 | end 66 | 67 | local function typeof_uint(arg) 68 | return typeof_unsigned(arg) and rawequal(floor(arg), arg) 69 | end 70 | 71 | local function typeof_uint8(arg) 72 | return typeof_uint(arg) and arg <= 255 73 | end 74 | 75 | local function typeof_uint16(arg) 76 | return typeof_uint(arg) and arg <= 65535 77 | end 78 | 79 | local function typeof_uint32(arg) 80 | return typeof_uint(arg) and arg <= 4294967295 81 | end 82 | 83 | local function typeof_nan(arg) 84 | return arg ~= arg 85 | end 86 | 87 | local function typeof_non(arg) 88 | return arg == nil or arg == false or arg == 0 or arg == '' or arg ~= arg 89 | end 90 | 91 | 92 | local _M = { 93 | version = 0.1, 94 | ['nil'] = typeof_nil, 95 | ['boolean'] = typeof_bool, 96 | ['string'] = typeof_str, 97 | ['number'] = typeof_num, 98 | ['function'] = typeof_fun, 99 | ['table'] = typeof_table, 100 | ['thread'] = typeof_thread, 101 | ['userdata'] = typeof_userdata, 102 | ['finite'] = typeof_finite, 103 | ['unsigned'] = typeof_unsigned, 104 | ['int'] = typeof_int, 105 | ['int8'] = typeof_int8, 106 | ['int16'] = typeof_int16, 107 | ['int32'] = typeof_int32, 108 | ['uint'] = typeof_uint, 109 | ['uint8'] = typeof_uint8, 110 | ['uint16'] = typeof_uint16, 111 | ['uint32'] = typeof_uint32, 112 | ['nan'] = typeof_nan, 113 | ['non'] = typeof_non, 114 | -- alias 115 | ['Nil'] = typeof_nil, 116 | ['Function'] = typeof_fun 117 | } 118 | 119 | return _M 120 | -------------------------------------------------------------------------------- /lualib/zlib.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilibili/oresty/1d0b4be38da37468ac6e08d6562f920b42ce3264/lualib/zlib.so -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | user www-data; 2 | worker_processes auto; 3 | worker_rlimit_nofile 131072; 4 | events { 5 | worker_connections 16384; 6 | use epoll; 7 | accept_mutex off; 8 | multi_accept off; 9 | } 10 | 11 | http { 12 | types { 13 | text/html html htm shtml; 14 | text/css css; 15 | application/javascript js; 16 | text/plain txt; 17 | application/json json; 18 | text/xml xml; 19 | image/gif gif; 20 | image/png png; 21 | image/jpeg jpeg jpg; 22 | image/x-ms-bmp bmp; 23 | image/x-icon ico; 24 | image/webp webp; 25 | image/svg+xml svg svgz; 26 | } 27 | default_type application/octet-stream; 28 | server_tokens off; # global 29 | gzip on; 30 | gzip_disable msie6; 31 | gzip_proxied any; 32 | log_not_found off; 33 | sendfile on; # network 34 | keepalive_timeout 5; 35 | tcp_nopush on; 36 | tcp_nodelay off; 37 | large_client_header_buffers 4 256k; # proxy_in 38 | client_header_buffer_size 64k; 39 | client_max_body_size 5g; 40 | client_body_buffer_size 8k; 41 | proxy_buffering on; # proxy_out 42 | proxy_buffer_size 4k; 43 | proxy_buffers 1024 4k; 44 | proxy_busy_buffers_size 3M; 45 | proxy_temp_file_write_size 1M; 46 | proxy_temp_path /dev/shm/proxy_temp; 47 | proxy_max_temp_file_size 1024M; 48 | proxy_connect_timeout 5s; # proxy 49 | proxy_send_timeout 10; 50 | proxy_read_timeout 10; 51 | proxy_intercept_errors on; 52 | map $http_x_request_id $reqid { 53 | default $http_x_request_id; 54 | '' $request_id; 55 | } 56 | log_format curl 57 | 'status-$status ' 58 | 'req_comp-$request_completion ' 59 | 'sent-$body_bytes_sent req_time-$request_time ' 'req_len-$request_length ' 60 | 'remote-$remote_addr:$remote_port realip-$http_x_realip ' 61 | 'content_length-$sent_http_content_length ' 62 | 'content_range-"$sent_http_content_range" ' 63 | 'x_acache_path-$sent_http_x_acache_path ' 64 | 'x_upsig-$sent_http_x_upsig ' 65 | 'etag-$sent_http_etag ' 66 | 'ups_conn_t-$upstream_connect_time ups_resp_t-$upstream_response_time ' 67 | 'msec-$msec ' 68 | 'reqid-$reqid ' 69 | 'curl -X$request_method ' 70 | '-H"Range: $http_range" ' 71 | '-H"Content-Type: $http_content_type" ' 72 | '-H"User-Agent: $http_user_agent" ' 73 | '-H"Referer: $http_referer" ' 74 | '-H"X-Forwarded-For: $http_x_forwarded_for" ' 75 | '-H"X-Acache-Backend: $http_x_acache_backend" ' 76 | '-H"X-Acache-Host: $http_x_acache_host" ' 77 | '-H"X-Upos-Uri: $http_x_upos_uri" ' 78 | '-H"Host: $http_host" ' 79 | '"$scheme://$server_addr:$server_port$request_uri" ' 80 | ; 81 | include ./site.conf; 82 | } 83 | -------------------------------------------------------------------------------- /oresty_1.15.8.2-20200224162915_amd64.deb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bilibili/oresty/1d0b4be38da37468ac6e08d6562f920b42ce3264/oresty_1.15.8.2-20200224162915_amd64.deb -------------------------------------------------------------------------------- /reload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # tjx@20200313 4 | # 5 | 6 | test "$UID" -eq 0 || exec sudo "$0" "$@" 7 | 8 | systemctl reload oresty 9 | 10 | systemctl status oresty 11 | 12 | -------------------------------------------------------------------------------- /restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # tjx@20200313 4 | # 5 | 6 | test "$UID" -eq 0 || exec sudo "$0" "$@" 7 | 8 | systemctl restart oresty 9 | systemctl status oresty 10 | 11 | -------------------------------------------------------------------------------- /site.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Oresty入口配置文件 3 | # 4 | 5 | lua_code_cache on; 6 | lua_package_path './lualib/?.lua;?.lua;;'; 7 | lua_package_cpath './lualib/?.so;;'; 8 | lua_shared_dict prometheus_metrics 10M; 9 | resolver 8.8.8.8; 10 | 11 | # 12 | # /usr/local/{APPNAME}/nginx/log/... 13 | # 14 | access_log logs/access.log curl; 15 | error_log logs/error.log debug; 16 | 17 | init_by_lua_block { 18 | require 'site'.init() 19 | } 20 | init_worker_by_lua_block { 21 | require 'site'.init_worker() 22 | } 23 | 24 | server { 25 | 26 | listen 2233 default; 27 | # listen 80 default; 28 | 29 | set $template_root .; 30 | 31 | location /favicon.ico { 32 | return 404 ''; 33 | } 34 | 35 | location /static/ { 36 | alias /data/app/bvcopen_oresty/static/; 37 | } 38 | 39 | location / { 40 | default_type text/html; 41 | content_by_lua_block { 42 | require 'site'.content() 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /site.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- tjx@20200308 3 | -- 4 | 5 | local lup = require 'lib.lup' 6 | local route = require 'lib.route' 7 | local rt = route.new() 8 | 9 | local _M = {} 10 | 11 | function _M.init() 12 | local process = require 'ngx.process' 13 | 14 | local ok, err = process.enable_privileged_agent() 15 | if not ok then 16 | ngx.log(ngx.ERR, 'enables privileged agent failed error:', err) 17 | end 18 | 19 | ngx.log(ngx.INFO, 'process type: ', process.type()) 20 | 21 | end 22 | 23 | function _M.init_worker() 24 | local process = require 'ngx.process' 25 | -- 26 | -- privileged agent 27 | -- 28 | if process.type() == 'privileged agent' then 29 | ngx.log(ngx.ERR, 'This is the privileged agent with root privilege.') 30 | end 31 | 32 | end 33 | 34 | function _M.content() 35 | 36 | local params = lup._REQUEST() 37 | 38 | local ctrl = params.r or lup.basename(lup.trim(ngx.var.document_uri, '/')) 39 | 40 | if lup.empty(ctrl) then 41 | ctrl = 'home' 42 | end 43 | 44 | local method = params.method or ngx.req.get_method():lower() 45 | 46 | rt:dispatch(nil, ctrl, method, params) 47 | 48 | end 49 | 50 | return _M 51 | -------------------------------------------------------------------------------- /static/a.js: -------------------------------------------------------------------------------- 1 | a.js 2 | -------------------------------------------------------------------------------- /template/home.html: -------------------------------------------------------------------------------- 1 | {% layout = "template/layout.html" %} 2 | 3 |

api

4 | 5 | redis set a = {{time}} 6 |
7 | redis get a 8 | 9 |

模板传值

10 | 11 | message = {{ message }} 12 | 13 |

读取配置

14 | 15 | redis_host = {{ redis_host }} 16 | 17 |

列表

18 | 19 | 24 | 25 |

pre标签

26 | 
27 | total 2.4M
28 | drwxr-xr-x 10 tjx  tjx  4.0K Jun 26 21:46 .
29 | drwxr-xr-x 41 tjx  tjx  4.0K Jun 26 10:07 ..
30 | -rwxr-xr-x  1 tjx  tjx   204 Jun 26 14:30 check-env.sh
31 | lrwxrwxrwx  1 tjx  tjx    11 Apr 18 21:06 config -> config_dev/
32 | drwxr-xr-x  2 tjx  tjx    23 May 14 12:24 config_dev
33 | drwxr-xr-x  2 tjx  tjx    35 Jun 26 19:02 ctrl
34 | -rw-r--r--  1 tjx  tjx  1.8K Jun 26 10:51 gendeb-oresty.sh
35 | drwxr-xr-x  8 tjx  tjx  4.0K Jun 26 19:12 .git
36 | -rw-r--r--  1 tjx  tjx     8 Jun 26 19:12 .gitignore
37 | drwxr-xr-x  2 tjx  tjx  4.0K Jun 26 21:40 lib
38 | drwxr-xr-x  3 tjx  tjx   141 May 10 09:36 lualib
39 | drwxr-xr-x  4 tjx  tjx    28 Apr 18 21:35 nginx
40 | -rw-r--r--  1 root root 3.3K May 14 15:50 oresty
41 | -rw-r--r--  1 tjx  tjx  2.3M Jun 26 14:30 oresty_1.13.6.2-20180626105705_amd64.deb
42 | -rwxr-xr-x  1 tjx  tjx  3.3K May 14 15:49 oresty.template
43 | -rw-r--r--  1 tjx  tjx   501 Jun 26 21:46 README.md
44 | -rw-r--r--  1 tjx  tjx  1.4K Jun 26 21:40 site.conf
45 | drwxr-xr-x  2 tjx  tjx     6 Apr 18 21:17 static
46 | drwxr-xr-x  2 tjx  tjx    40 Jun 26 22:03 template
47 |
48 | -------------------------------------------------------------------------------- /template/layout.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Oresty 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 22 | 23 | 24 |
25 | {*view*} 26 |
27 | 28 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /version: -------------------------------------------------------------------------------- 1 | v46-20200315 2 | --------------------------------------------------------------------------------