├── .gitattributes ├── .gitignore ├── .mergify.yml ├── .travis.yml ├── Makefile ├── README.markdown ├── dist.ini ├── lib └── resty │ └── nsq │ ├── conn.lua │ ├── consumer.lua │ ├── producer.lua │ └── queue.lua └── t ├── consumer.t ├── producer.t └── queue.t /.gitattributes: -------------------------------------------------------------------------------- 1 | *.t linguist-language=Text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.swo 3 | *~ 4 | go 5 | t/servroot/ 6 | reindex 7 | *.t_ 8 | tags 9 | .perl-version 10 | -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | pull_request_rules: 2 | - name: tag no test cases update when update source code 3 | conditions: 4 | - files~=^lib/ 5 | - -files~=^t/ 6 | actions: 7 | label: 8 | add: 9 | - no-test-cases 10 | - name: tag test cases update when update source code 11 | conditions: 12 | - files~=^lib/ 13 | - files~=^t/ 14 | actions: 15 | label: 16 | - remove_all 17 | - name: ask to resolve conflict 18 | conditions: 19 | - conflict 20 | actions: 21 | comment: 22 | message: This pull request is now in conflicts. 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | 4 | os: linux 5 | 6 | language: c 7 | 8 | compiler: 9 | - gcc 10 | - clang 11 | 12 | services: 13 | # - redis-server 14 | 15 | cache: 16 | directories: 17 | - download-cache 18 | 19 | env: 20 | global: 21 | - JOBS=3 22 | - NGX_BUILD_JOBS=$JOBS 23 | - LUAJIT_PREFIX=/opt/luajit21 24 | - LUAJIT_LIB=$LUAJIT_PREFIX/lib 25 | - LUAJIT_INC=$LUAJIT_PREFIX/include/luajit-2.1 26 | - LUA_INCLUDE_DIR=$LUAJIT_INC 27 | - OPENSSL_PREFIX=/opt/ssl 28 | - OPENSSL_LIB=$OPENSSL_PREFIX/lib 29 | - OPENSSL_INC=$OPENSSL_PREFIX/include 30 | - OPENSSL_VER=1.0.2h 31 | - LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH 32 | - TEST_NGINX_SLEEP=0.006 33 | matrix: 34 | - NGINX_VERSION=1.13.6 35 | 36 | install: 37 | - if [ ! -d download-cache ]; then mkdir download-cache; fi 38 | - if [ ! -f download-cache/openssl-$OPENSSL_VER.tar.gz ]; then wget -O download-cache/openssl-$OPENSSL_VER.tar.gz https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz; fi 39 | - sudo apt-get install -qq -y cpanminus axel 40 | - sudo cpanm --notest Test::Nginx > build.log 2>&1 || (cat build.log && exit 1) 41 | - wget http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz 42 | - git clone https://github.com/openresty/openresty.git ../openresty 43 | - git clone https://github.com/openresty/nginx-devel-utils.git 44 | - git clone https://github.com/openresty/lua-cjson.git 45 | - git clone https://github.com/openresty/lua-resty-core.git 46 | - git clone https://github.com/openresty/lua-resty-lrucache.git 47 | - git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module 48 | - git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module 49 | - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx 50 | - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git 51 | 52 | before_script: 53 | - wget https://s3.amazonaws.com/bitly-downloads/nsq/nsq-1.0.0-compat.linux-amd64.go1.8.tar.gz -O nsq.tar.gz > build.log 2>&1 || (cat build.log && exit 1) 54 | - tar -zxvf nsq.tar.gz > build.log 2>&1 || (cat build.log && exit 1) 55 | - cd nsq*/bin && ./nsqd & 56 | 57 | script: 58 | - cd luajit2/ 59 | - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' > build.log 2>&1 || (cat build.log && exit 1) 60 | - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) 61 | - cd ../lua-cjson && make && sudo PATH=$PATH make install && cd .. 62 | - tar zxf download-cache/openssl-$OPENSSL_VER.tar.gz 63 | - cd openssl-$OPENSSL_VER/ 64 | - ./config shared --prefix=$OPENSSL_PREFIX -DPURIFY > build.log 2>&1 || (cat build.log && exit 1) 65 | - make -j$JOBS > build.log 2>&1 || (cat build.log && exit 1) 66 | - sudo make PATH=$PATH install_sw > build.log 2>&1 || (cat build.log && exit 1) 67 | - cd .. 68 | - export PATH=$PWD/work/nginx/sbin:$PWD/nginx-devel-utils:$PATH 69 | - export NGX_BUILD_CC=$CC 70 | - ngx-build $NGINX_VERSION --with-ipv6 --with-http_realip_module --with-http_ssl_module --add-module=../echo-nginx-module --add-module=../lua-nginx-module --with-debug 71 | - nginx -V 72 | - ldd `which nginx`|grep -E 'luajit|ssl|pcre' 73 | - prove -r t 74 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | OPENRESTY_PREFIX=/usr/local/openresty-debug 2 | 3 | PREFIX ?= /usr/local 4 | LUA_INCLUDE_DIR ?= $(PREFIX)/include 5 | LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION) 6 | INSTALL ?= install 7 | 8 | .PHONY: all test install 9 | 10 | all: ; 11 | 12 | install: all 13 | $(INSTALL) -d $(DESTDIR)$(LUA_LIB_DIR)/resty/nsq 14 | $(INSTALL) lib/resty/nsq/*.lua $(DESTDIR)$(LUA_LIB_DIR)/resty/nsq/ 15 | 16 | test: all 17 | PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r t 18 | 19 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | Name 2 | ==== 3 | 4 | lua-resty-nsq - Lua nsq client driver for the ngx_lua based on the cosocket API 5 | 6 | Table of Contents 7 | ================= 8 | 9 | * [Name](#name) 10 | * [Status](#status) 11 | * [Description](#description) 12 | * [Synopsis](#synopsis) 13 | * [Modules](#methods) 14 | * [resty.nsq.producer](#producer) 15 | * [Methods](#methods) 16 | * [new](#new) 17 | * [pub](#pub) 18 | * [nop](#nop) 19 | * [close](#close) 20 | * [resty.nsq.consumer](#consumer) 21 | * [Methods](#methods) 22 | * [new](#new) 23 | * [sub](#sub) 24 | * [rdy](#rdy) 25 | * [fin](#fin) 26 | * [req](#req) 27 | * [close](#close) 28 | * [NSQ Authentication](#nsq-authentication) 29 | * [Installation](#installation) 30 | * [Copyright and License](#copyright-and-license) 31 | * [See Also](#see-also) 32 | 33 | Status 34 | ====== 35 | 36 | [![Build Status](https://www.travis-ci.org/rainingmaster/lua-resty-nsq.svg?branch=master)](https://www.travis-ci.org/rainingmaster/lua-resty-nsq) 37 | 38 | This library is developing. 39 | 40 | Description 41 | =========== 42 | 43 | This Lua library is a NSQ client driver for the ngx_lua nginx module: 44 | 45 | This Lua library takes advantage of ngx_lua's cosocket API, which ensures 46 | 100% nonblocking behavior. 47 | 48 | Synopsis 49 | ======== 50 | 51 | ```lua 52 | lua_package_path "/path/to/lua-resty-nsq/lib/?.lua;;"; 53 | 54 | server { 55 | location /test { 56 | content_by_lua_block { 57 | local config = { 58 | read_timeout = 3, 59 | heartbeat = 1, 60 | } 61 | local producer = require "resty.nsq.producer" 62 | local consumer = require "resty.nsq.consumer" 63 | 64 | local cons = consumer:new() 65 | local prod = producer:new() 66 | 67 | local ok, err = cons:connect("127.0.0.1", 4150, config) 68 | if not ok then 69 | ngx.say("failed to connect: ", err) 70 | return 71 | end 72 | 73 | local ok, err = prod:connect("127.0.0.1", 4150) 74 | if not ok then 75 | ngx.say("failed to connect: ", err) 76 | return 77 | end 78 | 79 | ok, err = prod:pub("new_topic", "hellow world!") 80 | if not ok then 81 | ngx.say("failed to pub: ", err) 82 | return 83 | end 84 | 85 | ok, err = prod:close() 86 | if not ok then 87 | ngx.say("failed to close: ", err) 88 | return 89 | end 90 | 91 | ok, err = cons:sub("new_topic", "new_channel") 92 | if not ok then 93 | ngx.say("failed to sub: ", err) 94 | return 95 | end 96 | 97 | local function read(c) 98 | c:rdy(10) 99 | local ret = cons:message() 100 | ngx.say("sub success: ", require("cjson").encode(ret)) 101 | end 102 | 103 | local co = ngx.thread.spawn(read, cons) -- read message in new thread 104 | ngx.thread.wait(co) 105 | 106 | ok, err = cons:close() 107 | if not ok then 108 | ngx.say("failed to close: ", err) 109 | return 110 | end 111 | } 112 | } 113 | } 114 | ``` 115 | 116 | [Back to TOC](#table-of-contents) 117 | 118 | Modules 119 | ======= 120 | 121 | [Back to TOC](#table-of-contents) 122 | 123 | resty.nsq.producer 124 | -------- 125 | 126 | [Back to TOC](#table-of-contents) 127 | 128 | ### Methods 129 | 130 | [Back to TOC](#table-of-contents) 131 | 132 | #### new 133 | 134 | [Back to TOC](#table-of-contents) 135 | 136 | #### pub 137 | 138 | [Back to TOC](#table-of-contents) 139 | 140 | resty.nsq.consumer 141 | -------- 142 | 143 | [Back to TOC](#table-of-contents) 144 | 145 | ### Methods 146 | 147 | [Back to TOC](#table-of-contents) 148 | 149 | #### new 150 | 151 | [Back to TOC](#table-of-contents) 152 | 153 | Installation 154 | ==== 155 | 156 | export LUA_LIB_DIR=/path/to/lualib && make install 157 | 158 | [Back to TOC](#table-of-contents) 159 | 160 | TODO 161 | ==== 162 | 163 | [Back to TOC](#table-of-contents) 164 | 165 | Copyright and License 166 | ===================== 167 | 168 | This module is licensed under the BSD license. 169 | 170 | Copyright (C) 2018-2018, by rainingmaster. 171 | 172 | All rights reserved. 173 | 174 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 175 | 176 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 177 | 178 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 179 | 180 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 181 | 182 | [Back to TOC](#table-of-contents) 183 | 184 | See Also 185 | ======== 186 | * the ngx_lua module: https://github.com/openresty/lua-nginx-module/#readme 187 | * the nsq wired protocol specification: https://nsq.io/clients/tcp_protocol_spec.html 188 | * [the semaphore in openresty: ngx.sema](https://github.com/openresty/lua-resty-core/blob/master/lib/ngx/semaphore.md) 189 | * [the thread in openresty: ngx.thread](https://github.com/openresty/lua-nginx-module#ngxthreadspawn) 190 | 191 | [Back to TOC](#table-of-contents) 192 | 193 | -------------------------------------------------------------------------------- /dist.ini: -------------------------------------------------------------------------------- 1 | name=lua-resty-nsq 2 | abstract=Lua nsq client driver for the ngx_lua based on the cosocket API 3 | author=rainingmaster 4 | is_original=yes 5 | license=2bsd 6 | lib_dir=lib 7 | doc_dir=. 8 | repo_link=https://github.com/rainingmaster/lua-resty-nsq 9 | main_module=lib/resty/nsq/conn.lua 10 | -------------------------------------------------------------------------------- /lib/resty/nsq/conn.lua: -------------------------------------------------------------------------------- 1 | 2 | 3 | local nsq_queue = require "resty.nsq.queue" 4 | local bit = require "bit" 5 | local cjson = require "cjson" 6 | 7 | local strsub = string.sub 8 | local byte = string.byte 9 | local char = string.char 10 | local concat = table.concat 11 | local encode = cjson.encode 12 | local decode = cjson.decode 13 | local band = bit.band 14 | local bor = bit.bor 15 | local rshift = bit.rshift 16 | local lshift = bit.lshift 17 | local tcp = ngx.socket.tcp 18 | local ngx_log = ngx.log 19 | local ERR = ngx.ERR 20 | local w_exiting = ngx.worker.exiting 21 | local type = type 22 | local pairs = pairs 23 | local unpack = unpack 24 | local setmetatable = setmetatable 25 | local tonumber = tonumber 26 | local tostring = tostring 27 | local rawget = rawget 28 | local pow = math.pow 29 | local floor = math.floor 30 | 31 | 32 | local ok, new_tab = pcall(require, "table.new") 33 | if not ok or type(new_tab) ~= "function" then 34 | new_tab = function (narr, nrec) return {} end 35 | end 36 | 37 | 38 | local _M = new_tab(0, 19) 39 | 40 | _M._VERSION = '0.01' 41 | _M.new_tab = new_tab 42 | 43 | 44 | local frame_type_response = 0 45 | local frame_type_error = 1 46 | local frame_type_message = 2 47 | 48 | local heartbeat = 30 49 | 50 | 51 | local _user_agent = "lua-resty-lua/" .. _M._VERSION 52 | local _clientid = "lua-resty-lua-client" 53 | local _hostname = "lua-resyt-lua-hostname" 54 | 55 | 56 | local mt = { __index = _M } 57 | 58 | 59 | local function _num_2_byte4(n) 60 | return char(band(rshift(n, 24), 0xff), 61 | band(rshift(n, 16), 0xff), 62 | band(rshift(n, 8), 0xff), 63 | band(n, 0xff)) 64 | end 65 | 66 | 67 | local _2pow56 = pow(2, 56) 68 | local _2pow48 = pow(2, 48) 69 | local _2pow40 = pow(2, 40) 70 | local _2pow32 = pow(2, 32) 71 | local _2pow24 = pow(2, 24) 72 | local _10pow9 = pow(10, 9) 73 | 74 | local function _byte4_2_num(b1, b2, b3, b4) 75 | return b1 * _2pow24 + bor(lshift(b2, 16), bor(lshift(b3, 8), b4)) 76 | end 77 | 78 | 79 | -- we use timestamp in second 80 | -- NOTICE: lua only has 52 byte for int 81 | local function _int64_2_timestamp(b1, b2, b3, b4, b5, b6, b7, b8) 82 | return floor((b1 * _2pow56 + b2 * _2pow48 + b3 * _2pow40 + b4 * _2pow32 83 | + _byte4_2_num(b5, b6, b7, b8)) / _10pow9) 84 | end 85 | 86 | 87 | function _M.new(self) 88 | local sock, err = tcp() 89 | if not sock then 90 | return nil, err 91 | end 92 | 93 | return setmetatable({ 94 | sock = sock, 95 | resp_queue = nsq_queue:new(10), 96 | msg_queue = nsq_queue:new(500), 97 | exiting = false, 98 | }, mt) 99 | end 100 | 101 | 102 | local function close(self) 103 | return self.sock:close() 104 | end 105 | _M.close = close 106 | 107 | 108 | function _M.connect(self, addr, port, config) 109 | if config.read_timeout 110 | and config.heartbeat 111 | and config.read_timeout < config.heartbeat 112 | then 113 | error("heartbeat interval should less than read_timeout") 114 | end 115 | 116 | local sock = self.sock 117 | 118 | local ret, err = sock:connect(addr, port) 119 | if not ret then 120 | return nil, err 121 | end 122 | 123 | local connect_timeout = (config.connect_timeout or 10) * 1000 124 | local send_timeout = (config.send_timeout or 10) * 1000 125 | local read_timeout = (config.read_timeout or 35) * 1000 126 | 127 | sock:settimeouts(connect_timeout, send_timeout, read_timeout) 128 | 129 | local bytes, err = sock:send(" V2") 130 | if not bytes then 131 | return nil, err 132 | end 133 | 134 | return ret 135 | end 136 | 137 | 138 | local function _read_reply(self) 139 | if self.fatal then 140 | return nil, nil, "fatal error already happened" 141 | end 142 | 143 | local sock = self.sock 144 | 145 | local data, err = sock:receive(8) 146 | if not data then 147 | if err == "timeout" then 148 | sock:close() 149 | end 150 | 151 | self.fatal = true 152 | return nil, nil, err 153 | end 154 | 155 | local size = _byte4_2_num(byte(data, 1, 4)) - 4 -- length of frame_type 156 | local frame_type = _byte4_2_num(byte(data, 5, 8)) 157 | 158 | data, err = sock:receive(size) 159 | if not data then 160 | if err == "timeout" then 161 | sock:close() 162 | end 163 | 164 | self.fatal = true 165 | return nil, nil, err 166 | end 167 | 168 | -- ngx_log(ERR, "recv: ", data) 169 | 170 | if frame_type == frame_type_response then 171 | if strsub(data, 1, 1) == "{" then 172 | data = decode(data) 173 | end 174 | 175 | return data, frame_type 176 | end 177 | 178 | if frame_type == frame_type_error then 179 | return nil, frame_type, data 180 | end 181 | 182 | if frame_type == frame_type_message then 183 | local timestamp = _int64_2_timestamp(byte(data, 1, 8)) 184 | 185 | return { 186 | timestamp = timestamp, 187 | id = strsub(data, 11, 26), 188 | data = strsub(data, 27), -- 8 + 2 +16 189 | }, frame_type 190 | end 191 | 192 | self.fatal = true 193 | return nil, nil, "unkowned type" 194 | end 195 | _M.read = _read_reply 196 | 197 | 198 | local function _do_cmd(self, params, body, unrecv) 199 | if self.fatal then 200 | return nil, "fatal error already happened" 201 | end 202 | 203 | local sock = self.sock 204 | 205 | local req = { 206 | concat(params, " "), 207 | "\n", 208 | } 209 | 210 | if body then 211 | req[3] = _num_2_byte4(#body) 212 | req[4] = body 213 | end 214 | 215 | local bytes, err = sock:send(req) 216 | if not bytes then 217 | self.fatal = true 218 | 219 | return nil, err 220 | end 221 | 222 | if unrecv then 223 | return true 224 | end 225 | 226 | if self.read_looping then 227 | return self.resp_queue:pop(heartbeat) 228 | end 229 | 230 | local data, typ, err = _read_reply(self) 231 | if typ == frame_type_message then 232 | self.fatal = true 233 | return nil, "return message before read_looping" 234 | end 235 | 236 | return data, err 237 | end 238 | 239 | 240 | function _M.identify(self, config) 241 | local client = { 242 | ["client_id"] = config.clientid or _clientid, 243 | ["hostname"] = config.hostname or _user_agent, 244 | ["user_agent"] = _user_agent, 245 | ["tls_v1"] = false, 246 | ["feature_negotiation"] = true, 247 | ["heartbeat_interval"] = (config.heartbeat or heartbeat) * 1000, 248 | ["sample_rate"] = 0, 249 | -- ["deflate"] = config.deflate, 250 | -- ["deflate_level"] = config.deflate_level, 251 | -- ["snappy"] = config.snappy, 252 | -- ["output_buffer_size"] = config.output_buffer_size, 253 | -- ["output_buffer_timeout"] = -1, 254 | -- ["msg_timeout"] = config.msg_timeout, 255 | } 256 | 257 | return _do_cmd(self, { "IDENTIFY" }, encode(client)) 258 | end 259 | 260 | 261 | function _M.auth(self, secret) 262 | return _do_cmd(self, { "AUTH" }, secret) 263 | end 264 | 265 | 266 | function _M.sub(self, topic, channel) 267 | return _do_cmd(self, { "SUB", topic, channel }) 268 | end 269 | 270 | 271 | function _M.pub(self, topic, body) 272 | return _do_cmd(self, { "PUB", topic }, body) 273 | end 274 | 275 | 276 | function _M.fin(self, messageid) 277 | return _do_cmd(self, { "FIN", messageid }, nil, true) 278 | end 279 | 280 | 281 | function _M.req(self, messageid, timeout) 282 | return _do_cmd(self, { "REQ", messageid, timeout }, nil, true) 283 | end 284 | 285 | 286 | local function nop(self) 287 | return _do_cmd(self, { "NOP" }, nil, true) 288 | end 289 | _M.nop = nop 290 | 291 | 292 | function _M.rdy(self, count) 293 | return _do_cmd(self, { "RDY", count }, nil, true) 294 | end 295 | 296 | 297 | function _M.cls(self, count) 298 | local ret, err = _do_cmd(self, { "CLS" }, nil, true) 299 | if ret then 300 | self._connected = false 301 | end 302 | 303 | return ret, err 304 | end 305 | 306 | 307 | function _M.message(self, timeout) 308 | timeout = tonumber(timeout) or 5 309 | 310 | local ret, err = self.msg_queue:pop(timeout) 311 | if err == "timeout" and self.fatal then 312 | return nil, "fatal error already happened" 313 | end 314 | 315 | return ret, err 316 | end 317 | 318 | 319 | function _M.exit_loop(self) 320 | self.exiting = true 321 | end 322 | 323 | 324 | function _M.read_loop(self, lock) 325 | local sock = self.sock 326 | local resp_queue = self.resp_queue 327 | local msg_queue = self.msg_queue 328 | 329 | self.read_looping = true 330 | 331 | while not self.exiting and not w_exiting() do 332 | local data, typ, err = _read_reply(self) 333 | if typ == frame_type_message then 334 | msg_queue:push(data) 335 | 336 | elseif typ then 337 | if data == "_heartbeat_" then 338 | lock:wait(2 * heartbeat) 339 | nop(self) -- heartbeat 340 | lock:post(1) 341 | 342 | else 343 | resp_queue:push(data) 344 | end 345 | 346 | else 347 | self.read_looping = false 348 | return nil, err 349 | end 350 | end 351 | 352 | self.read_looping = false 353 | return nil, "exiting" 354 | end 355 | 356 | 357 | function _M.check_name(str) 358 | if not str 359 | or type(str) ~= "string" 360 | or #str > 64 361 | or #str < 1 362 | then 363 | return false 364 | end 365 | 366 | return true 367 | end 368 | 369 | 370 | return _M 371 | -------------------------------------------------------------------------------- /lib/resty/nsq/consumer.lua: -------------------------------------------------------------------------------- 1 | 2 | 3 | local nsq_conn = require "resty.nsq.conn" 4 | local semaphore = require "ngx.semaphore" 5 | 6 | local new_tab = nsq_conn.new_tab 7 | local check_name = nsq_conn.check_name 8 | local ngx_spawn = ngx.thread.spawn 9 | local ngx_wait = ngx.thread.wait 10 | local type = type 11 | local pairs = pairs 12 | local unpack = unpack 13 | local setmetatable = setmetatable 14 | local tonumber = tonumber 15 | local tostring = tostring 16 | local rawget = rawget 17 | 18 | 19 | local _M = new_tab(0, 8) 20 | 21 | _M._VERSION = nsq_conn._VERSION 22 | 23 | 24 | local mt = { __index = _M } 25 | 26 | 27 | function _M.new(self) 28 | local c, err = nsq_conn:new() 29 | if not c then 30 | return nil, err 31 | end 32 | 33 | local write_lock, err = semaphore:new() 34 | if not write_lock then 35 | return nil, err 36 | end 37 | 38 | write_lock:post(1) 39 | 40 | return setmetatable({ 41 | write_lock = write_lock, 42 | conn = c, 43 | connected = false, 44 | }, mt) 45 | end 46 | 47 | 48 | function _M.connect(self, addr, port, config) 49 | local conn = self.conn 50 | 51 | local ret, err 52 | repeat 53 | config = config or {} 54 | 55 | ret, err = conn:connect(addr, port, config) 56 | if not ret then 57 | break 58 | end 59 | 60 | ret, err = conn:identify(config) 61 | if not ret then 62 | break 63 | end 64 | 65 | if type(ret) == "table" and ret.auth_required then 66 | ret, err = conn:auth(config.secret or "") 67 | if not ret then 68 | break 69 | end 70 | end 71 | 72 | self.connected = true 73 | 74 | return ret, err 75 | until false 76 | 77 | conn:close() 78 | return nil, err 79 | end 80 | 81 | 82 | local function lock_wrap(self, funcname, ...) 83 | local retry = 1 84 | local ok, err = nil, "timeout" 85 | while err == "timeout" do 86 | ok, err = self.write_lock:wait(5) 87 | retry = retry + 1 88 | if retry > 10 then 89 | return nil, "lock failed" 90 | end 91 | end 92 | 93 | if not ok then 94 | return nil, err 95 | end 96 | 97 | local conn = self.conn 98 | local ret, err = conn[funcname](conn, ...) 99 | 100 | self.write_lock:post(1) 101 | 102 | return ret, err 103 | end 104 | 105 | 106 | function _M.close(self) 107 | local conn = self.conn 108 | 109 | if self.subscribed then 110 | lock_wrap(self, "cls") 111 | conn:exit_loop() 112 | 113 | ngx_wait(self.read_co) 114 | end 115 | 116 | return conn:close() 117 | end 118 | 119 | 120 | function _M.sub(self, topic, channel) 121 | if not check_name(topic) then 122 | error("bad topic") 123 | return 124 | end 125 | 126 | if not check_name(channel) then 127 | error("bad channel") 128 | return 129 | end 130 | 131 | if self.subscribed then 132 | return nil, "is subscribed" 133 | end 134 | 135 | local conn = self.conn 136 | 137 | local ret, err = conn:sub(topic, channel) 138 | if not ret then 139 | return nil, err 140 | end 141 | 142 | self.subscribed = true 143 | self.read_co = ngx_spawn(conn.read_loop, conn, self.write_lock) 144 | 145 | return ret 146 | end 147 | 148 | 149 | function _M.fin(self, id) 150 | if not id then 151 | error("bad params: id") 152 | return 153 | end 154 | 155 | if not self.subscribed then 156 | return nil, "has not subscribed" 157 | end 158 | 159 | return lock_wrap(self, "fin", id) 160 | end 161 | 162 | 163 | function _M.req(self, id, timeout) 164 | if not id then 165 | error("bad params: id") 166 | return 167 | end 168 | 169 | if not tonumber(timeout) then 170 | error("bad params: timeout") 171 | return 172 | end 173 | 174 | if not self.subscribed then 175 | return nil, "has not subscribed" 176 | end 177 | 178 | return lock_wrap(self, "req", id, timeout) 179 | end 180 | 181 | 182 | function _M.rdy(self, count) 183 | if not tonumber(count) then 184 | error("bad params: count") 185 | return 186 | end 187 | 188 | if not self.subscribed then 189 | return nil, "has not subscribed" 190 | end 191 | 192 | return lock_wrap(self, "rdy", count) 193 | end 194 | 195 | 196 | function _M.message(self, timeout) 197 | return self.conn:message(timeout) 198 | end 199 | 200 | 201 | return _M 202 | -------------------------------------------------------------------------------- /lib/resty/nsq/producer.lua: -------------------------------------------------------------------------------- 1 | 2 | 3 | local nsq_conn = require "resty.nsq.conn" 4 | 5 | local new_tab = nsq_conn.new_tab 6 | local check_name = nsq_conn.check_name 7 | local type = type 8 | local setmetatable = setmetatable 9 | local tonumber = tonumber 10 | local tostring = tostring 11 | local rawget = rawget 12 | 13 | 14 | local _M = new_tab(0, 5) 15 | 16 | _M._VERSION = nsq_conn._VERSION 17 | 18 | 19 | local mt = { __index = _M } 20 | 21 | 22 | function _M.new(self) 23 | local c, err = nsq_conn:new() 24 | if not c then 25 | return nil, err 26 | end 27 | 28 | return setmetatable({ conn = c, connected = false }, mt) 29 | end 30 | 31 | 32 | function _M.connect(self, addr, port, config) 33 | local conn = self.conn 34 | 35 | local ret, err 36 | repeat 37 | config = config or {} 38 | 39 | ret, err = conn:connect(addr, port, config) 40 | if not ret then 41 | break 42 | end 43 | 44 | ret, err = conn:identify(config) 45 | if not ret then 46 | break 47 | end 48 | 49 | if type(ret) == "table" and ret.auth_required then 50 | ret, err = conn:auth(config.secret or "") 51 | if not ret then 52 | break 53 | end 54 | end 55 | 56 | self.connected = true 57 | 58 | return ret, err 59 | until false 60 | 61 | conn:close() 62 | 63 | return nil, err 64 | end 65 | 66 | 67 | function _M.close(self) 68 | local conn = self.conn 69 | self.connected = false 70 | 71 | return conn:close() 72 | end 73 | 74 | 75 | function _M.nop(self) 76 | return self.conn:nop() 77 | end 78 | 79 | 80 | function _M.pub(self, topic, message) 81 | if not check_name(topic) then 82 | error("bad topic") 83 | return 84 | end 85 | 86 | if type(message) ~= "string" then 87 | error("bad message") 88 | return 89 | end 90 | 91 | local conn = self.conn 92 | 93 | return conn:pub(topic, message) 94 | end 95 | 96 | 97 | return _M 98 | -------------------------------------------------------------------------------- /lib/resty/nsq/queue.lua: -------------------------------------------------------------------------------- 1 | 2 | 3 | local bit = require "bit" 4 | local semaphore = require "ngx.semaphore" 5 | 6 | local setmetatable = setmetatable 7 | local tonumber = tonumber 8 | local error = error 9 | 10 | 11 | local ok, new_tab = pcall(require, "table.new") 12 | if not ok or type(new_tab) ~= "function" then 13 | new_tab = function (narr, nrec) return {} end 14 | end 15 | 16 | 17 | local _M = new_tab(0, 4) 18 | 19 | _M._VERSION = '0.01' 20 | _M.new_tab = new_tab 21 | 22 | 23 | local MAX_SIZE = 5000 24 | 25 | local mt = { __index = _M } 26 | 27 | 28 | function _M.new(self, size) 29 | size = tonumber(size) or 0 30 | if size < 1 or size > MAX_SIZE then 31 | error("bad params: size") 32 | return 33 | end 34 | 35 | local lock, err = semaphore.new() 36 | if not lock then 37 | error("new semaphore failed: ", err) 38 | return 39 | end 40 | 41 | return setmetatable({ 42 | list = new_tab(size, 0), 43 | size = size, 44 | head = nil, 45 | last = 1, 46 | lock = lock, 47 | }, mt) 48 | end 49 | 50 | 51 | local function pre_pos(size, pos) 52 | pos = pos - 1 53 | 54 | if pos < 1 then 55 | return size 56 | end 57 | 58 | return pos 59 | end 60 | 61 | 62 | local function next_pos(size, pos) 63 | pos = pos + 1 64 | 65 | if pos > size then 66 | return 1 67 | end 68 | 69 | return pos 70 | end 71 | 72 | 73 | function _M.pop(self, timeout) 74 | timeout = tonumber(timeout) 75 | if not timeout then 76 | error("bad params: timeout") 77 | return 78 | end 79 | 80 | local lock = self.lock 81 | local ok, err = lock:wait(timeout) 82 | if not ok then 83 | return nil, err 84 | end 85 | 86 | local head = self.head 87 | local ret = self.list[head] 88 | 89 | head = next_pos(self.size, head) 90 | if head == self.last then -- empty now 91 | head = nil 92 | end 93 | 94 | self.head = head 95 | 96 | return ret 97 | end 98 | 99 | 100 | function _M.push(self, val) 101 | local last = self.last 102 | local head = self.head 103 | 104 | if head == last then 105 | return nil, "fulled" 106 | end 107 | 108 | self.list[last] = val 109 | 110 | if not head then -- not empty now 111 | self.head = self.last 112 | end 113 | 114 | self.last = next_pos(self.size, last) 115 | self.lock:post(1) 116 | 117 | return true 118 | end 119 | 120 | 121 | return _M 122 | -------------------------------------------------------------------------------- /t/consumer.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * (3 * blocks()); 9 | 10 | my $pwd = cwd(); 11 | 12 | our $HttpConfig = qq{ 13 | lua_package_path "$pwd/lib/?.lua;$pwd/lua-resty-core/lib/?.lua;$pwd/lua-resty-lrucache/lib/?.lua;;"; 14 | 15 | init_by_lua_block { 16 | require "resty.core" 17 | } 18 | }; 19 | 20 | $ENV{TEST_NGINX_RESOLVER} = '114.114.114.114'; 21 | $ENV{TEST_NGINX_NSQ_PORT} ||= 4150; 22 | 23 | no_long_string(); 24 | #no_diff(); 25 | 26 | run_tests(); 27 | 28 | __DATA__ 29 | 30 | === TEST 1: sub: sanity 31 | --- http_config eval: $::HttpConfig 32 | --- config 33 | location /t { 34 | content_by_lua_block { 35 | local config = { 36 | read_timeout = 3, 37 | heartbeat = 1, 38 | } 39 | local producer = require "resty.nsq.producer" 40 | local consumer = require "resty.nsq.consumer" 41 | 42 | local cons = consumer:new() 43 | local prod = producer:new() 44 | 45 | local ok, err = cons:connect("127.0.0.1", $TEST_NGINX_NSQ_PORT, config) 46 | if not ok then 47 | ngx.say("failed to connect: ", err) 48 | return 49 | end 50 | 51 | local ok, err = prod:connect("127.0.0.1", $TEST_NGINX_NSQ_PORT) 52 | if not ok then 53 | ngx.say("failed to connect: ", err) 54 | return 55 | end 56 | 57 | ok, err = prod:pub("new_topic", "hellow world!") 58 | if not ok then 59 | ngx.say("failed to pub: ", err) 60 | return 61 | end 62 | 63 | ok, err = prod:close() 64 | if not ok then 65 | ngx.say("failed to close: ", err) 66 | return 67 | end 68 | 69 | ok, err = cons:sub("new_topic", "new_channel") 70 | if not ok then 71 | ngx.say("failed to sub: ", err) 72 | return 73 | end 74 | 75 | local function read(c) 76 | c:rdy(1) 77 | local ret = cons:message() 78 | c:fin(ret.id) 79 | ngx.say("sub success: ", require("cjson").encode(ret)) 80 | end 81 | 82 | local co = ngx.thread.spawn(read, cons) 83 | ngx.thread.wait(co) 84 | 85 | ok, err = cons:close() 86 | if not ok then 87 | ngx.say("failed to close: ", err) 88 | return 89 | end 90 | } 91 | } 92 | --- request 93 | GET /t 94 | --- response_body_like 95 | sub success: \{"timestamp":(.*),"data":"hellow world!","id":"(.*)"\} 96 | --- no_error_log 97 | [error] 98 | -------------------------------------------------------------------------------- /t/producer.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * (3 * blocks()); 9 | 10 | my $pwd = cwd(); 11 | 12 | our $HttpConfig = qq{ 13 | lua_package_path "$pwd/lib/?.lua;$pwd/lua-resty-core/lib/?.lua;$pwd/lua-resty-lrucache/lib/?.lua;;"; 14 | 15 | init_by_lua_block { 16 | require "resty.core" 17 | } 18 | }; 19 | 20 | $ENV{TEST_NGINX_RESOLVER} = '114.114.114.114'; 21 | $ENV{TEST_NGINX_NSQ_PORT} ||= 4150; 22 | 23 | no_long_string(); 24 | #no_diff(); 25 | 26 | run_tests(); 27 | 28 | __DATA__ 29 | 30 | === TEST 1: pub: sanity 31 | --- http_config eval: $::HttpConfig 32 | --- config 33 | location /t { 34 | content_by_lua_block { 35 | local producer = require "resty.nsq.producer" 36 | local prod = producer:new() 37 | 38 | local ok, err = prod:connect("127.0.0.1", $TEST_NGINX_NSQ_PORT) 39 | if not ok then 40 | ngx.say("failed to connect: ", err) 41 | return 42 | end 43 | 44 | ok, err = prod:pub("new_topic", "hellow world!") 45 | if not ok then 46 | ngx.say("failed to pub: ", err) 47 | return 48 | end 49 | 50 | ok, err = prod:close() 51 | if not ok then 52 | ngx.say("failed to close: ", err) 53 | return 54 | end 55 | 56 | ngx.say("pub success!") 57 | } 58 | } 59 | --- request 60 | GET /t 61 | --- response_body 62 | pub success! 63 | --- no_error_log 64 | [error] 65 | 66 | 67 | 68 | === TEST 2: pub: bad topic 69 | --- http_config eval: $::HttpConfig 70 | --- config 71 | location /t { 72 | content_by_lua_block { 73 | local producer = require "resty.nsq.producer" 74 | local prod = producer:new() 75 | 76 | ok, err = pcall(prod.pub, prod, "", "hellow world!") 77 | if not ok then 78 | ngx.say("failed to pub: ", err) 79 | return 80 | end 81 | 82 | ngx.say("pub success!") 83 | } 84 | } 85 | --- request 86 | GET /t 87 | --- response_body_like 88 | bad topic 89 | --- no_error_log 90 | [error] 91 | 92 | 93 | 94 | === TEST 3: pub: bad message 95 | --- http_config eval: $::HttpConfig 96 | --- config 97 | location /t { 98 | content_by_lua_block{ 99 | local producer = require "resty.nsq.producer" 100 | local prod = producer:new() 101 | 102 | ok, err = pcall(prod.pub, prod, "topic") 103 | if not ok then 104 | ngx.say("failed to pub: ", err) 105 | return 106 | end 107 | 108 | ngx.say("pub success!") 109 | } 110 | } 111 | --- request 112 | GET /t 113 | --- response_body_like 114 | bad message 115 | --- no_error_log 116 | [error] 117 | -------------------------------------------------------------------------------- /t/queue.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * (3 * blocks()); 9 | 10 | my $pwd = cwd(); 11 | 12 | our $HttpConfig = qq{ 13 | lua_package_path "$pwd/lib/?.lua;$pwd/lua-resty-core/lib/?.lua;$pwd/lua-resty-lrucache/lib/?.lua;;"; 14 | 15 | init_by_lua_block { 16 | require "resty.core" 17 | } 18 | }; 19 | 20 | 21 | no_long_string(); 22 | #no_diff(); 23 | 24 | run_tests(); 25 | 26 | __DATA__ 27 | 28 | === TEST 1: sanity 29 | --- http_config eval: $::HttpConfig 30 | --- config 31 | location /t { 32 | content_by_lua_block { 33 | local nsq_queue = require "resty.nsq.queue" 34 | local queue = nsq_queue:new(10) 35 | 36 | local function test_pop(q) 37 | local ret = q:pop(0.1) 38 | if ret then 39 | ngx.say(ret) 40 | end 41 | end 42 | 43 | local co = ngx.thread.spawn(test_pop, queue) 44 | 45 | ngx.sleep(0.01) 46 | queue:push("hello world") 47 | 48 | ngx.thread.wait(co) 49 | } 50 | } 51 | --- request 52 | GET /t 53 | --- response_body 54 | hello world 55 | --- no_error_log 56 | [error] 57 | 58 | 59 | 60 | === TEST 2: empty and timeout 61 | --- http_config eval: $::HttpConfig 62 | --- config 63 | location /t { 64 | content_by_lua_block { 65 | local nsq_queue = require "resty.nsq.queue" 66 | local queue = nsq_queue:new(10) 67 | 68 | local ret, err = queue:pop(0.5) 69 | ngx.say(ret, ":", err) 70 | } 71 | } 72 | --- request 73 | GET /t 74 | --- response_body 75 | nil:timeout 76 | --- no_error_log 77 | [error] 78 | 79 | 80 | 81 | === TEST 3: full 82 | --- http_config eval: $::HttpConfig 83 | --- config 84 | location /t { 85 | content_by_lua_block { 86 | local nsq_queue = require "resty.nsq.queue" 87 | local queue = nsq_queue:new(10) 88 | 89 | for i = 1, 11 do 90 | local ret, err = queue:push("hello world" .. i) 91 | if not ret then 92 | ngx.say(ret, ":", err, ", in ", i) 93 | end 94 | end 95 | } 96 | } 97 | --- request 98 | GET /t 99 | --- response_body 100 | nil:fulled, in 11 101 | --- no_error_log 102 | [error] 103 | 104 | 105 | 106 | === TEST 4: read and write 107 | --- http_config eval: $::HttpConfig 108 | --- config 109 | location /t { 110 | content_by_lua_block { 111 | local nsq_queue = require "resty.nsq.queue" 112 | local queue = nsq_queue:new(10) 113 | 114 | local function test_pop(q) 115 | for i = 1, 30 do 116 | local ret, err = q:pop(0.01) 117 | if ret then 118 | ngx.say("get: ", ret) 119 | -- ngx.log(ngx.ERR, "get: ", ret) 120 | end 121 | end 122 | end 123 | 124 | local co = ngx.thread.spawn(test_pop, queue) 125 | 126 | for i = 1, 30 do 127 | local ret, err = queue:push("hello world: " .. i) 128 | if not ret then 129 | ngx.say(ret, ":", err, ", in ", i) 130 | -- ngx.log(ngx.ERR, ret, ":", err, ", in ", i) 131 | end 132 | 133 | if i == 5 then 134 | ngx.sleep(0) 135 | elseif i == 11 then 136 | ngx.sleep(0) 137 | elseif i == 25 then 138 | ngx.sleep(0) 139 | end 140 | end 141 | 142 | ngx.thread.wait(co) 143 | } 144 | } 145 | --- request 146 | GET /t 147 | --- response_body 148 | get: hello world: 1 149 | get: hello world: 2 150 | get: hello world: 3 151 | get: hello world: 4 152 | get: hello world: 5 153 | get: hello world: 6 154 | get: hello world: 7 155 | get: hello world: 8 156 | get: hello world: 9 157 | get: hello world: 10 158 | get: hello world: 11 159 | nil:fulled, in 22 160 | nil:fulled, in 23 161 | nil:fulled, in 24 162 | nil:fulled, in 25 163 | get: hello world: 12 164 | get: hello world: 13 165 | get: hello world: 14 166 | get: hello world: 15 167 | get: hello world: 16 168 | get: hello world: 17 169 | get: hello world: 18 170 | get: hello world: 19 171 | get: hello world: 20 172 | get: hello world: 21 173 | get: hello world: 26 174 | get: hello world: 27 175 | get: hello world: 28 176 | get: hello world: 29 177 | get: hello world: 30 178 | --- no_error_log 179 | [error] 180 | --------------------------------------------------------------------------------