├── .gitattributes ├── src ├── ngx_http_redis2_handler.h ├── ngx_http_redis2_reply.h ├── ngx_http_redis2_util.h ├── ngx_http_redis2_module.h ├── multi_bulk_reply.rl ├── ddebug.h ├── common.rl ├── ngx_http_redis2_reply.rl ├── ngx_http_redis2_util.c ├── ngx_http_redis2_handler.c ├── ngx_http_redis2_module.c └── ngx_http_redis2_reply.c ├── .gitignore ├── util ├── build.sh └── fix-clang-warnings ├── misc └── serv.erl ├── config ├── Changes ├── t ├── if.t ├── eval.t ├── pipeline.t ├── bugs.t └── sanity.t ├── .travis.yml ├── valgrind.suppress └── README.markdown /.gitattributes: -------------------------------------------------------------------------------- 1 | *.t linguist-language=Text 2 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_handler.h: -------------------------------------------------------------------------------- 1 | #ifndef NGX_HTTP_REDIS2_HANDLER_H 2 | #define NGX_HTTP_REDIS2_HANDLER_H 3 | 4 | 5 | #include "ngx_http_redis2_module.h" 6 | 7 | ngx_int_t ngx_http_redis2_handler(ngx_http_request_t *r); 8 | 9 | 10 | #endif /* NGX_HTTP_REDIS2_HANDLER_H */ 11 | 12 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_reply.h: -------------------------------------------------------------------------------- 1 | #ifndef NGX_HTTP_REDIS2_REPLY_H 2 | #define NGX_HTTP_REDIS2_REPLY_H 3 | 4 | 5 | #include "ngx_http_redis2_module.h" 6 | 7 | 8 | ngx_int_t ngx_http_redis2_process_reply( 9 | ngx_http_redis2_ctx_t *ctx, ssize_t bytes); 10 | 11 | 12 | #endif /* NGX_HTTP_REDIS2_REPLY_H */ 13 | 14 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_util.h: -------------------------------------------------------------------------------- 1 | #ifndef NGX_HTTP_REDIS2_UTIL_H 2 | #define NGX_HTTP_REDIS2_UTIL_H 3 | 4 | 5 | #include "ngx_http_redis2_module.h" 6 | 7 | 8 | #ifndef ngx_str_set 9 | #define ngx_str_set(str, text) \ 10 | (str)->len = sizeof(text) - 1; (str)->data = (u_char *) text 11 | #endif 12 | 13 | 14 | char *ngx_http_redis2_set_complex_value_slot(ngx_conf_t *cf, 15 | ngx_command_t *cmd, void *conf); 16 | ngx_http_upstream_srv_conf_t *ngx_http_redis2_upstream_add( 17 | ngx_http_request_t *r, ngx_url_t *url); 18 | ngx_int_t ngx_http_redis2_build_query(ngx_http_request_t *r, 19 | ngx_array_t *queries, ngx_buf_t **b); 20 | 21 | #endif /* NGX_HTTP_REDIS2_UTIL_H */ 22 | 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | analyze 2 | *.mobi 3 | genmobi.sh 4 | .libs 5 | *.swp 6 | *.slo 7 | *.la 8 | *.swo 9 | *.lo 10 | *~ 11 | *.o 12 | print.txt 13 | .rsync 14 | *.tar.gz 15 | dist 16 | build[789] 17 | build 18 | tags 19 | update-readme 20 | *.tmp 21 | test/Makefile 22 | test/blib 23 | test.sh 24 | t.sh 25 | t/t.sh 26 | test/t/servroot/ 27 | releng 28 | reset 29 | *.t_ 30 | src/handler.h 31 | src/util.c 32 | src/module.h 33 | src/module.c 34 | src/drizzle.c 35 | src/processor.h 36 | src/handler.c 37 | src/util.h 38 | src/drizzle.h 39 | src/processor.c 40 | src/output.c 41 | src/output.h 42 | libdrizzle 43 | ctags 44 | src/stream.h 45 | nginx 46 | keepalive 47 | reindex 48 | src/module.h 49 | src/module.c 50 | util/bench 51 | src/cipher.c 52 | src/cipher.h 53 | pack.sh 54 | *.html 55 | t/servroot/ 56 | src/reply.c 57 | src/reply.h 58 | src/reply.rl 59 | all 60 | go 61 | buildroot/ 62 | build1[0-9] 63 | *.plist 64 | Makefile 65 | addr2line 66 | -------------------------------------------------------------------------------- /util/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this file is mostly meant to be used by the author himself. 4 | 5 | ragel -I src -T1 src/ngx_http_redis2_reply.rl || exit 1 6 | ./util/fix-clang-warnings 7 | 8 | root=`pwd` 9 | version=$1 10 | home=~ 11 | force=$2 12 | 13 | #--with-cc-opt="-O3" \ 14 | #--with-cc-opt="-fast" \ 15 | #--with-cc=gcc46 \ 16 | 17 | ngx-build $force $version \ 18 | --with-ld-opt="-L$PCRE_LIB -Wl,-rpath,$PCRE_LIB:$LIBDRIZZLE_LIB:/usr/local/lib" \ 19 | --with-cc-opt="-O3 -funsigned-char" \ 20 | --with-http_addition_module \ 21 | --add-module=$root $opts \ 22 | --add-module=$root/../eval-nginx-module \ 23 | --add-module=$root/../echo-nginx-module \ 24 | --add-module=$root/../ndk-nginx-module \ 25 | --add-module=$root/../set-misc-nginx-module \ 26 | --add-module=$root/../lua-nginx-module \ 27 | --with-debug 28 | #--add-module=$root/../eval-nginx-module \ 29 | #--add-module=$home/work/nginx/nginx_upstream_hash-0.3 \ 30 | #--without-http_ssi_module # we cannot disable ssi because echo_location_async depends on it (i dunno why?!) 31 | 32 | -------------------------------------------------------------------------------- /util/fix-clang-warnings: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | use File::Temp 'tempfile'; 6 | 7 | my $infile = "src/ngx_http_redis2_reply.c"; 8 | my ($out, $outfile) = tempfile(); 9 | open my $in, $infile 10 | or die "Cannot open $infile for reading: $!\n"; 11 | 12 | my $hits = 0; 13 | while (<$in>) { 14 | if (/ \b reply_ (?: en_main | first_final ) \b /x) 15 | { 16 | #warn "HIT!"; 17 | $hits++; 18 | next; 19 | } 20 | print $out $_; 21 | } 22 | 23 | close $in; 24 | close $out; 25 | 26 | if ($hits) { 27 | my $cmd = "cp $outfile $infile"; 28 | system($cmd) == 0 29 | or die "Cannot run command \"$cmd\": $!"; 30 | } 31 | #die; 32 | 33 | __END__ 34 | 35 | This script is to fix the following clang warnings when using Ragel 6.8/6.9/etc: 36 | 37 | src/ngx_http_redis2_reply.c:19:18: error: unused variable 'reply_first_final' [-Werror,-Wunused-const-variable] 38 | static const int reply_first_final = 52; 39 | ^ 40 | src/ngx_http_redis2_reply.c:22:18: error: unused variable 'reply_en_main' [-Werror,-Wunused-const-variable] 41 | static const int reply_en_main = 1; 42 | ^ 43 | 2 errors generated. 44 | -------------------------------------------------------------------------------- /misc/serv.erl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | 3 | -define(PORT, 4891). 4 | -define(TIMEOUT, 3000). 5 | 6 | main(_) -> io:format("listening to port ~w~n", [?PORT]), server(). 7 | 8 | server() -> 9 | case gen_tcp:listen(?PORT, [binary, {active, false}, {reuseaddr, true}, {nodelay, true}]) of 10 | {ok, LSock} -> wait_connect(LSock); 11 | E -> io:format("error occured: ~w~n", [E]) 12 | end. 13 | 14 | wait_connect(LSock) -> 15 | % io:format("trying to accept..."), 16 | case gen_tcp:accept(LSock) of 17 | {ok, Sock} -> 18 | Pid = spawn( 19 | fun () -> 20 | receive start -> ok end, 21 | get_request(Sock) 22 | end 23 | ), 24 | gen_tcp:controlling_process(Sock, Pid), 25 | Pid ! start; 26 | {error, Error} -> 27 | io:format("failed to accept: ~w~n", [Error]); 28 | E -> io:format("error: ~w~n", [E]) 29 | end, 30 | wait_connect(LSock). 31 | 32 | get_request(Sock) -> 33 | % io:format("get request~n"), 34 | send_response(Sock). 35 | 36 | send_response(Sock) -> 37 | gen_tcp:send(Sock, "$1a\r\n"), 38 | gen_tcp:close(Sock). 39 | 40 | -------------------------------------------------------------------------------- /config: -------------------------------------------------------------------------------- 1 | ngx_addon_name=ngx_http_redis2_module 2 | 3 | REDIS2_SRCS=" \ 4 | $ngx_addon_dir/src/ngx_http_redis2_module.c \ 5 | $ngx_addon_dir/src/ngx_http_redis2_handler.c \ 6 | $ngx_addon_dir/src/ngx_http_redis2_reply.c \ 7 | $ngx_addon_dir/src/ngx_http_redis2_util.c \ 8 | " 9 | 10 | REDIS2_DEPS=" \ 11 | $ngx_addon_dir/src/ddebug.h \ 12 | $ngx_addon_dir/src/ngx_http_redis2_module.h \ 13 | $ngx_addon_dir/src/ngx_http_redis2_handler.h \ 14 | $ngx_addon_dir/src/ngx_http_redis2_reply.h \ 15 | $ngx_addon_dir/src/ngx_http_redis2_util.h \ 16 | " 17 | 18 | if [ -n "$ngx_module_link" ]; then 19 | ngx_module_type=HTTP 20 | ngx_module_name=$ngx_addon_name 21 | ngx_module_srcs="$REDIS2_SRCS" 22 | ngx_module_deps="$REDIS2_DEPS" 23 | 24 | . auto/module 25 | else 26 | HTTP_MODULES="$HTTP_MODULES $ngx_addon_name" 27 | NGX_ADDON_SRCS="$NGX_ADDON_SRCS $REDIS2_SRCS" 28 | NGX_ADDON_DEPS="$NGX_ADDON_DEPS $REDIS2_DEPS" 29 | fi 30 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_module.h: -------------------------------------------------------------------------------- 1 | #ifndef NGX_HTTP_REDIS2_MODULE_H 2 | #define NGX_HTTP_REDIS2_MODULE_H 3 | 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | extern ngx_module_t ngx_http_redis2_module; 11 | 12 | typedef struct { 13 | ngx_http_upstream_conf_t upstream; 14 | ngx_str_t literal_query; /* for redis2_literal_raw_query */ 15 | ngx_http_complex_value_t *complex_query; /* for redis2_raw_query */ 16 | ngx_http_complex_value_t *complex_query_count; /* for redis2_raw_query */ 17 | ngx_http_complex_value_t *complex_target; /* for redis2_pass */ 18 | ngx_array_t *queries; /* for redis2_query */ 19 | 20 | } ngx_http_redis2_loc_conf_t; 21 | 22 | 23 | typedef struct ngx_http_redis2_ctx_s ngx_http_redis2_ctx_t; 24 | 25 | typedef ngx_int_t (*ngx_http_redis2_filter_handler_ptr) 26 | (ngx_http_redis2_ctx_t *ctx, ssize_t bytes); 27 | 28 | 29 | struct ngx_http_redis2_ctx_s { 30 | ngx_int_t query_count; 31 | ngx_http_request_t *request; 32 | int state; 33 | size_t chunk_size; 34 | size_t chunk_bytes_read; 35 | size_t chunks_read; 36 | size_t chunk_count; 37 | 38 | ngx_http_redis2_filter_handler_ptr filter; 39 | }; 40 | 41 | 42 | #endif /* NGX_HTTP_REDIS2_MODULE_H */ 43 | 44 | -------------------------------------------------------------------------------- /Changes: -------------------------------------------------------------------------------- 1 | v0.07 - 20 July, 2011 2 | * now we have implemented redis command pipelining by using multiple redis2_query directives in a single location. also updated the document to reflect this change. 3 | * implemented the new redis2_raw_queries directive which supports multiple pipelined redis commands in a single TCP query. 4 | * fixed github issue #5: CRLF in chunk data confused the response parser when packet segmentation happens. thanks dlogar. 5 | * fixed an issue in the multi-bulk reply parser that only chunks were allowed as elements. now single line replies are all allowed, but recursive multi-bulk replies are still not allowed. 6 | * fixed an issue regarding defining global variables in C header files: we should have defined the global ngx_http_redis2_module in a single compilation unit. thanks @姜大炮. 7 | * documented the redis2_next_upstream directive as per Lance. 8 | * documented the limited redis pub/sub feature support. 9 | * documented the various timeout config directives. thanks Mike Ferrier. 10 | * now we only issue a warning when there is left-over bytes in the redis response stream. 11 | * added a memory allocation failure check. (calio) 12 | * fixed a typo in the source; it does not affect runtime results though. thanks calio for catching it. 13 | * fixed one spot of unused-but-set-variable warning issued by gcc 4.6. 14 | * now we allow empty parameters in the redis2_query directive. 15 | * added a performance tuning section as per HowToMeetLadies. 16 | * fixed a typo in the source code reported by Rares Mirica. 17 | 18 | -------------------------------------------------------------------------------- /src/multi_bulk_reply.rl: -------------------------------------------------------------------------------- 1 | %%{ 2 | machine multi_bulk_reply; 3 | 4 | include common "common.rl"; 5 | 6 | action test_chunk_count { 7 | #if 0 8 | fprintf(stderr, "test chunk count: %d < %d\n", 9 | (int) ctx->chunks_read, (int) ctx->chunk_count), 10 | #endif 11 | ctx->chunks_read < ctx->chunk_count 12 | } 13 | 14 | action start_reading_chunk { 15 | dd("start reading bulk"); 16 | ctx->chunks_read = 0; 17 | } 18 | 19 | action start_reading_count { 20 | dd("start reading bulk count"); 21 | ctx->chunk_count = 0; 22 | } 23 | 24 | action read_count { 25 | ctx->chunk_count *= 10; 26 | ctx->chunk_count += *p - '0'; 27 | dd("chunk count: %d", (int) ctx->chunk_count); 28 | } 29 | 30 | action multi_bulk_finalize { 31 | dd("finalize multi bulks"); 32 | 33 | if (ctx->chunks_read == ctx->chunk_count) { 34 | dd("done multi bunlk reading!"); 35 | done = 1; 36 | } 37 | } 38 | 39 | reply = single_line_reply @read_chunk 40 | | chunk 41 | ; 42 | 43 | protected_chunk = reply when test_chunk_count 44 | ; 45 | 46 | chunk_count = ([1-9] digit*) >start_reading_count $read_count 47 | ; 48 | 49 | multi_bulk_reply = "*" "-1" CRLF @multi_bulk_finalize 50 | | "*" "0"+ CRLF @multi_bulk_finalize 51 | | "*" chunk_count CRLF @start_reading_chunk 52 | protected_chunk+ 53 | @multi_bulk_finalize 54 | ; 55 | 56 | }%% 57 | 58 | -------------------------------------------------------------------------------- /src/ddebug.h: -------------------------------------------------------------------------------- 1 | #ifndef DDEBUG_H 2 | #define DDEBUG_H 3 | 4 | #include 5 | #include 6 | 7 | #if defined(DDEBUG) && (DDEBUG) 8 | 9 | # if (NGX_HAVE_VARIADIC_MACROS) 10 | 11 | # define dd(...) fprintf(stderr, "redis2 *** "); \ 12 | fprintf(stderr, __VA_ARGS__); \ 13 | fprintf(stderr, " at %s line %d.\n", __FILE__, __LINE__) 14 | 15 | # else 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | static ngx_inline void 23 | dd(const char* fmt, ...) { 24 | } 25 | 26 | # endif 27 | 28 | #else 29 | 30 | # if (NGX_HAVE_VARIADIC_MACROS) 31 | 32 | # define dd(...) 33 | 34 | # else 35 | 36 | #include 37 | 38 | static ngx_inline void 39 | dd(const char* fmt, ...) { 40 | } 41 | 42 | # endif 43 | 44 | #endif 45 | 46 | #if defined(DDEBUG) && (DDEBUG) 47 | 48 | #define dd_check_read_event_handler(r) \ 49 | dd("r->read_event_handler = %s", \ 50 | r->read_event_handler == ngx_http_block_reading ? \ 51 | "ngx_http_block_reading" : \ 52 | r->read_event_handler == ngx_http_test_reading ? \ 53 | "ngx_http_test_reading" : \ 54 | r->read_event_handler == ngx_http_request_empty_handler ? \ 55 | "ngx_http_request_empty_handler" : "UNKNOWN") 56 | 57 | #define dd_check_write_event_handler(r) \ 58 | dd("r->write_event_handler = %s", \ 59 | r->write_event_handler == ngx_http_handler ? \ 60 | "ngx_http_handler" : \ 61 | r->write_event_handler == ngx_http_core_run_phases ? \ 62 | "ngx_http_core_run_phases" : \ 63 | r->write_event_handler == ngx_http_request_empty_handler ? \ 64 | "ngx_http_request_empty_handler" : "UNKNOWN") 65 | 66 | #else 67 | 68 | #define dd_check_read_event_handler(r) 69 | #define dd_check_write_event_handler(r) 70 | 71 | #endif 72 | 73 | #endif /* DDEBUG_H */ 74 | 75 | -------------------------------------------------------------------------------- /t/if.t: -------------------------------------------------------------------------------- 1 | # vi:ft= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | #repeat_each(2); 7 | 8 | plan tests => repeat_each() * (3 * blocks()); 9 | 10 | $ENV{TEST_NGINX_REDIS_PORT} ||= 6379; 11 | 12 | #master_on; 13 | #worker_connections 1024; 14 | 15 | #no_diff; 16 | no_long_string; 17 | 18 | #log_level 'warn'; 19 | 20 | run_tests(); 21 | 22 | __DATA__ 23 | 24 | === TEST 1: location if (query) 25 | --- config 26 | location /redis { 27 | set $key 'default'; 28 | if ($uri ~ "(special.*)") { 29 | set $key $1; 30 | } 31 | redis2_query lrange $key 0 -1; 32 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 33 | } 34 | --- request 35 | GET /redis/special 36 | --- response_body eval 37 | "*0\r\n" 38 | --- no_error_log 39 | [error] 40 | 41 | 42 | 43 | === TEST 2: location if (literal query) 44 | --- config 45 | location /redis { 46 | if ($uri ~ "(special.*)") { 47 | } 48 | redis2_literal_raw_query 'flushall\r\n'; 49 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 50 | } 51 | --- request 52 | GET /redis/special 53 | --- response_body eval 54 | "+OK\r\n" 55 | --- no_error_log 56 | [error] 57 | 58 | 59 | 60 | === TEST 3: location if (raw queries) 61 | --- config 62 | location /redis { 63 | if ($uri ~ "(special.*)") { 64 | set $n 2; 65 | } 66 | redis2_raw_queries $n 'flushall\r\nget foo\r\n'; 67 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 68 | } 69 | --- request 70 | GET /redis/special 71 | --- response_body eval 72 | "+OK\r\n\$-1\r\n" 73 | --- no_error_log 74 | [error] 75 | 76 | 77 | 78 | === TEST 4: location if (raw query) 79 | --- config 80 | location /redis { 81 | if ($uri ~ "(special.*)") { 82 | set $n 2; 83 | } 84 | redis2_raw_query 'flushall\r\n'; 85 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 86 | } 87 | --- request 88 | GET /redis/special 89 | --- response_body eval 90 | "+OK\r\n" 91 | --- no_error_log 92 | [error] 93 | 94 | -------------------------------------------------------------------------------- /src/common.rl: -------------------------------------------------------------------------------- 1 | %%{ 2 | machine common; 3 | 4 | action read_char { 5 | dd("reading %c", *p); 6 | } 7 | 8 | CR = "\r"; 9 | LF = "\n"; 10 | CRLF = CR LF; # $read_char; 11 | 12 | action finalize { 13 | dd("done!"); 14 | done = 1; 15 | } 16 | 17 | action read_size { 18 | ctx->chunk_size *= 10; 19 | ctx->chunk_size += *p - '0'; 20 | dd("read chunk size: %d", (int) ctx->chunk_size); 21 | } 22 | 23 | action start_reading_size { 24 | dd("start reading chunk size"); 25 | ctx->chunk_bytes_read = 0; 26 | ctx->chunk_size = 0; 27 | } 28 | 29 | action start_reading_data { 30 | dd("start reading data"); 31 | ctx->chunk_bytes_read = 0; 32 | } 33 | 34 | action test_len { 35 | #if 0 36 | fprintf(stderr, "test chunk len: %d < %d\n", 37 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 38 | #endif 39 | ctx->chunk_bytes_read++ < ctx->chunk_size 40 | } 41 | 42 | chunk_size = ([1-9] digit*) >start_reading_size $read_size 43 | ; 44 | 45 | chunk_data_octet = any when test_len 46 | ; 47 | 48 | chunk_data = chunk_data_octet+; 49 | 50 | action read_chunk { 51 | ctx->chunks_read++; 52 | dd("have read chunk %d, %.*s", (int) ctx->chunks_read, 53 | (int) (p - (signed char *) b->last), (signed char *) b->last); 54 | } 55 | 56 | action check_data_complete { 57 | #if 0 58 | fprintf(stderr, 59 | "check_data_complete: chunk bytes read: %d, chunk size: %d\n", 60 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 61 | #endif 62 | ctx->chunk_bytes_read == ctx->chunk_size + 1 63 | } 64 | 65 | trailer = CRLF @read_chunk 66 | ; 67 | 68 | chunk = "$" "0"+ CRLF trailer 69 | | "$-" digit+ trailer 70 | | "$" chunk_size CRLF chunk_data CR when check_data_complete LF @read_chunk 71 | ; 72 | 73 | single_line_reply = [:\+\-] (any* -- CRLF) CRLF; 74 | 75 | }%% 76 | 77 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: focal 3 | 4 | branches: 5 | only: 6 | - "master" 7 | 8 | os: linux 9 | 10 | language: c 11 | 12 | compiler: 13 | - gcc 14 | 15 | addons: 16 | apt: 17 | packages: 18 | - axel 19 | - cpanminus 20 | - libtest-base-perl 21 | - libtext-diff-perl 22 | - liburi-perl 23 | - libwww-perl 24 | - libtest-longstring-perl 25 | - liblist-moreutils-perl 26 | 27 | cache: 28 | apt: true 29 | 30 | env: 31 | global: 32 | - LUAJIT_PREFIX=/opt/luajit21 33 | - LUAJIT_LIB=$LUAJIT_PREFIX/lib 34 | - LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH 35 | - LUAJIT_INC=$LUAJIT_PREFIX/include/luajit-2.1 36 | - LUA_INCLUDE_DIR=$LUAJIT_INC 37 | - LUA_CMODULE_DIR=/lib 38 | - JOBS=3 39 | - NGX_BUILD_JOBS=$JOBS 40 | matrix: 41 | - NGINX_VERSION=1.29.4 42 | 43 | services: 44 | - redis-server 45 | 46 | install: 47 | - git clone https://github.com/openresty/nginx-devel-utils.git 48 | - git clone https://github.com/openresty/openresty.git ../openresty 49 | - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx 50 | - git clone https://github.com/simpl/ngx_devel_kit.git ../ndk-nginx-module 51 | - git clone https://github.com/openresty/test-nginx.git 52 | - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git 53 | - git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module 54 | - git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core 55 | - git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache 56 | - git clone https://github.com/openresty/nginx-eval-module.git ../eval-nginx-module 57 | - git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module 58 | - git clone https://github.com/openresty/set-misc-nginx-module.git ../set-misc-nginx-module 59 | 60 | script: 61 | - cd luajit2 62 | - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' > build.log 2>&1 || (cat build.log && exit 1) 63 | - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) 64 | - cd .. 65 | - cd test-nginx && sudo cpanm . && cd .. 66 | - export PATH=$PWD/work/nginx/sbin:$PWD/nginx-devel-utils:$PATH 67 | - export NGX_BUILD_CC=$CC 68 | - sh util/build.sh $NGINX_VERSION > build.log 2>&1 || (cat build.log && exit 1) 69 | - nginx -V 70 | - ldd `which nginx`|grep luajit 71 | - prove -I. -r t 72 | -------------------------------------------------------------------------------- /t/eval.t: -------------------------------------------------------------------------------- 1 | # vi:ft= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | plan tests => repeat_each() * (2 * blocks()); 7 | 8 | $ENV{TEST_NGINX_REDIS_PORT} ||= 6379; 9 | $ENV{TEST_NGINX_MEMCACHED_PORT} ||= 11984; 10 | $ENV{LUA_CPATH} ||= '/home/lz/luax/?.so'; 11 | 12 | #master_on; 13 | #worker_connections 1024; 14 | 15 | #no_diff; 16 | 17 | #log_level 'warn'; 18 | 19 | run_tests(); 20 | 21 | __DATA__ 22 | 23 | === TEST 1: no subrequest in memory 24 | --- config 25 | location /foo { 26 | eval_override_content_type 'application/octet-stream'; 27 | eval_subrequest_in_memory off; 28 | eval $res { 29 | redis2_query set one first; 30 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 31 | } 32 | echo [$res]; 33 | } 34 | --- request 35 | GET /foo 36 | --- response_body_like: ^\[\+OK\r\n\]$ 37 | --- error_code: 200 38 | 39 | 40 | 41 | === TEST 2: subrequest in memory 42 | --- config 43 | location /foo { 44 | eval_override_content_type 'application/octet-stream'; 45 | eval_subrequest_in_memory on; 46 | eval $res { 47 | #redis2_literal_raw_query 'set one 5\r\nfirst\r\n'; 48 | #redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 49 | set $memcached_key foo; 50 | memcached_pass 127.0.0.1:$TEST_NGINX_MEMCACHED_PORT; 51 | } 52 | echo [$res]; 53 | } 54 | --- request 55 | GET /foo 56 | --- response_body_like: ^\[\+OK\r\n\]$ 57 | --- error_code: 200 58 | --- SKIP 59 | 60 | 61 | 62 | === TEST 3: subrequest in memory 63 | --- config 64 | location = /redis { 65 | internal; 66 | redis2_query get $query_string; 67 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 68 | } 69 | 70 | location ~ ^/([a-zA-Z0-9]+)$ { 71 | set $path $1; 72 | rewrite_by_lua ' 73 | local resp = ngx.location.capture("/redis", 74 | { args = ngx.var.path }) 75 | 76 | if resp.status ~= ngx.HTTP_OK then 77 | return ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) 78 | end 79 | 80 | local parser = require("redis.parser") 81 | local res, rc = parser.parse_reply(resp.body) 82 | -- print("rc = ", rc, " ", parser.BULK_REPLY) 83 | if rc ~= parser.BULK_REPLY then 84 | return ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) 85 | end 86 | 87 | print("res = ", res) 88 | 89 | -- ngx.print(ngx.var.path) 90 | return ngx.redirect(res) 91 | '; 92 | 93 | content_by_lua return; 94 | } 95 | --- request 96 | GET /foo 97 | --- response_headers_like 98 | Location: http://[^/]+/hello/world 99 | --- response_body_like: 302 Found 100 | --- error_code: 302 101 | --- SKIP 102 | 103 | -------------------------------------------------------------------------------- /t/pipeline.t: -------------------------------------------------------------------------------- 1 | # vi:ft= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * (2 * blocks() + 1); 9 | 10 | $ENV{TEST_NGINX_REDIS_PORT} ||= 6379; 11 | 12 | #master_on; 13 | #worker_connections 1024; 14 | 15 | #no_diff; 16 | 17 | log_level 'warn'; 18 | 19 | run_tests(); 20 | 21 | __DATA__ 22 | 23 | === TEST 1: advanced query (2 pipelined) 24 | --- config 25 | location /a { 26 | redis2_query set hello world; 27 | redis2_query get hello; 28 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 29 | } 30 | --- request 31 | GET /a 32 | --- response_body eval 33 | "+OK\r\n\$5\r\nworld\r\n" 34 | 35 | 36 | 37 | === TEST 2: 4 pipelined 38 | --- config 39 | location /a { 40 | redis2_query set hello world; 41 | redis2_query get hello; 42 | redis2_query set hello agentzh; 43 | redis2_query get hello; 44 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 45 | } 46 | --- request 47 | GET /a 48 | --- response_body eval 49 | "+OK\r\n\$5\r\nworld\r\n+OK\r\n\$7\r\nagentzh\r\n" 50 | 51 | 52 | 53 | === TEST 3: redis2_raw_queries (2 queries) 54 | --- config 55 | location /pipelined2 { 56 | set_unescape_uri $n $arg_n; 57 | set_unescape_uri $cmds $arg_cmds; 58 | 59 | redis2_raw_queries $n $cmds; 60 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 61 | } 62 | --- request 63 | GET /pipelined2?n=2&cmds=flushall%0D%0Aget%20key%0D%0A 64 | --- response_body eval 65 | "+OK\r\n\$-1\r\n" 66 | 67 | 68 | 69 | === TEST 4: redis2_raw_queries (2 queries, smaller N arg, bad) 70 | --- config 71 | location /pipelined2 { 72 | set_unescape_uri $n $arg_n; 73 | set_unescape_uri $cmds $arg_cmds; 74 | 75 | redis2_raw_queries $n $cmds; 76 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 77 | } 78 | --- request 79 | GET /pipelined2?n=1&cmds=flushall%0D%0Aget%20key%0D%0A 80 | --- response_body eval 81 | "+OK\r\n" 82 | 83 | 84 | 85 | === TEST 5: redis2_raw_queries (2 queries, larger N arg, bad) 86 | --- config 87 | redis2_read_timeout 100ms; 88 | location /pipelined2 { 89 | set_unescape_uri $n $arg_n; 90 | set_unescape_uri $cmds $arg_cmds; 91 | 92 | redis2_raw_queries $n $cmds; 93 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 94 | } 95 | --- request 96 | GET /pipelined2?n=3&cmds=flushall%0D%0Aget%20key%0D%0A 97 | --- ignore_response 98 | --- no_error_log 99 | [alert] 100 | [crit] 101 | --- error_log eval 102 | qr/upstream timed out .*? while reading upstream/ 103 | 104 | 105 | 106 | === TEST 6: advanced query (2 pipelined) - keepalive 107 | --- http_config 108 | upstream backend { 109 | server 127.0.0.1:$TEST_NGINX_REDIS_PORT; 110 | keepalive 100; 111 | } 112 | --- config 113 | location /a { 114 | redis2_query set hello world; 115 | redis2_query get hello; 116 | redis2_pass backend; 117 | } 118 | --- request 119 | GET /a 120 | --- response_body eval 121 | "+OK\r\n\$5\r\nworld\r\n" 122 | 123 | -------------------------------------------------------------------------------- /valgrind.suppress: -------------------------------------------------------------------------------- 1 | { 2 | 3 | Memcheck:Leak 4 | fun:malloc 5 | fun:ngx_alloc 6 | fun:ngx_create_pool 7 | fun:ngx_event_accept 8 | fun:ngx_epoll_process_events 9 | fun:ngx_process_events_and_timers 10 | fun:ngx_single_process_cycle 11 | } 12 | { 13 | 14 | Memcheck:Leak 15 | fun:malloc 16 | fun:ngx_alloc 17 | fun:ngx_create_pool 18 | fun:ngx_http_init_request 19 | fun:ngx_epoll_process_events 20 | fun:ngx_process_events_and_timers 21 | fun:ngx_single_process_cycle 22 | } 23 | { 24 | 25 | Memcheck:Addr4 26 | fun:ngx_init_cycle 27 | fun:ngx_master_process_cycle 28 | fun:main 29 | } 30 | { 31 | 32 | Memcheck:Leak 33 | fun:malloc 34 | fun:ngx_calloc 35 | fun:ngx_event_process_init 36 | } 37 | { 38 | 39 | Memcheck:Cond 40 | fun:ngx_conf_flush_files 41 | } 42 | { 43 | 44 | Memcheck:Param 45 | epoll_ctl(event) 46 | fun:epoll_ctl 47 | } 48 | { 49 | 50 | Memcheck:Leak 51 | fun:malloc 52 | fun:ngx_alloc 53 | fun:ngx_event_process_init 54 | } 55 | { 56 | nginx-core-process-init 57 | Memcheck:Leak 58 | fun:malloc 59 | fun:ngx_alloc 60 | fun:ngx_event_process_init 61 | } 62 | { 63 | nginx-core-crc32-init 64 | Memcheck:Leak 65 | fun:malloc 66 | fun:ngx_alloc 67 | fun:ngx_crc32_table_init 68 | fun:main 69 | } 70 | { 71 | palloc_large_for_init_request 72 | Memcheck:Leak 73 | fun:malloc 74 | fun:ngx_alloc 75 | fun:ngx_palloc_large 76 | fun:ngx_palloc 77 | fun:ngx_pcalloc 78 | fun:ngx_http_init_request 79 | fun:ngx_epoll_process_events 80 | fun:ngx_process_events_and_timers 81 | } 82 | { 83 | palloc_large_for_create_temp_buf 84 | Memcheck:Leak 85 | fun:malloc 86 | fun:ngx_alloc 87 | fun:ngx_palloc_large 88 | fun:ngx_palloc 89 | fun:ngx_create_temp_buf 90 | fun:ngx_http_init_request 91 | fun:ngx_epoll_process_events 92 | fun:ngx_process_events_and_timers 93 | } 94 | { 95 | accept_create_pool 96 | Memcheck:Leak 97 | fun:memalign 98 | fun:posix_memalign 99 | fun:ngx_memalign 100 | fun:ngx_create_pool 101 | fun:ngx_event_accept 102 | fun:ngx_epoll_process_events 103 | fun:ngx_process_events_and_timers 104 | } 105 | { 106 | create_pool_for_init_req 107 | Memcheck:Leak 108 | fun:memalign 109 | fun:posix_memalign 110 | fun:ngx_memalign 111 | fun:ngx_create_pool 112 | fun:ngx_http_init_request 113 | fun:ngx_epoll_process_events 114 | fun:ngx_process_events_and_timers 115 | } 116 | { 117 | 118 | Memcheck:Leak 119 | fun:malloc 120 | fun:ngx_alloc 121 | fun:ngx_create_pool 122 | fun:ngx_http_upstream_connect 123 | fun:ngx_http_upstream_init_request 124 | } 125 | { 126 | 127 | Memcheck:Cond 128 | fun:index 129 | fun:expand_dynamic_string_token 130 | fun:_dl_map_object 131 | fun:map_doit 132 | fun:_dl_catch_error 133 | fun:do_preload 134 | fun:dl_main 135 | } 136 | { 137 | 138 | Memcheck:Leak 139 | match-leak-kinds: definite 140 | fun:malloc 141 | fun:ngx_alloc 142 | fun:ngx_set_environment 143 | fun:ngx_single_process_cycle 144 | } 145 | { 146 | 147 | Memcheck:Leak 148 | match-leak-kinds: definite 149 | fun:malloc 150 | fun:ngx_alloc 151 | fun:ngx_set_environment 152 | fun:ngx_worker_process_init 153 | fun:ngx_worker_process_cycle 154 | } 155 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_reply.rl: -------------------------------------------------------------------------------- 1 | #ifndef DDEBUG 2 | #define DDEBUG 0 3 | #endif 4 | #include "ddebug.h" 5 | 6 | #include "ngx_http_redis2_reply.h" 7 | #include "ngx_http_redis2_util.h" 8 | #include 9 | 10 | %%{ 11 | machine reply; 12 | 13 | include multi_bulk_reply "multi_bulk_reply.rl"; 14 | 15 | main := single_line_reply @finalize 16 | | chunk @finalize 17 | | multi_bulk_reply 18 | ; 19 | 20 | }%% 21 | 22 | %% write data; 23 | 24 | ngx_int_t 25 | ngx_http_redis2_process_reply(ngx_http_redis2_ctx_t *ctx, ssize_t bytes) 26 | { 27 | ngx_buf_t *b; 28 | ngx_http_upstream_t *u; 29 | ngx_str_t buf; 30 | ngx_flag_t done; 31 | ngx_chain_t *cl = NULL; 32 | ngx_chain_t **ll = NULL; 33 | 34 | int cs; 35 | signed char *p; 36 | signed char *orig_p; 37 | ssize_t orig_len; 38 | signed char *pe; 39 | 40 | u = ctx->request->upstream; 41 | b = &u->buffer; 42 | 43 | orig_p = (signed char *) b->last; 44 | orig_len = bytes; 45 | 46 | while (ctx->query_count) { 47 | done = 0; 48 | 49 | if (ctx->state == NGX_ERROR) { 50 | dd("init the state machine"); 51 | 52 | %% write init; 53 | 54 | ctx->state = cs; 55 | 56 | } else { 57 | cs = ctx->state; 58 | dd("resumed the old state %d", cs); 59 | } 60 | 61 | p = (signed char *) b->last; 62 | pe = (signed char *) b->last + bytes; 63 | 64 | dd("response body: %.*s", (int) bytes, p); 65 | 66 | %% write exec; 67 | 68 | dd("state after exec: %d, done: %d, %.*s", cs, (int) done, 69 | (int) (bytes - ((u_char *) p - b->last)), p); 70 | 71 | ctx->state = cs; 72 | 73 | if (!done && cs == reply_error) { 74 | if (cl) { 75 | cl->buf->last = cl->buf->pos; 76 | cl = NULL; 77 | *ll = NULL; 78 | } 79 | 80 | buf.data = b->pos; 81 | buf.len = b->last - b->pos + bytes; 82 | 83 | ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0, 84 | "Redis server returned invalid response near pos %z in " 85 | "\"%V\"", 86 | (ssize_t) ((u_char *) p - b->pos), &buf); 87 | 88 | u->length = 0; 89 | 90 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 91 | } 92 | 93 | if (cl == NULL) { 94 | for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { 95 | ll = &cl->next; 96 | } 97 | 98 | cl = ngx_chain_get_free_buf(ctx->request->pool, &u->free_bufs); 99 | if (cl == NULL) { 100 | u->length = 0; 101 | return NGX_ERROR; 102 | } 103 | 104 | cl->buf->flush = 1; 105 | cl->buf->memory = 1; 106 | 107 | *ll = cl; 108 | 109 | dd("response body: %.*s", (int) bytes, p); 110 | 111 | cl->buf->pos = b->last; 112 | cl->buf->last = (u_char *) p; 113 | cl->buf->tag = u->output.tag; 114 | 115 | } else { 116 | cl->buf->last = (u_char *) p; 117 | } 118 | 119 | bytes -= (ssize_t) ((u_char *) p - b->last); 120 | b->last = (u_char *) p; 121 | 122 | if (done) { 123 | dd("response parser done"); 124 | 125 | ctx->query_count--; 126 | 127 | if (ctx->query_count == 0) { 128 | if (cs == reply_error) { 129 | buf.data = (u_char *) p; 130 | buf.len = orig_p - p + orig_len; 131 | 132 | ngx_log_error(NGX_LOG_WARN, ctx->request->connection->log, 133 | 0, "Redis server returned extra bytes: \"%V\" (len %z)", 134 | &buf, buf.len); 135 | 136 | #if 0 137 | if (cl) { 138 | cl->buf->last = cl->buf->pos; 139 | cl = NULL; 140 | *ll = NULL; 141 | } 142 | 143 | u->length = 0; 144 | 145 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 146 | #endif 147 | 148 | } else { 149 | #if defined(nginx_version) && nginx_version >= 1001004 150 | u->keepalive = 1; 151 | #endif 152 | } 153 | 154 | u->length = 0; 155 | 156 | break; 157 | 158 | } else { 159 | ctx->state = NGX_ERROR; 160 | /* continue */ 161 | } 162 | 163 | } else { 164 | /* need more data */ 165 | break; 166 | } 167 | } 168 | 169 | return NGX_OK; 170 | } 171 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_util.c: -------------------------------------------------------------------------------- 1 | #ifndef DDEBUG 2 | #define DDEBUG 0 3 | #endif 4 | #include "ddebug.h" 5 | 6 | #include "ngx_http_redis2_util.h" 7 | 8 | 9 | static size_t ngx_get_num_size(uint64_t i); 10 | 11 | 12 | char * 13 | ngx_http_redis2_set_complex_value_slot(ngx_conf_t *cf, ngx_command_t *cmd, 14 | void *conf) 15 | { 16 | char *p = conf; 17 | ngx_http_complex_value_t **field; 18 | ngx_str_t *value; 19 | ngx_http_compile_complex_value_t ccv; 20 | 21 | field = (ngx_http_complex_value_t **) (p + cmd->offset); 22 | 23 | if (*field) { 24 | return "is duplicate"; 25 | } 26 | 27 | *field = ngx_palloc(cf->pool, sizeof(ngx_http_complex_value_t)); 28 | if (*field == NULL) { 29 | return NGX_CONF_ERROR; 30 | } 31 | 32 | value = cf->args->elts; 33 | 34 | if (value[1].len == 0) { 35 | ngx_memzero(*field, sizeof(ngx_http_complex_value_t)); 36 | return NGX_OK; 37 | } 38 | 39 | ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); 40 | 41 | ccv.cf = cf; 42 | ccv.value = &value[1]; 43 | ccv.complex_value = *field; 44 | 45 | if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { 46 | return NGX_CONF_ERROR; 47 | } 48 | 49 | return NGX_CONF_OK; 50 | } 51 | 52 | 53 | ngx_http_upstream_srv_conf_t * 54 | ngx_http_redis2_upstream_add(ngx_http_request_t *r, ngx_url_t *url) 55 | { 56 | ngx_http_upstream_main_conf_t *umcf; 57 | ngx_http_upstream_srv_conf_t **uscfp; 58 | ngx_uint_t i; 59 | 60 | umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module); 61 | 62 | uscfp = umcf->upstreams.elts; 63 | 64 | for (i = 0; i < umcf->upstreams.nelts; i++) { 65 | 66 | if (uscfp[i]->host.len != url->host.len 67 | || ngx_strncasecmp(uscfp[i]->host.data, url->host.data, 68 | url->host.len) != 0) 69 | { 70 | dd("upstream_add: host not match"); 71 | continue; 72 | } 73 | 74 | if (uscfp[i]->port != url->port) { 75 | dd("upstream_add: port not match: %d != %d", 76 | (int) uscfp[i]->port, (int) url->port); 77 | continue; 78 | } 79 | 80 | #if (nginx_version < 1011006) 81 | if (uscfp[i]->default_port 82 | && url->default_port 83 | && uscfp[i]->default_port != url->default_port) 84 | { 85 | dd("upstream_add: default_port not match"); 86 | continue; 87 | } 88 | #endif 89 | 90 | return uscfp[i]; 91 | } 92 | 93 | dd("no upstream found: %.*s", (int) url->host.len, url->host.data); 94 | 95 | return NULL; 96 | } 97 | 98 | 99 | static size_t 100 | ngx_get_num_size(uint64_t i) 101 | { 102 | size_t n = 0; 103 | 104 | do { 105 | i = i / 10; 106 | n++; 107 | } while (i > 0); 108 | 109 | return n; 110 | } 111 | 112 | 113 | ngx_int_t 114 | ngx_http_redis2_build_query(ngx_http_request_t *r, ngx_array_t *queries, 115 | ngx_buf_t **b) 116 | { 117 | ngx_uint_t i, j; 118 | ngx_uint_t n; 119 | ngx_str_t *arg; 120 | ngx_array_t *args; 121 | size_t len; 122 | ngx_array_t **query_args; 123 | ngx_http_complex_value_t **complex_arg; 124 | u_char *p; 125 | ngx_http_redis2_loc_conf_t *rlcf; 126 | 127 | rlcf = ngx_http_get_module_loc_conf(r, ngx_http_redis2_module); 128 | 129 | query_args = rlcf->queries->elts; 130 | 131 | n = 0; 132 | for (i = 0; i < rlcf->queries->nelts; i++) { 133 | for (j = 0; j < query_args[i]->nelts; j++) { 134 | n++; 135 | } 136 | } 137 | 138 | args = ngx_array_create(r->pool, n, sizeof(ngx_str_t)); 139 | 140 | if (args == NULL) { 141 | return NGX_ERROR; 142 | } 143 | 144 | len = 0; 145 | n = 0; 146 | 147 | for (i = 0; i < rlcf->queries->nelts; i++) { 148 | complex_arg = query_args[i]->elts; 149 | 150 | len += sizeof("*") - 1 151 | + ngx_get_num_size(query_args[i]->nelts) 152 | + sizeof("\r\n") - 1 153 | ; 154 | 155 | for (j = 0; j < query_args[i]->nelts; j++) { 156 | n++; 157 | 158 | arg = ngx_array_push(args); 159 | if (arg == NULL) { 160 | return NGX_ERROR; 161 | } 162 | 163 | if (ngx_http_complex_value(r, complex_arg[j], arg) != NGX_OK) { 164 | return NGX_ERROR; 165 | } 166 | 167 | len += sizeof("$") - 1 168 | + ngx_get_num_size(arg->len) 169 | + sizeof("\r\n") - 1 170 | + arg->len 171 | + sizeof("\r\n") - 1 172 | ; 173 | } 174 | } 175 | 176 | *b = ngx_create_temp_buf(r->pool, len); 177 | if (*b == NULL) { 178 | return NGX_ERROR; 179 | } 180 | 181 | p = (*b)->last; 182 | 183 | arg = args->elts; 184 | 185 | n = 0; 186 | for (i = 0; i < rlcf->queries->nelts; i++) { 187 | *p++ = '*'; 188 | p = ngx_sprintf(p, "%uz", query_args[i]->nelts); 189 | *p++ = '\r'; *p++ = '\n'; 190 | 191 | for (j = 0; j < query_args[i]->nelts; j++) { 192 | *p++ = '$'; 193 | p = ngx_sprintf(p, "%uz", arg[n].len); 194 | *p++ = '\r'; *p++ = '\n'; 195 | p = ngx_copy(p, arg[n].data, arg[n].len); 196 | *p++ = '\r'; *p++ = '\n'; 197 | 198 | n++; 199 | } 200 | } 201 | 202 | dd("query: %.*s", (int) (p - (*b)->pos), (*b)->pos); 203 | 204 | if (p - (*b)->pos != (ssize_t) len) { 205 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 206 | "redis2: redis2_query buffer error %uz != %uz", 207 | (size_t) (p - (*b)->pos), len); 208 | 209 | return NGX_ERROR; 210 | } 211 | 212 | (*b)->last = p; 213 | 214 | return NGX_OK; 215 | } 216 | 217 | -------------------------------------------------------------------------------- /t/bugs.t: -------------------------------------------------------------------------------- 1 | # vi:ft= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | #repeat_each(2); 7 | 8 | plan tests => repeat_each() * (3 * blocks()); 9 | 10 | $ENV{TEST_NGINX_REDIS_PORT} ||= 6379; 11 | 12 | #master_on; 13 | #worker_connections 1024; 14 | 15 | #no_diff; 16 | no_long_string; 17 | 18 | #log_level 'warn'; 19 | 20 | run_tests(); 21 | 22 | __DATA__ 23 | 24 | === TEST 1: 0 in bulk size 25 | --- config 26 | location /set { 27 | redis2_query set 'counters::stats::list' '[["mafiaclans.eu", 12], ["picfu.net", 5], ["www.test.com", 0], ["www.ayom.com", 0], ["www.21dezember2012.org", 0], ["the-indie.ch", 0], ["spiele-check.de", 0], ["online-right-now.net", 0], ["google.com", 0]]'; 28 | 29 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 30 | } 31 | 32 | location /get { 33 | redis2_raw_query 'get counters::stats::list\r\n'; 34 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 35 | } 36 | 37 | location /main2 { 38 | content_by_lua ' 39 | local res = ngx.location.capture("/set"); 40 | ngx.print(res.body) 41 | res = ngx.location.capture("/get"); 42 | ngx.print(res.body) 43 | '; 44 | } 45 | location /main { 46 | # echo_location is buggy...sigh. 47 | echo_location /set; 48 | echo_location /get; 49 | } 50 | --- request 51 | GET /main2 52 | --- response_body eval 53 | qq{+OK\r 54 | \$207\r 55 | [["mafiaclans.eu", 12], ["picfu.net", 5], ["www.test.com", 0], ["www.ayom.com", 0], ["www.21dezember2012.org", 0], ["the-indie.ch", 0], ["spiele-check.de", 0], ["online-right-now.net", 0], ["google.com", 0]]\r 56 | } 57 | --- no_error_log 58 | [error] 59 | 60 | 61 | 62 | === TEST 2: "0" digit in bulk count number 63 | --- config 64 | location /set { 65 | redis2_query set a hello; 66 | 67 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 68 | } 69 | 70 | location /get { 71 | redis2_query mget a a a a a a a a a a; 72 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 73 | } 74 | 75 | location /main { 76 | content_by_lua ' 77 | local res = ngx.location.capture("/set"); 78 | ngx.print(res.body) 79 | res = ngx.location.capture("/get"); 80 | ngx.print(res.body) 81 | '; 82 | } 83 | --- request 84 | GET /main 85 | --- response_body eval 86 | qq{+OK\r 87 | \*10\r 88 | } . "\$5\r 89 | hello\r 90 | " x 10 91 | --- no_error_log 92 | [error] 93 | 94 | 95 | 96 | === TEST 3: zero bulk count 97 | --- config 98 | location /set { 99 | redis2_query flushall; 100 | 101 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 102 | } 103 | 104 | location /get { 105 | redis2_query lrange blah 0 0; 106 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 107 | } 108 | 109 | location /main { 110 | content_by_lua ' 111 | local res = ngx.location.capture("/set"); 112 | ngx.print(res.body) 113 | res = ngx.location.capture("/get"); 114 | ngx.print(res.body) 115 | '; 116 | } 117 | --- request 118 | GET /main 119 | --- response_body eval 120 | qq{+OK\r 121 | \*0\r 122 | } 123 | --- no_error_log 124 | [error] 125 | 126 | 127 | 128 | === TEST 4: \r\n in content: sanity check 129 | --- config 130 | location /set { 131 | redis2_query set A 'hello\r\nworld'; 132 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 133 | } 134 | 135 | location /get { 136 | redis2_query get A; 137 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 138 | } 139 | 140 | location /main { 141 | content_by_lua ' 142 | local res = ngx.location.capture("/set"); 143 | ngx.print(res.body) 144 | res = ngx.location.capture("/get"); 145 | ngx.print(res.body) 146 | '; 147 | } 148 | --- request 149 | GET /main 150 | --- response_body eval 151 | qq{+OK\r 152 | \$12\r 153 | hello\r 154 | world\r 155 | } 156 | --- no_error_log 157 | [error] 158 | 159 | 160 | 161 | === TEST 5: github issue #5: Get content from Redis with "\r\n" in text fails with "Redis server returned extra bytes 162 | --- config 163 | location /set { 164 | redis2_query flushall; 165 | redis2_query hset "/step2redisallhget?auctionid=2l" "content" "a 1line \r\n and 2nd"; 166 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 167 | } 168 | 169 | location /get { 170 | redis2_query hget "/step2redisallhget?auctionid=2l" "content"; 171 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 172 | } 173 | 174 | location /main { 175 | content_by_lua ' 176 | local res = ngx.location.capture("/set"); 177 | ngx.print(res.body) 178 | res = ngx.location.capture("/get"); 179 | ngx.print(res.body) 180 | '; 181 | } 182 | --- request 183 | GET /main 184 | --- response_body eval 185 | qq{+OK\r 186 | :1\r 187 | \$18\r\na 1line \r\n and 2nd\r 188 | } 189 | --- no_error_log 190 | [error] 191 | 192 | 193 | 194 | === TEST 6: set empty param 195 | --- config 196 | location /set { 197 | redis2_query flushall; 198 | redis2_query set foo ""; 199 | redis2_query get foo; 200 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 201 | } 202 | --- request 203 | GET /set 204 | --- response_body eval 205 | qq{+OK\r 206 | +OK\r 207 | \$0\r 208 | \r 209 | } 210 | --- no_error_log 211 | [error] 212 | 213 | 214 | 215 | === TEST 7: two pipelined multi-bulk replies 216 | --- config 217 | location /set { 218 | redis2_query subscribe foo; 219 | redis2_query unsubscribe foo; 220 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 221 | } 222 | --- request 223 | GET /set 224 | --- response_body eval 225 | qq{*3\r 226 | \$9\r 227 | subscribe\r 228 | \$3\r 229 | foo\r 230 | :1\r 231 | *3\r 232 | \$11\r 233 | unsubscribe\r 234 | \$3\r 235 | foo\r 236 | :0\r 237 | } 238 | --- no_error_log 239 | [error] 240 | 241 | 242 | 243 | === TEST 8: empty multi-bulk replies 244 | --- config 245 | location /set { 246 | redis2_query flushall; 247 | redis2_query blpop key 1; 248 | redis2_query set key 3; 249 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 250 | } 251 | --- request 252 | GET /set 253 | --- response_body eval 254 | qq{+OK\r 255 | *-1\r 256 | +OK\r 257 | } 258 | --- timeout: 5 259 | --- no_error_log 260 | [error] 261 | 262 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_handler.c: -------------------------------------------------------------------------------- 1 | #ifndef DDEBUG 2 | #define DDEBUG 0 3 | #endif 4 | #include "ddebug.h" 5 | 6 | #include "ngx_http_redis2_handler.h" 7 | #include "ngx_http_redis2_reply.h" 8 | #include "ngx_http_redis2_util.h" 9 | 10 | 11 | static ngx_int_t ngx_http_redis2_create_request(ngx_http_request_t *r); 12 | static ngx_int_t ngx_http_redis2_reinit_request(ngx_http_request_t *r); 13 | static ngx_int_t ngx_http_redis2_process_header(ngx_http_request_t *r); 14 | static ngx_int_t ngx_http_redis2_filter_init(void *data); 15 | static ngx_int_t ngx_http_redis2_filter(void *data, ssize_t bytes); 16 | static void ngx_http_redis2_abort_request(ngx_http_request_t *r); 17 | static void ngx_http_redis2_finalize_request(ngx_http_request_t *r, 18 | ngx_int_t rc); 19 | 20 | 21 | ngx_int_t 22 | ngx_http_redis2_handler(ngx_http_request_t *r) 23 | { 24 | ngx_int_t rc; 25 | ngx_http_upstream_t *u; 26 | ngx_http_redis2_ctx_t *ctx; 27 | ngx_http_redis2_loc_conf_t *rlcf; 28 | ngx_str_t target; 29 | ngx_url_t url; 30 | 31 | if (ngx_http_set_content_type(r) != NGX_OK) { 32 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 33 | } 34 | 35 | if (ngx_http_upstream_create(r) != NGX_OK) { 36 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 37 | } 38 | 39 | u = r->upstream; 40 | 41 | rlcf = ngx_http_get_module_loc_conf(r, ngx_http_redis2_module); 42 | 43 | if (rlcf->complex_target) { 44 | /* variables used in the redis2_pass directive */ 45 | 46 | if (ngx_http_complex_value(r, rlcf->complex_target, &target) 47 | != NGX_OK) 48 | { 49 | return NGX_ERROR; 50 | } 51 | 52 | if (target.len == 0) { 53 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 54 | "handler: empty \"redis2_pass\" target"); 55 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 56 | } 57 | 58 | url.host = target; 59 | url.port = 0; 60 | url.no_resolve = 1; 61 | 62 | rlcf->upstream.upstream = ngx_http_redis2_upstream_add(r, &url); 63 | 64 | if (rlcf->upstream.upstream == NULL) { 65 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 66 | "redis2: upstream \"%V\" not found", &target); 67 | 68 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 69 | } 70 | } 71 | 72 | ngx_str_set(&u->schema, "redis2://"); 73 | u->output.tag = (ngx_buf_tag_t) &ngx_http_redis2_module; 74 | 75 | u->conf = &rlcf->upstream; 76 | 77 | u->create_request = ngx_http_redis2_create_request; 78 | u->reinit_request = ngx_http_redis2_reinit_request; 79 | u->process_header = ngx_http_redis2_process_header; 80 | u->abort_request = ngx_http_redis2_abort_request; 81 | u->finalize_request = ngx_http_redis2_finalize_request; 82 | 83 | ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_redis2_ctx_t)); 84 | if (ctx == NULL) { 85 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 86 | } 87 | 88 | ctx->request = r; 89 | ctx->state = NGX_ERROR; 90 | 91 | ngx_http_set_ctx(r, ctx, ngx_http_redis2_module); 92 | 93 | u->input_filter_init = ngx_http_redis2_filter_init; 94 | u->input_filter = ngx_http_redis2_filter; 95 | u->input_filter_ctx = ctx; 96 | 97 | rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); 98 | 99 | if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { 100 | return rc; 101 | } 102 | 103 | return NGX_DONE; 104 | } 105 | 106 | 107 | static ngx_int_t 108 | ngx_http_redis2_create_request(ngx_http_request_t *r) 109 | { 110 | ngx_buf_t *b; 111 | ngx_chain_t *cl; 112 | ngx_http_redis2_loc_conf_t *rlcf; 113 | ngx_str_t query; 114 | ngx_str_t query_count; 115 | ngx_int_t rc; 116 | ngx_http_redis2_ctx_t *ctx; 117 | ngx_int_t n; 118 | 119 | ctx = ngx_http_get_module_ctx(r, ngx_http_redis2_module); 120 | 121 | rlcf = ngx_http_get_module_loc_conf(r, ngx_http_redis2_module); 122 | 123 | if (rlcf->queries) { 124 | ctx->query_count = rlcf->queries->nelts; 125 | 126 | rc = ngx_http_redis2_build_query(r, rlcf->queries, &b); 127 | if (rc != NGX_OK) { 128 | return rc; 129 | } 130 | 131 | } else if (rlcf->literal_query.len == 0) { 132 | if (rlcf->complex_query == NULL) { 133 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 134 | "no redis2 query specified or the query is empty"); 135 | 136 | return NGX_ERROR; 137 | } 138 | 139 | if (ngx_http_complex_value(r, rlcf->complex_query, &query) 140 | != NGX_OK) 141 | { 142 | return NGX_ERROR; 143 | } 144 | 145 | if (query.len == 0) { 146 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 147 | "the redis query is empty"); 148 | 149 | return NGX_ERROR; 150 | } 151 | 152 | if (rlcf->complex_query_count == NULL) { 153 | ctx->query_count = 1; 154 | 155 | } else { 156 | if (ngx_http_complex_value(r, rlcf->complex_query_count, 157 | &query_count) 158 | != NGX_OK) 159 | { 160 | return NGX_ERROR; 161 | } 162 | 163 | if (query_count.len == 0) { 164 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 165 | "the N argument to redis2_raw_queries is empty"); 166 | 167 | return NGX_ERROR; 168 | } 169 | 170 | n = ngx_atoi(query_count.data, query_count.len); 171 | if (n == NGX_ERROR || n == 0) { 172 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 173 | "the N argument to redis2_raw_queries is " 174 | "invalid"); 175 | 176 | return NGX_ERROR; 177 | } 178 | 179 | ctx->query_count = n; 180 | } 181 | 182 | b = ngx_create_temp_buf(r->pool, query.len); 183 | if (b == NULL) { 184 | return NGX_ERROR; 185 | } 186 | 187 | b->last = ngx_copy(b->pos, query.data, query.len); 188 | 189 | } else { 190 | ctx->query_count = 1; 191 | 192 | b = ngx_calloc_buf(r->pool); 193 | if (b == NULL) { 194 | return NGX_ERROR; 195 | } 196 | 197 | b->pos = rlcf->literal_query.data; 198 | b->last = b->pos + rlcf->literal_query.len; 199 | b->memory = 1; 200 | } 201 | 202 | cl = ngx_alloc_chain_link(r->pool); 203 | if (cl == NULL) { 204 | return NGX_ERROR; 205 | } 206 | 207 | cl->buf = b; 208 | cl->next = NULL; 209 | 210 | r->upstream->request_bufs = cl; 211 | 212 | ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 213 | "http redis2 request: \"%V\"", &rlcf->literal_query); 214 | 215 | return NGX_OK; 216 | } 217 | 218 | 219 | static ngx_int_t 220 | ngx_http_redis2_reinit_request(ngx_http_request_t *r) 221 | { 222 | return NGX_OK; 223 | } 224 | 225 | 226 | static ngx_int_t 227 | ngx_http_redis2_process_header(ngx_http_request_t *r) 228 | { 229 | ngx_http_upstream_t *u; 230 | ngx_http_redis2_ctx_t *ctx; 231 | ngx_buf_t *b; 232 | u_char chr; 233 | ngx_str_t buf; 234 | 235 | u = r->upstream; 236 | b = &u->buffer; 237 | 238 | if (b->last - b->pos < (ssize_t) sizeof(u_char)) { 239 | return NGX_AGAIN; 240 | } 241 | 242 | ctx = ngx_http_get_module_ctx(r, ngx_http_redis2_module); 243 | 244 | /* the first char is the response header */ 245 | 246 | chr = *b->pos; 247 | 248 | dd("response header: %c (ascii %d)", chr, chr); 249 | 250 | switch (chr) { 251 | case '+': 252 | case '-': 253 | case ':': 254 | case '$': 255 | case '*': 256 | ctx->filter = ngx_http_redis2_process_reply; 257 | break; 258 | 259 | default: 260 | buf.data = b->pos; 261 | buf.len = b->last - b->pos; 262 | 263 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 264 | "redis2 sent invalid response: \"%V\"", &buf); 265 | 266 | return NGX_HTTP_UPSTREAM_INVALID_HEADER; 267 | } 268 | 269 | u->headers_in.status_n = NGX_HTTP_OK; 270 | u->state->status = NGX_HTTP_OK; 271 | 272 | return NGX_OK; 273 | } 274 | 275 | 276 | static ngx_int_t 277 | ngx_http_redis2_filter_init(void *data) 278 | { 279 | #if 0 280 | ngx_http_redis2_ctx_t *ctx = data; 281 | 282 | ngx_http_upstream_t *u; 283 | 284 | u = ctx->request->upstream; 285 | #endif 286 | 287 | return NGX_OK; 288 | } 289 | 290 | 291 | static ngx_int_t 292 | ngx_http_redis2_filter(void *data, ssize_t bytes) 293 | { 294 | ngx_http_redis2_ctx_t *ctx = data; 295 | 296 | return ctx->filter(ctx, bytes); 297 | } 298 | 299 | 300 | static void 301 | ngx_http_redis2_abort_request(ngx_http_request_t *r) 302 | { 303 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 304 | "abort http redis2 request"); 305 | return; 306 | } 307 | 308 | 309 | static void 310 | ngx_http_redis2_finalize_request(ngx_http_request_t *r, ngx_int_t rc) 311 | { 312 | ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, 313 | "finalize http redis2 request"); 314 | 315 | if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { 316 | r->headers_out.status = rc; 317 | } 318 | 319 | return; 320 | } 321 | -------------------------------------------------------------------------------- /t/sanity.t: -------------------------------------------------------------------------------- 1 | # vi:ft= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * (2 * blocks() + 1); 9 | 10 | $ENV{TEST_NGINX_REDIS_PORT} ||= 6379; 11 | 12 | #master_on; 13 | #worker_connections 1024; 14 | 15 | #no_diff; 16 | 17 | #log_level 'warn'; 18 | 19 | run_tests(); 20 | 21 | __DATA__ 22 | 23 | === TEST 1: no query 24 | --- config 25 | location /foo { 26 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 27 | } 28 | --- request 29 | GET /foo 30 | --- response_body_like: 500 Internal Server Error 31 | --- error_code: 500 32 | 33 | 34 | 35 | === TEST 2: empty query 36 | --- config 37 | location /foo { 38 | redis2_literal_raw_query ""; 39 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 40 | } 41 | --- request 42 | GET /foo 43 | --- response_body_like: 500 Internal Server Error 44 | --- error_code: 500 45 | 46 | 47 | 48 | === TEST 3: simple set query 49 | --- config 50 | location /foo { 51 | redis2_query set one first; 52 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 53 | } 54 | --- request 55 | GET /foo 56 | --- response_body eval 57 | "+OK\r\n" 58 | 59 | 60 | 61 | === TEST 4: simple get query 62 | --- config 63 | location /foo { 64 | redis2_literal_raw_query 'get not_exist_yet\r\n'; 65 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 66 | } 67 | --- request 68 | GET /foo 69 | --- response_body eval 70 | "+OK\r\n" 71 | --- SKIP 72 | 73 | 74 | 75 | === TEST 5: bad command 76 | --- config 77 | location /foo { 78 | redis2_literal_raw_query 'bad_cmd\r\n'; 79 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 80 | } 81 | --- request 82 | GET /foo 83 | --- response_body_like eval 84 | qr/-ERR unknown command [`']bad_cmd[`'](?:, with args beginning with: )?/ 85 | 86 | 87 | 88 | === TEST 6: integer reply 89 | --- config 90 | location /foo { 91 | redis2_literal_raw_query 'incr counter\r\n'; 92 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 93 | } 94 | --- request 95 | GET /foo 96 | --- response_body_like: ^:\d+\r\n$ 97 | 98 | 99 | 100 | === TEST 7: bulk reply 101 | --- config 102 | location /foo { 103 | redis2_literal_raw_query 'get not_exist_yet\r\n'; 104 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 105 | } 106 | --- request 107 | GET /foo 108 | --- response_body eval 109 | "\$-1\r\n" 110 | 111 | 112 | 113 | === TEST 8: bulk reply 114 | --- config 115 | location /set { 116 | redis2_literal_raw_query '*3\r\n$3\r\nset\r\n$3\r\none\r\n$5\r\nfirst\r\n'; 117 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 118 | } 119 | 120 | location /get { 121 | redis2_literal_raw_query '*2\r\n$3\r\nget\r\n$3\r\none\r\n'; 122 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 123 | } 124 | 125 | location /main { 126 | echo_location /set; 127 | echo_location /get; 128 | } 129 | --- request 130 | GET /main 131 | --- response_body eval 132 | "+OK\r\n\$5\r\nfirst\r\n" 133 | 134 | 135 | 136 | === TEST 9: bulk reply 137 | --- config 138 | location /set { 139 | redis2_query set one first; 140 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 141 | } 142 | 143 | location /get { 144 | redis2_query get one; 145 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 146 | } 147 | 148 | location /main { 149 | echo_location /set; 150 | echo_location /get; 151 | } 152 | --- request 153 | GET /main 154 | --- stap2 155 | global active = 0 156 | global pool 157 | 158 | F(ngx_http_init_request) { 159 | println("init req") 160 | active = 1 161 | } 162 | 163 | M(create-pool-done) { 164 | if (active) { 165 | printf("create pool: %p\n", $arg1) 166 | print_ubacktrace() 167 | active = 0; 168 | pool = $arg1 169 | } 170 | } 171 | 172 | F(ngx_destroy_pool) { 173 | if ($pool == pool) { 174 | printf("destroy: %p\n", $pool) 175 | print_ubacktrace() 176 | } 177 | } 178 | 179 | probe process("/lib64/libc.so.6").function("free") { 180 | if ($mem == pool) { 181 | printf("free: %p\n", $mem) 182 | } 183 | } 184 | --- stap2 185 | global pools 186 | 187 | M(create-pool-done) { 188 | key = sprintf("%p", $arg1) 189 | pools[key] = ubacktrace() 190 | 191 | # FIXME: the following line is necessary, and i don't know why 192 | key = sprintf("%p", $arg1) 193 | printf("create pool: %s\n", key) 194 | 195 | #print_ubacktrace() 196 | #print_ustack(ubacktrace()) 197 | #printf("%s", sprint_ubacktrace()) 198 | } 199 | 200 | F(ngx_destroy_pool) { 201 | key = sprintf("%p", $pool) 202 | delete pools[key] 203 | printf("destroy pool: %s\n", key) 204 | } 205 | 206 | probe end { 207 | println("leaked pools:\n") 208 | foreach (k in pools) { 209 | printf("%s: [%s]\n", k, pools[k]) 210 | } 211 | } 212 | 213 | --- response_body eval 214 | "+OK\r\n\$5\r\nfirst\r\n" 215 | 216 | 217 | 218 | === TEST 10: bulk reply (empty) 219 | --- config 220 | location /set { 221 | set $empty ''; 222 | redis2_query set one $empty; 223 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 224 | } 225 | 226 | location /get { 227 | redis2_query get one; 228 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 229 | } 230 | 231 | location /main { 232 | echo_location /set; 233 | echo_location /get; 234 | } 235 | --- request 236 | GET /main 237 | --- response_body eval 238 | "+OK\r\n\$0\r\n\r\n" 239 | 240 | 241 | 242 | === TEST 11: multi bulk reply (empty) 243 | --- config 244 | location /set_foo { 245 | set $empty ''; 246 | redis2_query set foo $empty; 247 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 248 | } 249 | 250 | location /set_bar { 251 | set $empty ''; 252 | redis2_query set bar $empty; 253 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 254 | } 255 | 256 | location /mget { 257 | redis2_literal_raw_query 'mget foo bar\r\n'; 258 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 259 | } 260 | 261 | location /main { 262 | echo_location /set_foo; 263 | echo_location /set_bar; 264 | echo_location /mget; 265 | } 266 | --- request 267 | GET /main 268 | --- response_body eval 269 | "+OK\r 270 | +OK\r 271 | *2\r 272 | \$0\r 273 | \r 274 | \$0\r 275 | \r 276 | " 277 | 278 | 279 | 280 | === TEST 12: multi bulk reply (empty) 281 | --- config 282 | location /main { 283 | set $query 'ping\r\n'; 284 | redis2_raw_query $query; 285 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 286 | } 287 | --- request 288 | GET /main 289 | --- response_body eval 290 | "+PONG\r\n" 291 | 292 | 293 | 294 | === TEST 13: multi bulk reply (empty) 295 | --- http_config 296 | upstream blah { 297 | server 127.0.0.1:$TEST_NGINX_REDIS_PORT; 298 | } 299 | --- config 300 | location /main { 301 | set $query 'ping\r\n'; 302 | redis2_raw_query $query; 303 | set $backend blah; 304 | redis2_pass $backend; 305 | } 306 | --- request 307 | GET /main 308 | --- response_body eval 309 | "+PONG\r\n" 310 | 311 | 312 | 313 | === TEST 14: eval compatibility 314 | --- config 315 | location /main { 316 | default_type 'application/octet-stream'; 317 | eval $res { 318 | set $query 'ping\r\n'; 319 | redis2_raw_query $query; 320 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 321 | } 322 | echo "[$res]"; 323 | } 324 | --- request 325 | GET /main 326 | --- response_body eval 327 | "[+PONG\r\n]" 328 | --- timeout: 5 329 | --- SKIP 330 | 331 | 332 | 333 | === TEST 15: lua compatibility (GET subrequest) 334 | --- config 335 | location /redis { 336 | internal; 337 | set_unescape_uri $query $arg_query; 338 | redis2_raw_query $query; 339 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 340 | } 341 | 342 | location /main { 343 | content_by_lua ' 344 | local res = ngx.location.capture("/redis", 345 | { args = { query = "ping\\r\\n" } } 346 | ) 347 | ngx.print("[" .. res.body .. "]") 348 | '; 349 | } 350 | --- request 351 | GET /main 352 | --- response_body eval 353 | "[+PONG\r\n]" 354 | 355 | 356 | 357 | === TEST 16: lua compatibility (POST subrequest) 358 | --- config 359 | location /redis { 360 | internal; 361 | redis2_raw_query $echo_request_body; 362 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 363 | } 364 | 365 | location /main { 366 | content_by_lua ' 367 | local res = ngx.location.capture("/redis", 368 | { method = ngx.HTTP_PUT, 369 | body = "ping\\r\\n" } 370 | ) 371 | ngx.print("[" .. res.body .. "]") 372 | '; 373 | } 374 | --- request 375 | GET /main 376 | --- response_body eval 377 | "[+PONG\r\n]" 378 | 379 | 380 | 381 | === TEST 17: CRLF in data 382 | --- http_config 383 | upstream backend { 384 | server 127.0.0.1:$TEST_NGINX_REDIS_PORT; 385 | keepalive 1; 386 | } 387 | --- config 388 | location /a { 389 | redis2_literal_raw_query '*3\r\n$3\r\nset\r\n$4\r\ncrlf\r\n$2\r\n\r\n\r\n'; 390 | redis2_pass backend; 391 | } 392 | location /b { 393 | redis2_literal_raw_query 'get crlf\r\n'; 394 | redis2_pass backend; 395 | } 396 | location /main { 397 | echo_location /a; 398 | echo_location /b; 399 | } 400 | --- request 401 | GET /main 402 | --- response_body eval 403 | "+OK\r\n\$2\r\n\r\n\r\n" 404 | 405 | 406 | 407 | === TEST 18: Unicode chars in data 408 | --- http_config 409 | upstream backend { 410 | server 127.0.0.1:$TEST_NGINX_REDIS_PORT; 411 | keepalive 1; 412 | } 413 | --- config 414 | location /a { 415 | redis2_literal_raw_query '*3\r\n$3\r\nset\r\n$4\r\ncrlf\r\n$6\r\n亦春\r\n'; 416 | redis2_pass backend; 417 | } 418 | location /b { 419 | redis2_literal_raw_query 'get crlf\r\n'; 420 | redis2_pass backend; 421 | } 422 | location /main { 423 | echo_location /a; 424 | echo_location /b; 425 | } 426 | --- request 427 | GET /main 428 | --- response_body eval 429 | "+OK\r\n\$6\r\n亦春\r\n" 430 | 431 | 432 | 433 | === TEST 19: advanced query 434 | --- config 435 | location /a { 436 | redis2_query set hello world; 437 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 438 | } 439 | location /b { 440 | redis2_query get hello; 441 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 442 | } 443 | location /main { 444 | echo_location /a; 445 | echo_location /b; 446 | } 447 | --- request 448 | GET /main 449 | --- response_body eval 450 | "+OK\r\n\$5\r\nworld\r\n" 451 | 452 | 453 | 454 | === TEST 20: request body 455 | --- config 456 | location /t { 457 | # these two settings must be the same to prevent 458 | # automatic buffering large request bodies 459 | # to temp files: 460 | client_body_buffer_size 8k; 461 | client_max_body_size 8k; 462 | 463 | redis2_query flushall; 464 | redis2_query lpush q1 $echo_request_body; 465 | redis2_query lpop q1; 466 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 467 | } 468 | --- request 469 | POST /t 470 | hello world 471 | --- response_body eval 472 | "+OK\r\n:1\r\n\$11\r\nhello world\r\n" 473 | --- no_error_log 474 | [error] 475 | 476 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_module.c: -------------------------------------------------------------------------------- 1 | #ifndef DDEBUG 2 | #define DDEBUG 0 3 | #endif 4 | #include "ddebug.h" 5 | 6 | #include "ngx_http_redis2_module.h" 7 | #include "ngx_http_redis2_handler.h" 8 | #include "ngx_http_redis2_util.h" 9 | 10 | 11 | static void *ngx_http_redis2_create_loc_conf(ngx_conf_t *cf); 12 | static char *ngx_http_redis2_merge_loc_conf(ngx_conf_t *cf, 13 | void *parent, void *child); 14 | static char *ngx_http_redis2_raw_queries(ngx_conf_t *cf, ngx_command_t *cmd, 15 | void *conf); 16 | static char *ngx_http_redis2_query(ngx_conf_t *cf, ngx_command_t *cmd, 17 | void *conf); 18 | static char *ngx_http_redis2_pass(ngx_conf_t *cf, ngx_command_t *cmd, 19 | void *conf); 20 | 21 | 22 | static ngx_conf_bitmask_t ngx_http_redis2_next_upstream_masks[] = { 23 | { ngx_string("error"), NGX_HTTP_UPSTREAM_FT_ERROR }, 24 | { ngx_string("timeout"), NGX_HTTP_UPSTREAM_FT_TIMEOUT }, 25 | { ngx_string("invalid_response"), NGX_HTTP_UPSTREAM_FT_INVALID_HEADER }, 26 | { ngx_string("not_found"), NGX_HTTP_UPSTREAM_FT_HTTP_404 }, 27 | { ngx_string("off"), NGX_HTTP_UPSTREAM_FT_OFF }, 28 | { ngx_null_string, 0 } 29 | }; 30 | 31 | 32 | static ngx_command_t ngx_http_redis2_commands[] = { 33 | 34 | { ngx_string("redis2_query"), 35 | NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_1MORE, 36 | ngx_http_redis2_query, 37 | NGX_HTTP_LOC_CONF_OFFSET, 38 | 0, 39 | NULL }, 40 | 41 | { ngx_string("redis2_raw_query"), 42 | NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1, 43 | ngx_http_redis2_set_complex_value_slot, 44 | NGX_HTTP_LOC_CONF_OFFSET, 45 | offsetof(ngx_http_redis2_loc_conf_t, complex_query), 46 | NULL }, 47 | 48 | { ngx_string("redis2_raw_queries"), 49 | NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE2, 50 | ngx_http_redis2_raw_queries, 51 | NGX_HTTP_LOC_CONF_OFFSET, 52 | 0, 53 | NULL }, 54 | 55 | { ngx_string("redis2_literal_raw_query"), 56 | NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1, 57 | ngx_conf_set_str_slot, 58 | NGX_HTTP_LOC_CONF_OFFSET, 59 | offsetof(ngx_http_redis2_loc_conf_t, literal_query), 60 | NULL }, 61 | 62 | 63 | { ngx_string("redis2_pass"), 64 | NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1, 65 | ngx_http_redis2_pass, 66 | NGX_HTTP_LOC_CONF_OFFSET, 67 | 0, 68 | NULL }, 69 | 70 | { ngx_string("redis2_bind"), 71 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 72 | ngx_http_upstream_bind_set_slot, 73 | NGX_HTTP_LOC_CONF_OFFSET, 74 | offsetof(ngx_http_redis2_loc_conf_t, upstream.local), 75 | NULL }, 76 | 77 | { ngx_string("redis2_connect_timeout"), 78 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 79 | ngx_conf_set_msec_slot, 80 | NGX_HTTP_LOC_CONF_OFFSET, 81 | offsetof(ngx_http_redis2_loc_conf_t, upstream.connect_timeout), 82 | NULL }, 83 | 84 | { ngx_string("redis2_send_timeout"), 85 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 86 | ngx_conf_set_msec_slot, 87 | NGX_HTTP_LOC_CONF_OFFSET, 88 | offsetof(ngx_http_redis2_loc_conf_t, upstream.send_timeout), 89 | NULL }, 90 | 91 | { ngx_string("redis2_buffer_size"), 92 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 93 | ngx_conf_set_size_slot, 94 | NGX_HTTP_LOC_CONF_OFFSET, 95 | offsetof(ngx_http_redis2_loc_conf_t, upstream.buffer_size), 96 | NULL }, 97 | 98 | { ngx_string("redis2_read_timeout"), 99 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, 100 | ngx_conf_set_msec_slot, 101 | NGX_HTTP_LOC_CONF_OFFSET, 102 | offsetof(ngx_http_redis2_loc_conf_t, upstream.read_timeout), 103 | NULL }, 104 | 105 | { ngx_string("redis2_next_upstream"), 106 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE, 107 | ngx_conf_set_bitmask_slot, 108 | NGX_HTTP_LOC_CONF_OFFSET, 109 | offsetof(ngx_http_redis2_loc_conf_t, upstream.next_upstream), 110 | &ngx_http_redis2_next_upstream_masks }, 111 | 112 | ngx_null_command 113 | }; 114 | 115 | 116 | static ngx_http_module_t ngx_http_redis2_module_ctx = { 117 | NULL, /* preconfiguration */ 118 | NULL, /* postconfiguration */ 119 | 120 | NULL, /* create main configuration */ 121 | NULL, /* init main configuration */ 122 | 123 | NULL, /* create server configuration */ 124 | NULL, /* merge server configuration */ 125 | 126 | ngx_http_redis2_create_loc_conf, /* create location configration */ 127 | ngx_http_redis2_merge_loc_conf /* merge location configration */ 128 | }; 129 | 130 | 131 | ngx_module_t ngx_http_redis2_module = { 132 | NGX_MODULE_V1, 133 | &ngx_http_redis2_module_ctx, /* module context */ 134 | ngx_http_redis2_commands, /* module directives */ 135 | NGX_HTTP_MODULE, /* module type */ 136 | NULL, /* init master */ 137 | NULL, /* init module */ 138 | NULL, /* init process */ 139 | NULL, /* init thread */ 140 | NULL, /* exit thread */ 141 | NULL, /* exit process */ 142 | NULL, /* exit master */ 143 | NGX_MODULE_V1_PADDING 144 | }; 145 | 146 | 147 | static void * 148 | ngx_http_redis2_create_loc_conf(ngx_conf_t *cf) 149 | { 150 | ngx_http_redis2_loc_conf_t *conf; 151 | 152 | conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_redis2_loc_conf_t)); 153 | if (conf == NULL) { 154 | return NULL; 155 | } 156 | 157 | /* 158 | * set by ngx_pcalloc(): 159 | * 160 | * conf->upstream.bufs.num = 0; 161 | * conf->upstream.next_upstream = 0; 162 | * conf->upstream.temp_path = NULL; 163 | * conf->upstream.uri = { 0, NULL }; 164 | * conf->upstream.location = NULL; 165 | * conf->complex_query = NULL; 166 | * conf->literal_query = { 0, NULL }; 167 | * conf->queries = NULL; 168 | */ 169 | 170 | conf->upstream.connect_timeout = NGX_CONF_UNSET_MSEC; 171 | conf->upstream.send_timeout = NGX_CONF_UNSET_MSEC; 172 | conf->upstream.read_timeout = NGX_CONF_UNSET_MSEC; 173 | 174 | conf->upstream.buffer_size = NGX_CONF_UNSET_SIZE; 175 | 176 | /* the hardcoded values */ 177 | conf->upstream.cyclic_temp_file = 0; 178 | conf->upstream.buffering = 0; 179 | conf->upstream.ignore_client_abort = 1; 180 | conf->upstream.send_lowat = 0; 181 | conf->upstream.bufs.num = 0; 182 | conf->upstream.busy_buffers_size = 0; 183 | conf->upstream.max_temp_file_size = 0; 184 | conf->upstream.temp_file_write_size = 0; 185 | conf->upstream.intercept_errors = 1; 186 | conf->upstream.intercept_404 = 1; 187 | conf->upstream.pass_request_headers = 0; 188 | conf->upstream.pass_request_body = 0; 189 | 190 | return conf; 191 | } 192 | 193 | 194 | static char * 195 | ngx_http_redis2_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) 196 | { 197 | ngx_http_redis2_loc_conf_t *prev = parent; 198 | ngx_http_redis2_loc_conf_t *conf = child; 199 | 200 | ngx_conf_merge_msec_value(conf->upstream.connect_timeout, 201 | prev->upstream.connect_timeout, 60000); 202 | 203 | ngx_conf_merge_msec_value(conf->upstream.send_timeout, 204 | prev->upstream.send_timeout, 60000); 205 | 206 | ngx_conf_merge_msec_value(conf->upstream.read_timeout, 207 | prev->upstream.read_timeout, 60000); 208 | 209 | ngx_conf_merge_size_value(conf->upstream.buffer_size, 210 | prev->upstream.buffer_size, 211 | (size_t) ngx_pagesize); 212 | 213 | ngx_conf_merge_bitmask_value(conf->upstream.next_upstream, 214 | prev->upstream.next_upstream, 215 | (NGX_CONF_BITMASK_SET 216 | |NGX_HTTP_UPSTREAM_FT_ERROR 217 | |NGX_HTTP_UPSTREAM_FT_TIMEOUT)); 218 | 219 | if (conf->upstream.next_upstream & NGX_HTTP_UPSTREAM_FT_OFF) { 220 | conf->upstream.next_upstream = NGX_CONF_BITMASK_SET 221 | |NGX_HTTP_UPSTREAM_FT_OFF; 222 | } 223 | 224 | if (conf->upstream.upstream == NULL) { 225 | conf->upstream.upstream = prev->upstream.upstream; 226 | } 227 | 228 | if (conf->complex_query == NULL) { 229 | conf->complex_query = prev->complex_query; 230 | } 231 | 232 | if (conf->complex_query_count == NULL) { 233 | conf->complex_query_count = prev->complex_query_count; 234 | } 235 | 236 | if (conf->queries == NULL) { 237 | conf->queries = prev->queries; 238 | } 239 | 240 | if (conf->literal_query.data == NULL) { 241 | conf->literal_query.data = prev->literal_query.data; 242 | conf->literal_query.len = prev->literal_query.len; 243 | } 244 | 245 | return NGX_CONF_OK; 246 | } 247 | 248 | 249 | static char * 250 | ngx_http_redis2_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 251 | { 252 | ngx_http_redis2_loc_conf_t *rlcf = conf; 253 | 254 | ngx_str_t *value; 255 | ngx_http_core_loc_conf_t *clcf; 256 | ngx_uint_t n; 257 | ngx_url_t url; 258 | 259 | ngx_http_compile_complex_value_t ccv; 260 | 261 | if (rlcf->upstream.upstream) { 262 | return "is duplicate"; 263 | } 264 | 265 | clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module); 266 | 267 | clcf->handler = ngx_http_redis2_handler; 268 | 269 | if (clcf->name.data[clcf->name.len - 1] == '/') { 270 | clcf->auto_redirect = 1; 271 | } 272 | 273 | value = cf->args->elts; 274 | 275 | n = ngx_http_script_variables_count(&value[1]); 276 | if (n) { 277 | rlcf->complex_target = ngx_palloc(cf->pool, 278 | sizeof(ngx_http_complex_value_t)); 279 | 280 | if (rlcf->complex_target == NULL) { 281 | return NGX_CONF_ERROR; 282 | } 283 | 284 | ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); 285 | ccv.cf = cf; 286 | ccv.value = &value[1]; 287 | ccv.complex_value = rlcf->complex_target; 288 | 289 | if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { 290 | return NGX_CONF_ERROR; 291 | } 292 | 293 | return NGX_CONF_OK; 294 | } 295 | 296 | rlcf->complex_target = NULL; 297 | 298 | ngx_memzero(&url, sizeof(ngx_url_t)); 299 | 300 | url.url = value[1]; 301 | url.no_resolve = 1; 302 | 303 | rlcf->upstream.upstream = ngx_http_upstream_add(cf, &url, 0); 304 | if (rlcf->upstream.upstream == NULL) { 305 | return NGX_CONF_ERROR; 306 | } 307 | 308 | return NGX_CONF_OK; 309 | } 310 | 311 | 312 | static char * 313 | ngx_http_redis2_query(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 314 | { 315 | ngx_http_redis2_loc_conf_t *rlcf = conf; 316 | ngx_str_t *value; 317 | ngx_array_t **query; 318 | ngx_uint_t n; 319 | ngx_http_complex_value_t **arg; 320 | ngx_uint_t i; 321 | 322 | ngx_http_compile_complex_value_t ccv; 323 | 324 | if (rlcf->literal_query.len) { 325 | return "conflicts with redis2_literal_raw_query"; 326 | } 327 | 328 | if (rlcf->complex_query) { 329 | return "conflicts with redis2_raw_query"; 330 | } 331 | 332 | if (rlcf->queries == NULL) { 333 | rlcf->queries = ngx_array_create(cf->pool, 1, sizeof(ngx_array_t *)); 334 | 335 | if (rlcf->queries == NULL) { 336 | return NGX_CONF_ERROR; 337 | } 338 | } 339 | 340 | query = ngx_array_push(rlcf->queries); 341 | if (query == NULL) { 342 | return NGX_CONF_ERROR; 343 | } 344 | 345 | n = cf->args->nelts - 1; 346 | 347 | *query = ngx_array_create(cf->pool, n, sizeof(ngx_http_complex_value_t *)); 348 | 349 | if (*query == NULL) { 350 | return NGX_CONF_ERROR; 351 | } 352 | 353 | value = cf->args->elts; 354 | 355 | for (i = 1; i <= n; i++) { 356 | arg = ngx_array_push(*query); 357 | if (arg == NULL) { 358 | return NGX_CONF_ERROR; 359 | } 360 | 361 | *arg = ngx_palloc(cf->pool, sizeof(ngx_http_complex_value_t)); 362 | if (*arg == NULL) { 363 | return NGX_CONF_ERROR; 364 | } 365 | 366 | if (value[i].len == 0) { 367 | ngx_memzero(*arg, sizeof(ngx_http_complex_value_t)); 368 | continue; 369 | } 370 | 371 | ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); 372 | ccv.cf = cf; 373 | ccv.value = &value[i]; 374 | ccv.complex_value = *arg; 375 | 376 | if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { 377 | return NGX_CONF_ERROR; 378 | } 379 | } 380 | 381 | return NGX_CONF_OK; 382 | } 383 | 384 | 385 | static char * 386 | ngx_http_redis2_raw_queries(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 387 | { 388 | ngx_http_redis2_loc_conf_t *rlcf = conf; 389 | ngx_str_t *value; 390 | 391 | ngx_http_compile_complex_value_t ccv; 392 | 393 | value = cf->args->elts; 394 | 395 | /* compile the N argument */ 396 | 397 | rlcf->complex_query_count = ngx_palloc(cf->pool, 398 | sizeof(ngx_http_complex_value_t)); 399 | 400 | if (rlcf->complex_query_count == NULL) { 401 | return NGX_CONF_ERROR; 402 | } 403 | 404 | ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); 405 | ccv.cf = cf; 406 | ccv.value = &value[1]; 407 | ccv.complex_value = rlcf->complex_query_count; 408 | 409 | if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { 410 | return NGX_CONF_ERROR; 411 | } 412 | 413 | /* compile the CMDS argument */ 414 | 415 | rlcf->complex_query = ngx_palloc(cf->pool, 416 | sizeof(ngx_http_complex_value_t)); 417 | 418 | if (rlcf->complex_query == NULL) { 419 | return NGX_CONF_ERROR; 420 | } 421 | 422 | ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); 423 | ccv.cf = cf; 424 | ccv.value = &value[2]; 425 | ccv.complex_value = rlcf->complex_query; 426 | 427 | if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { 428 | return NGX_CONF_ERROR; 429 | } 430 | 431 | return NGX_CONF_OK; 432 | } 433 | 434 | -------------------------------------------------------------------------------- /src/ngx_http_redis2_reply.c: -------------------------------------------------------------------------------- 1 | 2 | #line 1 "src/ngx_http_redis2_reply.rl" 3 | #ifndef DDEBUG 4 | #define DDEBUG 0 5 | #endif 6 | #include "ddebug.h" 7 | 8 | #include "ngx_http_redis2_reply.h" 9 | #include "ngx_http_redis2_util.h" 10 | #include 11 | 12 | 13 | #line 20 "src/ngx_http_redis2_reply.rl" 14 | 15 | 16 | 17 | #line 18 "src/ngx_http_redis2_reply.c" 18 | static const unsigned char _reply_cond_offsets[] = { 19 | 0, 0, 0, 0, 0, 0, 0, 0, 20 | 0, 0, 0, 0, 1, 4, 7, 7, 21 | 7, 7, 7, 7, 7, 7, 11, 14, 22 | 15, 17, 18, 21, 26, 28, 29, 30, 23 | 32, 33, 34, 37, 42, 50, 55, 60, 24 | 65, 70, 73, 74, 79, 82, 87, 92, 25 | 97, 98, 103, 103, 103, 103, 106, 110, 26 | 121, 126, 137 27 | }; 28 | 29 | static const char _reply_cond_lengths[] = { 30 | 0, 0, 0, 0, 0, 0, 0, 0, 31 | 0, 0, 0, 1, 3, 3, 0, 0, 32 | 0, 0, 0, 0, 0, 4, 3, 1, 33 | 2, 1, 3, 5, 2, 1, 1, 2, 34 | 1, 1, 3, 5, 8, 5, 5, 5, 35 | 5, 3, 1, 5, 3, 5, 5, 5, 36 | 1, 5, 0, 0, 0, 3, 4, 11, 37 | 5, 11, 9 38 | }; 39 | 40 | static const short _reply_cond_keys[] = { 41 | -128, 127, -128, 12, 13, 13, 14, 127, 42 | -128, 12, 13, 13, 14, 127, 36, 36, 43 | 43, 43, 45, 45, 58, 58, 45, 45, 44 | 48, 48, 49, 57, 48, 57, 13, 13, 45 | 48, 57, 10, 10, -128, 12, 13, 13, 46 | 14, 127, -128, 9, 10, 10, 11, 12, 47 | 13, 13, 14, 127, 13, 13, 48, 48, 48 | 10, 10, 13, 13, 13, 13, 48, 57, 49 | 10, 10, -128, 127, -128, 12, 13, 13, 50 | 14, 127, -128, 9, 10, 10, 11, 12, 51 | 13, 13, 14, 127, -128, 12, 13, 13, 52 | 14, 44, 45, 45, 46, 47, 48, 48, 53 | 49, 57, 58, 127, -128, 12, 13, 13, 54 | 14, 47, 48, 57, 58, 127, -128, 12, 55 | 13, 13, 14, 47, 48, 57, 58, 127, 56 | -128, 12, 13, 13, 14, 47, 48, 48, 57 | 49, 127, -128, 9, 10, 10, 11, 12, 58 | 13, 13, 14, 127, -128, 12, 13, 13, 59 | 14, 127, 10, 10, -128, 9, 10, 10, 60 | 11, 12, 13, 13, 14, 127, -128, 12, 61 | 13, 13, 14, 127, -128, 9, 10, 10, 62 | 11, 12, 13, 13, 14, 127, -128, 12, 63 | 13, 13, 14, 47, 48, 57, 58, 127, 64 | -128, 9, 10, 10, 11, 12, 13, 13, 65 | 14, 127, 10, 10, -128, 9, 10, 10, 66 | 11, 12, 13, 13, 14, 127, -128, 12, 67 | 13, 13, 14, 127, 36, 36, 43, 43, 68 | 45, 45, 58, 58, -128, 12, 13, 13, 69 | 14, 35, 36, 36, 37, 42, 43, 43, 70 | 44, 44, 45, 45, 46, 57, 58, 58, 71 | 59, 127, 13, 13, 36, 36, 43, 43, 72 | 45, 45, 58, 58, -128, 12, 13, 13, 73 | 14, 35, 36, 36, 37, 42, 43, 43, 74 | 44, 44, 45, 45, 46, 57, 58, 58, 75 | 59, 127, -128, 35, 36, 36, 37, 42, 76 | 43, 43, 44, 44, 45, 45, 46, 57, 77 | 58, 58, 59, 127, 0 78 | }; 79 | 80 | static const char _reply_cond_spaces[] = { 81 | 0, 0, 3, 0, 0, 3, 0, 2, 82 | 2, 2, 2, 2, 2, 2, 2, 2, 83 | 2, 2, 2, 2, 2, 2, 2, 2, 84 | 2, 2, 2, 2, 2, 2, 2, 2, 85 | 2, 4, 4, 5, 4, 4, 4, 4, 86 | 5, 4, 4, 5, 4, 4, 4, 4, 87 | 4, 4, 4, 5, 4, 4, 4, 4, 88 | 5, 4, 4, 4, 4, 5, 4, 4, 89 | 4, 4, 4, 4, 5, 4, 4, 5, 90 | 4, 2, 4, 4, 4, 5, 4, 4, 91 | 5, 4, 4, 4, 4, 5, 4, 4, 92 | 5, 4, 4, 4, 4, 4, 4, 5, 93 | 4, 2, 4, 4, 4, 5, 4, 0, 94 | 3, 0, 2, 2, 2, 2, 4, 5, 95 | 4, 4, 4, 4, 4, 4, 4, 4, 96 | 4, 2, 2, 2, 2, 2, 4, 5, 97 | 4, 4, 4, 4, 4, 4, 4, 4, 98 | 4, 4, 4, 4, 4, 4, 4, 4, 99 | 4, 4, 0 100 | }; 101 | 102 | static const short _reply_key_offsets[] = { 103 | 0, 0, 5, 9, 11, 14, 15, 17, 104 | 18, 19, 22, 23, 25, 32, 41, 45, 105 | 46, 47, 48, 50, 53, 54, 58, 62, 106 | 64, 67, 68, 71, 75, 77, 78, 79, 107 | 82, 83, 85, 92, 101, 118, 131, 145, 108 | 155, 164, 172, 173, 182, 194, 208, 222, 109 | 230, 231, 240, 241, 243, 243, 250, 254, 110 | 269, 274, 290 111 | }; 112 | 113 | static const short _reply_trans_keys[] = { 114 | 36, 42, 43, 45, 58, 45, 48, 49, 115 | 57, 48, 57, 13, 48, 57, 10, 13, 116 | 48, 10, 13, 13, 48, 57, 10, 384, 117 | 639, 1549, 1805, 2061, 384, 524, 526, 639, 118 | 266, 522, 1549, 1805, 2061, 384, 524, 526, 119 | 639, 45, 48, 49, 57, 49, 13, 10, 120 | 13, 48, 13, 48, 57, 10, 5668, 5675, 121 | 5677, 5690, 5677, 5680, 5681, 5689, 5680, 5689, 122 | 5645, 5680, 5689, 5642, 5645, 5504, 5759, 5642, 123 | 5645, 5504, 5759, 5645, 5680, 5642, 5645, 5645, 124 | 5680, 5689, 5642, 2944, 3199, 4621, 4877, 5133, 125 | 2944, 3084, 3086, 3199, 2826, 3082, 4621, 4877, 126 | 5133, 2944, 3084, 3086, 3199, 2861, 2864, 3117, 127 | 3120, 4621, 4877, 5133, 2865, 2873, 2944, 3084, 128 | 3086, 3119, 3121, 3129, 3130, 3199, 4621, 4877, 129 | 5133, 2864, 2873, 2944, 3084, 3086, 3119, 3120, 130 | 3129, 3130, 3199, 4365, 4621, 4877, 5133, 2864, 131 | 2873, 2944, 3084, 3086, 3119, 3120, 3129, 3130, 132 | 3199, 2864, 3120, 4365, 4621, 4877, 5133, 2944, 133 | 3084, 3086, 3199, 2826, 3082, 4621, 4877, 5133, 134 | 2944, 3084, 3086, 3199, 4365, 4621, 4877, 5133, 135 | 2944, 3084, 3086, 3199, 5642, 2826, 3082, 4621, 136 | 4877, 5133, 2944, 3084, 3086, 3199, 4365, 4621, 137 | 4877, 5133, 2688, 2828, 2830, 2943, 2944, 3084, 138 | 3086, 3199, 2826, 3082, 4365, 4621, 4877, 5133, 139 | 2688, 2828, 2830, 2943, 2944, 3084, 3086, 3199, 140 | 4365, 4621, 4877, 5133, 2864, 2873, 2944, 3084, 141 | 3086, 3119, 3120, 3129, 3130, 3199, 2826, 4621, 142 | 4877, 5133, 2944, 3084, 3086, 3199, 5642, 2826, 143 | 3082, 4621, 4877, 5133, 2944, 3084, 3086, 3199, 144 | 13, 10, 13, 1549, 1805, 2061, 384, 524, 145 | 526, 639, 5668, 5675, 5677, 5690, 2852, 2859, 146 | 2861, 2874, 3108, 3115, 3117, 3130, 4621, 4877, 147 | 5133, 2944, 3084, 3086, 3199, 5645, 5668, 5675, 148 | 5677, 5690, 2852, 2859, 2861, 2874, 3108, 3115, 149 | 3117, 3130, 4365, 4621, 4877, 5133, 2944, 3084, 150 | 3086, 3199, 2852, 2859, 2861, 2874, 3108, 3115, 151 | 3117, 3130, 2944, 3199, 0 152 | }; 153 | 154 | static const char _reply_single_lengths[] = { 155 | 0, 5, 2, 0, 1, 1, 2, 1, 156 | 1, 1, 1, 0, 3, 5, 2, 1, 157 | 1, 1, 2, 1, 1, 4, 2, 0, 158 | 1, 1, 1, 2, 2, 1, 1, 1, 159 | 1, 0, 3, 5, 7, 3, 4, 6, 160 | 5, 4, 1, 5, 4, 6, 4, 4, 161 | 1, 5, 1, 2, 0, 3, 4, 11, 162 | 5, 12, 8 163 | }; 164 | 165 | static const char _reply_range_lengths[] = { 166 | 0, 0, 1, 1, 1, 0, 0, 0, 167 | 0, 1, 0, 1, 2, 2, 1, 0, 168 | 0, 0, 0, 1, 0, 0, 1, 1, 169 | 1, 0, 1, 1, 0, 0, 0, 1, 170 | 0, 1, 2, 2, 5, 5, 5, 2, 171 | 2, 2, 0, 2, 4, 4, 5, 2, 172 | 0, 2, 0, 0, 0, 2, 0, 2, 173 | 0, 2, 1 174 | }; 175 | 176 | static const short _reply_index_offsets[] = { 177 | 0, 0, 6, 10, 12, 15, 17, 20, 178 | 22, 24, 27, 29, 31, 37, 45, 49, 179 | 51, 53, 55, 58, 61, 63, 68, 72, 180 | 74, 77, 79, 82, 86, 89, 91, 93, 181 | 96, 98, 100, 106, 114, 127, 136, 146, 182 | 155, 163, 170, 172, 180, 189, 200, 210, 183 | 217, 219, 227, 229, 232, 233, 239, 244, 184 | 258, 264, 279 185 | }; 186 | 187 | static const char _reply_indicies[] = { 188 | 0, 2, 3, 3, 3, 1, 4, 5, 189 | 6, 1, 7, 1, 8, 7, 1, 9, 190 | 1, 10, 5, 1, 11, 1, 8, 1, 191 | 12, 13, 1, 14, 1, 15, 1, 15, 192 | 8, 16, 15, 15, 1, 9, 17, 15, 193 | 8, 16, 15, 15, 1, 18, 19, 20, 194 | 1, 21, 1, 22, 1, 23, 1, 22, 195 | 19, 1, 24, 25, 1, 26, 1, 27, 196 | 28, 28, 28, 1, 29, 30, 31, 1, 197 | 32, 1, 33, 32, 1, 34, 1, 35, 198 | 28, 1, 34, 35, 28, 1, 36, 30, 199 | 1, 37, 1, 33, 1, 38, 39, 1, 200 | 40, 1, 41, 1, 41, 33, 42, 41, 201 | 41, 1, 34, 43, 41, 33, 42, 41, 202 | 41, 1, 29, 30, 44, 45, 41, 33, 203 | 42, 31, 41, 41, 46, 41, 1, 41, 204 | 33, 42, 32, 41, 41, 47, 41, 1, 205 | 33, 42, 33, 42, 32, 41, 41, 47, 206 | 41, 1, 30, 45, 36, 48, 49, 50, 207 | 41, 41, 1, 37, 51, 41, 33, 42, 208 | 41, 41, 1, 33, 42, 33, 42, 41, 209 | 41, 1, 52, 1, 52, 53, 41, 33, 210 | 42, 41, 41, 1, 35, 55, 35, 55, 211 | 28, 28, 54, 54, 1, 34, 43, 35, 212 | 55, 35, 55, 28, 28, 54, 54, 1, 213 | 38, 57, 58, 59, 39, 41, 41, 56, 214 | 41, 1, 40, 41, 33, 42, 41, 41, 215 | 1, 60, 1, 60, 43, 41, 33, 42, 216 | 41, 41, 1, 61, 3, 62, 61, 3, 217 | 1, 15, 8, 16, 15, 15, 1, 27, 218 | 28, 28, 28, 1, 27, 28, 28, 28, 219 | 63, 54, 54, 54, 41, 33, 42, 41, 220 | 41, 1, 33, 27, 28, 28, 28, 1, 221 | 27, 28, 28, 28, 63, 54, 54, 54, 222 | 33, 42, 33, 42, 41, 41, 1, 27, 223 | 28, 28, 28, 63, 54, 54, 54, 41, 224 | 1, 0 225 | }; 226 | 227 | static const char _reply_trans_targs[] = { 228 | 2, 0, 14, 50, 3, 6, 9, 4, 229 | 5, 52, 7, 8, 10, 9, 11, 12, 230 | 13, 53, 15, 18, 19, 16, 17, 52, 231 | 20, 19, 21, 22, 26, 23, 28, 31, 232 | 24, 25, 54, 27, 29, 30, 32, 31, 233 | 33, 34, 35, 55, 37, 39, 46, 38, 234 | 40, 42, 43, 41, 56, 57, 44, 45, 235 | 46, 47, 48, 49, 58, 51, 52, 36 236 | }; 237 | 238 | static const char _reply_trans_actions[] = { 239 | 0, 0, 0, 0, 0, 0, 1, 0, 240 | 0, 2, 0, 0, 0, 3, 0, 0, 241 | 0, 2, 0, 0, 4, 0, 0, 5, 242 | 0, 6, 7, 0, 0, 0, 0, 1, 243 | 0, 0, 8, 0, 0, 0, 0, 3, 244 | 0, 0, 0, 8, 0, 0, 1, 0, 245 | 0, 0, 0, 0, 8, 8, 0, 0, 246 | 3, 0, 0, 0, 8, 0, 9, 0 247 | }; 248 | 249 | static const int reply_start = 1; 250 | static const int reply_error = 0; 251 | 252 | 253 | 254 | #line 23 "src/ngx_http_redis2_reply.rl" 255 | 256 | ngx_int_t 257 | ngx_http_redis2_process_reply(ngx_http_redis2_ctx_t *ctx, ssize_t bytes) 258 | { 259 | ngx_buf_t *b; 260 | ngx_http_upstream_t *u; 261 | ngx_str_t buf; 262 | ngx_flag_t done; 263 | ngx_chain_t *cl = NULL; 264 | ngx_chain_t **ll = NULL; 265 | 266 | int cs; 267 | signed char *p; 268 | signed char *orig_p; 269 | ssize_t orig_len; 270 | signed char *pe; 271 | 272 | u = ctx->request->upstream; 273 | b = &u->buffer; 274 | 275 | orig_p = (signed char *) b->last; 276 | orig_len = bytes; 277 | 278 | while (ctx->query_count) { 279 | done = 0; 280 | 281 | if (ctx->state == NGX_ERROR) { 282 | dd("init the state machine"); 283 | 284 | 285 | #line 288 "src/ngx_http_redis2_reply.c" 286 | { 287 | cs = reply_start; 288 | } 289 | 290 | #line 53 "src/ngx_http_redis2_reply.rl" 291 | 292 | ctx->state = cs; 293 | 294 | } else { 295 | cs = ctx->state; 296 | dd("resumed the old state %d", cs); 297 | } 298 | 299 | p = (signed char *) b->last; 300 | pe = (signed char *) b->last + bytes; 301 | 302 | dd("response body: %.*s", (int) bytes, p); 303 | 304 | 305 | #line 308 "src/ngx_http_redis2_reply.c" 306 | { 307 | int _klen; 308 | const short *_keys; 309 | int _trans; 310 | short _widec; 311 | 312 | if ( p == pe ) 313 | goto _test_eof; 314 | if ( cs == 0 ) 315 | goto _out; 316 | _resume: 317 | _widec = (*p); 318 | _klen = _reply_cond_lengths[cs]; 319 | _keys = _reply_cond_keys + (_reply_cond_offsets[cs]*2); 320 | if ( _klen > 0 ) { 321 | const short *_lower = _keys; 322 | const short *_mid; 323 | const short *_upper = _keys + (_klen<<1) - 2; 324 | while (1) { 325 | if ( _upper < _lower ) 326 | break; 327 | 328 | _mid = _lower + (((_upper-_lower) >> 1) & ~1); 329 | if ( _widec < _mid[0] ) 330 | _upper = _mid - 2; 331 | else if ( _widec > _mid[1] ) 332 | _lower = _mid + 2; 333 | else { 334 | switch ( _reply_cond_spaces[_reply_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { 335 | case 0: { 336 | _widec = (short)(128 + ((*p) - -128)); 337 | if ( 338 | #line 34 "src/common.rl" 339 | 340 | #if 0 341 | fprintf(stderr, "test chunk len: %d < %d\n", 342 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 343 | #endif 344 | ctx->chunk_bytes_read++ < ctx->chunk_size 345 | ) _widec += 256; 346 | break; 347 | } 348 | case 1: { 349 | _widec = (short)(640 + ((*p) - -128)); 350 | if ( 351 | #line 56 "src/common.rl" 352 | 353 | #if 0 354 | fprintf(stderr, 355 | "check_data_complete: chunk bytes read: %d, chunk size: %d\n", 356 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 357 | #endif 358 | ctx->chunk_bytes_read == ctx->chunk_size + 1 359 | ) _widec += 256; 360 | break; 361 | } 362 | case 2: { 363 | _widec = (short)(5248 + ((*p) - -128)); 364 | if ( 365 | #line 6 "src/multi_bulk_reply.rl" 366 | 367 | #if 0 368 | fprintf(stderr, "test chunk count: %d < %d\n", 369 | (int) ctx->chunks_read, (int) ctx->chunk_count), 370 | #endif 371 | ctx->chunks_read < ctx->chunk_count 372 | ) _widec += 256; 373 | break; 374 | } 375 | case 3: { 376 | _widec = (short)(1152 + ((*p) - -128)); 377 | if ( 378 | #line 34 "src/common.rl" 379 | 380 | #if 0 381 | fprintf(stderr, "test chunk len: %d < %d\n", 382 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 383 | #endif 384 | ctx->chunk_bytes_read++ < ctx->chunk_size 385 | ) _widec += 256; 386 | if ( 387 | #line 56 "src/common.rl" 388 | 389 | #if 0 390 | fprintf(stderr, 391 | "check_data_complete: chunk bytes read: %d, chunk size: %d\n", 392 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 393 | #endif 394 | ctx->chunk_bytes_read == ctx->chunk_size + 1 395 | ) _widec += 512; 396 | break; 397 | } 398 | case 4: { 399 | _widec = (short)(2176 + ((*p) - -128)); 400 | if ( 401 | #line 34 "src/common.rl" 402 | 403 | #if 0 404 | fprintf(stderr, "test chunk len: %d < %d\n", 405 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 406 | #endif 407 | ctx->chunk_bytes_read++ < ctx->chunk_size 408 | ) _widec += 256; 409 | if ( 410 | #line 6 "src/multi_bulk_reply.rl" 411 | 412 | #if 0 413 | fprintf(stderr, "test chunk count: %d < %d\n", 414 | (int) ctx->chunks_read, (int) ctx->chunk_count), 415 | #endif 416 | ctx->chunks_read < ctx->chunk_count 417 | ) _widec += 512; 418 | break; 419 | } 420 | case 5: { 421 | _widec = (short)(3200 + ((*p) - -128)); 422 | if ( 423 | #line 34 "src/common.rl" 424 | 425 | #if 0 426 | fprintf(stderr, "test chunk len: %d < %d\n", 427 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 428 | #endif 429 | ctx->chunk_bytes_read++ < ctx->chunk_size 430 | ) _widec += 256; 431 | if ( 432 | #line 56 "src/common.rl" 433 | 434 | #if 0 435 | fprintf(stderr, 436 | "check_data_complete: chunk bytes read: %d, chunk size: %d\n", 437 | (int) ctx->chunk_bytes_read, (int) ctx->chunk_size), 438 | #endif 439 | ctx->chunk_bytes_read == ctx->chunk_size + 1 440 | ) _widec += 512; 441 | if ( 442 | #line 6 "src/multi_bulk_reply.rl" 443 | 444 | #if 0 445 | fprintf(stderr, "test chunk count: %d < %d\n", 446 | (int) ctx->chunks_read, (int) ctx->chunk_count), 447 | #endif 448 | ctx->chunks_read < ctx->chunk_count 449 | ) _widec += 1024; 450 | break; 451 | } 452 | } 453 | break; 454 | } 455 | } 456 | } 457 | 458 | _keys = _reply_trans_keys + _reply_key_offsets[cs]; 459 | _trans = _reply_index_offsets[cs]; 460 | 461 | _klen = _reply_single_lengths[cs]; 462 | if ( _klen > 0 ) { 463 | const short *_lower = _keys; 464 | const short *_mid; 465 | const short *_upper = _keys + _klen - 1; 466 | while (1) { 467 | if ( _upper < _lower ) 468 | break; 469 | 470 | _mid = _lower + ((_upper-_lower) >> 1); 471 | if ( _widec < *_mid ) 472 | _upper = _mid - 1; 473 | else if ( _widec > *_mid ) 474 | _lower = _mid + 1; 475 | else { 476 | _trans += (unsigned int)(_mid - _keys); 477 | goto _match; 478 | } 479 | } 480 | _keys += _klen; 481 | _trans += _klen; 482 | } 483 | 484 | _klen = _reply_range_lengths[cs]; 485 | if ( _klen > 0 ) { 486 | const short *_lower = _keys; 487 | const short *_mid; 488 | const short *_upper = _keys + (_klen<<1) - 2; 489 | while (1) { 490 | if ( _upper < _lower ) 491 | break; 492 | 493 | _mid = _lower + (((_upper-_lower) >> 1) & ~1); 494 | if ( _widec < _mid[0] ) 495 | _upper = _mid - 2; 496 | else if ( _widec > _mid[1] ) 497 | _lower = _mid + 2; 498 | else { 499 | _trans += (unsigned int)((_mid - _keys)>>1); 500 | goto _match; 501 | } 502 | } 503 | _trans += _klen; 504 | } 505 | 506 | _match: 507 | _trans = _reply_indicies[_trans]; 508 | cs = _reply_trans_targs[_trans]; 509 | 510 | if ( _reply_trans_actions[_trans] == 0 ) 511 | goto _again; 512 | 513 | switch ( _reply_trans_actions[_trans] ) { 514 | case 9: 515 | #line 12 "src/common.rl" 516 | { 517 | dd("done!"); 518 | done = 1; 519 | } 520 | break; 521 | case 3: 522 | #line 17 "src/common.rl" 523 | { 524 | ctx->chunk_size *= 10; 525 | ctx->chunk_size += *p - '0'; 526 | dd("read chunk size: %d", (int) ctx->chunk_size); 527 | } 528 | break; 529 | case 7: 530 | #line 14 "src/multi_bulk_reply.rl" 531 | { 532 | dd("start reading bulk"); 533 | ctx->chunks_read = 0; 534 | } 535 | break; 536 | case 6: 537 | #line 24 "src/multi_bulk_reply.rl" 538 | { 539 | ctx->chunk_count *= 10; 540 | ctx->chunk_count += *p - '0'; 541 | dd("chunk count: %d", (int) ctx->chunk_count); 542 | } 543 | break; 544 | case 5: 545 | #line 30 "src/multi_bulk_reply.rl" 546 | { 547 | dd("finalize multi bulks"); 548 | 549 | if (ctx->chunks_read == ctx->chunk_count) { 550 | dd("done multi bunlk reading!"); 551 | done = 1; 552 | } 553 | } 554 | break; 555 | case 1: 556 | #line 23 "src/common.rl" 557 | { 558 | dd("start reading chunk size"); 559 | ctx->chunk_bytes_read = 0; 560 | ctx->chunk_size = 0; 561 | } 562 | #line 17 "src/common.rl" 563 | { 564 | ctx->chunk_size *= 10; 565 | ctx->chunk_size += *p - '0'; 566 | dd("read chunk size: %d", (int) ctx->chunk_size); 567 | } 568 | break; 569 | case 2: 570 | #line 50 "src/common.rl" 571 | { 572 | ctx->chunks_read++; 573 | dd("have read chunk %d, %.*s", (int) ctx->chunks_read, 574 | (int) (p - (signed char *) b->last), (signed char *) b->last); 575 | } 576 | #line 12 "src/common.rl" 577 | { 578 | dd("done!"); 579 | done = 1; 580 | } 581 | break; 582 | case 8: 583 | #line 50 "src/common.rl" 584 | { 585 | ctx->chunks_read++; 586 | dd("have read chunk %d, %.*s", (int) ctx->chunks_read, 587 | (int) (p - (signed char *) b->last), (signed char *) b->last); 588 | } 589 | #line 30 "src/multi_bulk_reply.rl" 590 | { 591 | dd("finalize multi bulks"); 592 | 593 | if (ctx->chunks_read == ctx->chunk_count) { 594 | dd("done multi bunlk reading!"); 595 | done = 1; 596 | } 597 | } 598 | break; 599 | case 4: 600 | #line 19 "src/multi_bulk_reply.rl" 601 | { 602 | dd("start reading bulk count"); 603 | ctx->chunk_count = 0; 604 | } 605 | #line 24 "src/multi_bulk_reply.rl" 606 | { 607 | ctx->chunk_count *= 10; 608 | ctx->chunk_count += *p - '0'; 609 | dd("chunk count: %d", (int) ctx->chunk_count); 610 | } 611 | break; 612 | #line 615 "src/ngx_http_redis2_reply.c" 613 | } 614 | 615 | _again: 616 | if ( cs == 0 ) 617 | goto _out; 618 | if ( ++p != pe ) 619 | goto _resume; 620 | _test_eof: {} 621 | _out: {} 622 | } 623 | 624 | #line 67 "src/ngx_http_redis2_reply.rl" 625 | 626 | dd("state after exec: %d, done: %d, %.*s", cs, (int) done, 627 | (int) (bytes - ((u_char *) p - b->last)), p); 628 | 629 | ctx->state = cs; 630 | 631 | if (!done && cs == reply_error) { 632 | if (cl) { 633 | cl->buf->last = cl->buf->pos; 634 | cl = NULL; 635 | *ll = NULL; 636 | } 637 | 638 | buf.data = b->pos; 639 | buf.len = b->last - b->pos + bytes; 640 | 641 | ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0, 642 | "Redis server returned invalid response near pos %z in " 643 | "\"%V\"", 644 | (ssize_t) ((u_char *) p - b->pos), &buf); 645 | 646 | u->length = 0; 647 | 648 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 649 | } 650 | 651 | if (cl == NULL) { 652 | for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { 653 | ll = &cl->next; 654 | } 655 | 656 | cl = ngx_chain_get_free_buf(ctx->request->pool, &u->free_bufs); 657 | if (cl == NULL) { 658 | u->length = 0; 659 | return NGX_ERROR; 660 | } 661 | 662 | cl->buf->flush = 1; 663 | cl->buf->memory = 1; 664 | 665 | *ll = cl; 666 | 667 | dd("response body: %.*s", (int) bytes, p); 668 | 669 | cl->buf->pos = b->last; 670 | cl->buf->last = (u_char *) p; 671 | cl->buf->tag = u->output.tag; 672 | 673 | } else { 674 | cl->buf->last = (u_char *) p; 675 | } 676 | 677 | bytes -= (ssize_t) ((u_char *) p - b->last); 678 | b->last = (u_char *) p; 679 | 680 | if (done) { 681 | dd("response parser done"); 682 | 683 | ctx->query_count--; 684 | 685 | if (ctx->query_count == 0) { 686 | if (cs == reply_error) { 687 | buf.data = (u_char *) p; 688 | buf.len = orig_p - p + orig_len; 689 | 690 | ngx_log_error(NGX_LOG_WARN, ctx->request->connection->log, 691 | 0, "Redis server returned extra bytes: \"%V\" (len %z)", 692 | &buf, buf.len); 693 | 694 | #if 0 695 | if (cl) { 696 | cl->buf->last = cl->buf->pos; 697 | cl = NULL; 698 | *ll = NULL; 699 | } 700 | 701 | u->length = 0; 702 | 703 | return NGX_HTTP_INTERNAL_SERVER_ERROR; 704 | #endif 705 | 706 | } else { 707 | #if defined(nginx_version) && nginx_version >= 1001004 708 | u->keepalive = 1; 709 | #endif 710 | } 711 | 712 | u->length = 0; 713 | 714 | break; 715 | 716 | } else { 717 | ctx->state = NGX_ERROR; 718 | /* continue */ 719 | } 720 | 721 | } else { 722 | /* need more data */ 723 | break; 724 | } 725 | } 726 | 727 | return NGX_OK; 728 | } 729 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | Name 2 | ==== 3 | 4 | ngx_redis2 - Nginx upstream module for the Redis 2.0 protocol 5 | 6 | *This module is not distributed with the Nginx source.* See [the installation instructions](#installation). 7 | 8 | Table of Contents 9 | ================= 10 | 11 | * [Name](#name) 12 | * [Status](#status) 13 | * [Version](#version) 14 | * [Synopsis](#synopsis) 15 | * [Description](#description) 16 | * [Directives](#directives) 17 | * [redis2_query](#redis2_query) 18 | * [redis2_raw_query](#redis2_raw_query) 19 | * [redis2_raw_queries](#redis2_raw_queries) 20 | * [redis2_literal_raw_query](#redis2_literal_raw_query) 21 | * [redis2_pass](#redis2_pass) 22 | * [redis2_connect_timeout](#redis2_connect_timeout) 23 | * [redis2_send_timeout](#redis2_send_timeout) 24 | * [redis2_read_timeout](#redis2_read_timeout) 25 | * [redis2_buffer_size](#redis2_buffer_size) 26 | * [redis2_next_upstream](#redis2_next_upstream) 27 | * [Connection Pool](#connection-pool) 28 | * [Selecting Redis Databases](#selecting-redis-databases) 29 | * [Lua Interoperability](#lua-interoperability) 30 | * [Pipelined Redis Requests by Lua](#pipelined-redis-requests-by-lua) 31 | * [Redis Publish/Subscribe Support](#redis-publishsubscribe-support) 32 | * [Limitations For Redis Publish/Subscribe](#limitations-for-redis-publishsubscribe) 33 | * [Performance Tuning](#performance-tuning) 34 | * [Installation](#installation) 35 | * [Compatibility](#compatibility) 36 | * [Community](#community) 37 | * [English Mailing List](#english-mailing-list) 38 | * [Chinese Mailing List](#chinese-mailing-list) 39 | * [Bugs and Patches](#bugs-and-patches) 40 | * [Source Repository](#source-repository) 41 | * [TODO](#todo) 42 | * [Author](#author) 43 | * [Getting involved](#getting-involved) 44 | * [Copyright & License](#copyright--license) 45 | * [SEE ALSO](#see-also) 46 | 47 | Status 48 | ====== 49 | 50 | This module is already production ready. 51 | 52 | Version 53 | ======= 54 | 55 | This document describes ngx_redis2 [v0.15](https://github.com/openresty/redis2-nginx-module/tags) released on 19 April 2018. 56 | 57 | Synopsis 58 | ======== 59 | 60 | ```nginx 61 | 62 | location = /foo { 63 | set $value 'first'; 64 | redis2_query set one $value; 65 | redis2_pass 127.0.0.1:6379; 66 | } 67 | 68 | # GET /get?key=some_key 69 | location = /get { 70 | set_unescape_uri $key $arg_key; # this requires ngx_set_misc 71 | redis2_query get $key; 72 | redis2_pass foo.com:6379; 73 | } 74 | 75 | # GET /set?key=one&val=first%20value 76 | location = /set { 77 | set_unescape_uri $key $arg_key; # this requires ngx_set_misc 78 | set_unescape_uri $val $arg_val; # this requires ngx_set_misc 79 | redis2_query set $key $val; 80 | redis2_pass foo.com:6379; 81 | } 82 | 83 | # multiple pipelined queries 84 | location = /foo { 85 | set $value 'first'; 86 | redis2_query set one $value; 87 | redis2_query get one; 88 | redis2_query set one two; 89 | redis2_query get one; 90 | redis2_pass 127.0.0.1:6379; 91 | } 92 | 93 | location = /bar { 94 | # $ is not special here... 95 | redis2_literal_raw_query '*1\r\n$4\r\nping\r\n'; 96 | redis2_pass 127.0.0.1:6379; 97 | } 98 | 99 | location = /bar { 100 | # variables can be used below and $ is special 101 | redis2_raw_query 'get one\r\n'; 102 | redis2_pass 127.0.0.1:6379; 103 | } 104 | 105 | # GET /baz?get%20foo%0d%0a 106 | location = /baz { 107 | set_unescape_uri $query $query_string; # this requires the ngx_set_misc module 108 | redis2_raw_query $query; 109 | redis2_pass 127.0.0.1:6379; 110 | } 111 | 112 | location = /init { 113 | redis2_query del key1; 114 | redis2_query lpush key1 C; 115 | redis2_query lpush key1 B; 116 | redis2_query lpush key1 A; 117 | redis2_pass 127.0.0.1:6379; 118 | } 119 | 120 | location = /get { 121 | redis2_query lrange key1 0 -1; 122 | redis2_pass 127.0.0.1:6379; 123 | } 124 | ``` 125 | 126 | [Back to TOC](#table-of-contents) 127 | 128 | Description 129 | =========== 130 | 131 | This is an Nginx upstream module that makes nginx talk to a [Redis](http://redis.io/) 2.x server in a non-blocking way. The full Redis 2.0 unified protocol has been implemented including the Redis pipelining support. 132 | 133 | This module returns the raw TCP response from the Redis server. It's recommended to use my [lua-redis-parser](http://github.com/openresty/lua-redis-parser) (written in pure C) to parse these responses into lua data structure when combined with [lua-nginx-module](http://github.com/openresty/lua-nginx-module). 134 | 135 | When used in conjunction with [lua-nginx-module](http://github.com/openresty/lua-nginx-module), it is recommended to use the [lua-resty-redis](http://github.com/openresty/lua-resty-redis) library instead of this module though, because the former is much more flexible and memory-efficient. 136 | 137 | If you only want to use the `get` redis command, you can try out the [HttpRedisModule](http://wiki.nginx.org/HttpRedisModule). It returns the parsed content part of the Redis response because only `get` is needed to implement. 138 | 139 | Another option is to parse the redis responses on your client side yourself. 140 | 141 | [Back to TOC](#table-of-contents) 142 | 143 | Directives 144 | ========== 145 | 146 | [Back to TOC](#table-of-contents) 147 | 148 | redis2_query 149 | ------------ 150 | **syntax:** *redis2_query cmd arg1 arg2 ...* 151 | 152 | **default:** *no* 153 | 154 | **context:** *location, location if* 155 | 156 | Specify a Redis command by specifying its individual arguments (including the Redis command name itself) in a similar way to the `redis-cli` utility. 157 | 158 | Multiple instances of this directive are allowed in a single location and these queries will be pipelined. For example, 159 | 160 | ```nginx 161 | 162 | location = /pipelined { 163 | redis2_query set hello world; 164 | redis2_query get hello; 165 | 166 | redis2_pass 127.0.0.1:$TEST_NGINX_REDIS_PORT; 167 | } 168 | ``` 169 | 170 | then `GET /pipelined` will yield two successive raw Redis responses 171 | 172 | ```nginx 173 | 174 | +OK 175 | $5 176 | world 177 | ``` 178 | 179 | while newlines here are actually `CR LF` (`\r\n`). 180 | 181 | [Back to TOC](#table-of-contents) 182 | 183 | redis2_raw_query 184 | ---------------- 185 | **syntax:** *redis2_raw_query QUERY* 186 | 187 | **default:** *no* 188 | 189 | **context:** *location, location if* 190 | 191 | Specify raw Redis queries and nginx variables are recognized in the `QUERY` argument. 192 | 193 | Only *one* Redis command is allowed in the `QUERY` argument, or you'll receive an error. If you want to specify multiple pipelined commands in a single query, use the [redis2_raw_queries](#redis2_raw_queries) directive instead. 194 | 195 | [Back to TOC](#table-of-contents) 196 | 197 | redis2_raw_queries 198 | ------------------ 199 | **syntax:** *redis2_raw_queries N QUERIES* 200 | 201 | **default:** *no* 202 | 203 | **context:** *location, location if* 204 | 205 | Specify `N` commands in the `QUERIES` argument. Both the `N` and `QUERIES` 206 | arguments can take Nginx variables. 207 | 208 | Here's some examples 209 | ```nginx 210 | 211 | location = /pipelined { 212 | redis2_raw_queries 3 "flushall\r\nget key1\r\nget key2\r\n"; 213 | redis2_pass 127.0.0.1:6379; 214 | } 215 | 216 | # GET /pipelined2?n=2&cmds=flushall%0D%0Aget%20key%0D%0A 217 | location = /pipelined2 { 218 | set_unescape_uri $n $arg_n; 219 | set_unescape_uri $cmds $arg_cmds; 220 | 221 | redis2_raw_queries $n $cmds; 222 | 223 | redis2_pass 127.0.0.1:6379; 224 | } 225 | ``` 226 | Note that in the second sample above, the [set_unescape_uri](http://github.com/openresty/set-misc-nginx-module#set_unescape_uri) directive is provided by the [set-misc-nginx-module](http://github.com/openresty/set-misc-nginx-module). 227 | 228 | [Back to TOC](#table-of-contents) 229 | 230 | redis2_literal_raw_query 231 | ------------------------ 232 | **syntax:** *redis2_literal_raw_query QUERY* 233 | 234 | **default:** *no* 235 | 236 | **context:** *location, location if* 237 | 238 | Specify a raw Redis query but Nginx variables in it will not be *not* recognized. In other words, you're free to use the dollar sign character (`$`) in your `QUERY` argument. 239 | 240 | Only One redis command is allowed in the `QUERY` argument. 241 | 242 | [Back to TOC](#table-of-contents) 243 | 244 | redis2_pass 245 | ----------- 246 | **syntax:** *redis2_pass <upstream_name>* 247 | 248 | **syntax:** *redis2_pass <host>:<port>* 249 | 250 | **default:** *no* 251 | 252 | **context:** *location, location if* 253 | 254 | **phase:** *content* 255 | 256 | Specify the Redis server backend. 257 | 258 | [Back to TOC](#table-of-contents) 259 | 260 | redis2_connect_timeout 261 | ---------------------- 262 | **syntax:** *redis2_connect_timeout <time>* 263 | 264 | **default:** *60s* 265 | 266 | **context:** *http, server, location* 267 | 268 | The timeout for connecting to the Redis server, in seconds by default. 269 | 270 | It's wise to always explicitly specify the time unit to avoid confusion. Time units supported are `s`(seconds), `ms`(milliseconds), `y`(years), `M`(months), `w`(weeks), `d`(days), `h`(hours), and `m`(minutes). 271 | 272 | This time must be less than 597 hours. 273 | 274 | [Back to TOC](#table-of-contents) 275 | 276 | redis2_send_timeout 277 | ------------------- 278 | **syntax:** *redis2_send_timeout <time>* 279 | 280 | **default:** *60s* 281 | 282 | **context:** *http, server, location* 283 | 284 | The timeout for sending TCP requests to the Redis server, in seconds by default. 285 | 286 | It's wise to always explicitly specify the time unit to avoid confusion. Time units supported are `s`(seconds), `ms`(milliseconds), `y`(years), `M`(months), `w`(weeks), `d`(days), `h`(hours), and `m`(minutes). 287 | 288 | [Back to TOC](#table-of-contents) 289 | 290 | redis2_read_timeout 291 | ------------------- 292 | **syntax:** *redis2_read_timeout <time>* 293 | 294 | **default:** *60s* 295 | 296 | **context:** *http, server, location* 297 | 298 | The timeout for reading TCP responses from the redis server, in seconds by default. 299 | 300 | It's wise to always explicitly specify the time unit to avoid confusion. Time units supported are `s`(seconds), `ms`(milliseconds), `y`(years), `M`(months), `w`(weeks), `d`(days), `h`(hours), and `m`(minutes). 301 | 302 | [Back to TOC](#table-of-contents) 303 | 304 | redis2_buffer_size 305 | ------------------ 306 | **syntax:** *redis2_buffer_size <size>* 307 | 308 | **default:** *4k/8k* 309 | 310 | **context:** *http, server, location* 311 | 312 | This buffer size is used for reading Redis replies, but it's not required to be as big as the largest possible Redis reply. 313 | 314 | This default size is the page size, may be 4k or 8k. 315 | 316 | [Back to TOC](#table-of-contents) 317 | 318 | redis2_next_upstream 319 | -------------------- 320 | **syntax:** *redis2_next_upstream [ error | timeout | invalid_response | off ]* 321 | 322 | **default:** *error timeout* 323 | 324 | **context:** *http, server, location* 325 | 326 | Specify which failure conditions should cause the request to be forwarded to another 327 | upstream server. Applies only when the value in [redis2_pass](#redis2_pass) is an upstream with two or more 328 | servers. 329 | 330 | Here's an artificial example: 331 | ```nginx 332 | 333 | upstream redis_cluster { 334 | server 127.0.0.1:6379; 335 | server 127.0.0.1:6380; 336 | } 337 | 338 | server { 339 | location = /redis { 340 | redis2_next_upstream error timeout invalid_response; 341 | redis2_query get foo; 342 | redis2_pass redis_cluster; 343 | } 344 | } 345 | ``` 346 | 347 | [Back to TOC](#table-of-contents) 348 | 349 | Connection Pool 350 | =============== 351 | 352 | You can use the excellent [HttpUpstreamKeepaliveModule](http://wiki.nginx.org/HttpUpstreamKeepaliveModule) with this module to provide TCP connection pool for Redis. 353 | 354 | A sample config snippet looks like this 355 | 356 | ```nginx 357 | 358 | http { 359 | upstream backend { 360 | server 127.0.0.1:6379; 361 | 362 | # a pool with at most 1024 connections 363 | # and do not distinguish the servers: 364 | keepalive 1024; 365 | } 366 | 367 | server { 368 | ... 369 | location = /redis { 370 | set_unescape_uri $query $arg_query; 371 | redis2_query $query; 372 | redis2_pass backend; 373 | } 374 | } 375 | } 376 | ``` 377 | 378 | [Back to TOC](#table-of-contents) 379 | 380 | Selecting Redis Databases 381 | ========================= 382 | 383 | Redis provides the [select](http://redis.io/commands/SELECT) command to switch Redis databaess. This command is no different from other normal commands 384 | like [get](http://redis.io/commands/GET) or [set](http://redis.io/commands/SET). So you can use them in [redis2_query](#redis2_query) directives, for 385 | example, 386 | 387 | ```nginx 388 | redis2_query select 8; 389 | redis2_query get foo; 390 | ``` 391 | 392 | [Back to TOC](#table-of-contents) 393 | 394 | Lua Interoperability 395 | ==================== 396 | 397 | This module can be served as a non-blocking redis2 client for [lua-nginx-module](http://github.com/openresty/lua-nginx-module) (but nowadays it is recommended to use the [lua-resty-redis](http://github.com/openresty/lua-resty-redis) library instead, which is much simpler to use and more efficient most of the time). 398 | Here's an example using a GET subrequest: 399 | 400 | ```nginx 401 | 402 | location = /redis { 403 | internal; 404 | 405 | # set_unescape_uri is provided by ngx_set_misc 406 | set_unescape_uri $query $arg_query; 407 | 408 | redis2_raw_query $query; 409 | redis2_pass 127.0.0.1:6379; 410 | } 411 | 412 | location = /main { 413 | content_by_lua ' 414 | local res = ngx.location.capture("/redis", 415 | { args = { query = "ping\\r\\n" } } 416 | ) 417 | ngx.print("[" .. res.body .. "]") 418 | '; 419 | } 420 | ``` 421 | 422 | Then accessing `/main` yields 423 | 424 | 425 | [+PONG\r\n] 426 | 427 | 428 | where `\r\n` is `CRLF`. That is, this module returns the *raw* TCP responses from the remote redis server. For Lua-based application developers, they may want to utilize the [lua-redis-parser](http://github.com/openresty/lua-redis-parser) library (written in pure C) to parse such raw responses into Lua data structures. 429 | 430 | When moving the inlined Lua code into an external `.lua` file, it's important to use the escape sequence `\r\n` directly. We used `\\r\\n` above just because the Lua code itself needs quoting when being put into an Nginx string literal. 431 | 432 | You can also use POST/PUT subrequests to transfer the raw Redis request via request body, which does not require URI escaping and unescaping, thus saving some CPU cycles. Here's such an example: 433 | 434 | ```nginx 435 | 436 | location = /redis { 437 | internal; 438 | 439 | # $echo_request_body is provided by the ngx_echo module 440 | redis2_raw_query $echo_request_body; 441 | 442 | redis2_pass 127.0.0.1:6379; 443 | } 444 | 445 | location = /main { 446 | content_by_lua ' 447 | local res = ngx.location.capture("/redis", 448 | { method = ngx.HTTP_PUT, 449 | body = "ping\\r\\n" } 450 | ) 451 | ngx.print("[" .. res.body .. "]") 452 | '; 453 | } 454 | ``` 455 | 456 | > **Important** The nginx variable `$request_body` *only* contains content buffered into memory. If nginx writes the request body to a temporary file (default behavior) the `$request_body` will **not** include the entire content or may be empty. It's recommended to use `$echo_request_body` which reads the full request body (regardless of being in buffer or temporary file). 457 | 458 | This yeilds exactly the same output as the previous (GET) sample. 459 | 460 | One can also use Lua to pick up a concrete Redis backend based on some complicated hashing rules. For instance, 461 | 462 | ```nginx 463 | 464 | upstream redis-a { 465 | server foo.bar.com:6379; 466 | } 467 | 468 | upstream redis-b { 469 | server bar.baz.com:6379; 470 | } 471 | 472 | upstream redis-c { 473 | server blah.blah.org:6379; 474 | } 475 | 476 | server { 477 | ... 478 | 479 | location = /redis { 480 | set_unescape_uri $query $arg_query; 481 | redis2_query $query; 482 | redis2_pass $arg_backend; 483 | } 484 | 485 | location = /foo { 486 | content_by_lua " 487 | -- pick up a server randomly 488 | local servers = {'redis-a', 'redis-b', 'redis-c'} 489 | local i = ngx.time() % #servers + 1; 490 | local srv = servers[i] 491 | 492 | local res = ngx.location.capture('/redis', 493 | { args = { 494 | query = '...', 495 | backend = srv 496 | } 497 | } 498 | ) 499 | ngx.say(res.body) 500 | "; 501 | } 502 | } 503 | ``` 504 | 505 | [Back to TOC](#table-of-contents) 506 | 507 | Pipelined Redis Requests by Lua 508 | ------------------------------- 509 | 510 | Here's a complete example demonstrating how to use Lua to issue multiple pipelined Redis requests via this Nginx module. 511 | 512 | First of all, we include the following in our `nginx.conf` file: 513 | 514 | ```nginx 515 | 516 | location = /redis2 { 517 | internal; 518 | 519 | redis2_raw_queries $args $echo_request_body; 520 | redis2_pass 127.0.0.1:6379; 521 | } 522 | 523 | location = /test { 524 | content_by_lua_file conf/test.lua; 525 | } 526 | ``` 527 | 528 | Basically we use URI query args to pass the number of Redis requests and request body to pass the pipelined Redis request string. 529 | 530 | And then we create the `conf/test.lua` file (whose path is relative to the server root of Nginx) to include the following Lua code: 531 | 532 | ```lua 533 | 534 | -- conf/test.lua 535 | local parser = require "redis.parser" 536 | 537 | local reqs = { 538 | {"set", "foo", "hello world"}, 539 | {"get", "foo"} 540 | } 541 | 542 | local raw_reqs = {} 543 | for i, req in ipairs(reqs) do 544 | table.insert(raw_reqs, parser.build_query(req)) 545 | end 546 | 547 | local res = ngx.location.capture("/redis2?" .. #reqs, 548 | { body = table.concat(raw_reqs, "") }) 549 | 550 | if res.status ~= 200 or not res.body then 551 | ngx.log(ngx.ERR, "failed to query redis") 552 | ngx.exit(500) 553 | end 554 | 555 | local replies = parser.parse_replies(res.body, #reqs) 556 | for i, reply in ipairs(replies) do 557 | ngx.say(reply[1]) 558 | end 559 | ``` 560 | 561 | Here we assume that your Redis server is listening on the default port (6379) of the localhost. We also make use of the [lua-redis-parser](http://github.com/openresty/lua-redis-parser) library to construct raw Redis queries for us and also use it to parse the replies. 562 | 563 | Accessing the `/test` location via HTTP clients like `curl` yields the following output 564 | 565 | 566 | OK 567 | hello world 568 | 569 | 570 | A more realistic setting is to use a proper upstream definition for our Redis backend and enable TCP connection pool via the [keepalive](http://wiki.nginx.org/HttpUpstreamKeepaliveModule#keepalive) directive in it. 571 | 572 | [Back to TOC](#table-of-contents) 573 | 574 | Redis Publish/Subscribe Support 575 | =============================== 576 | 577 | This module has limited support for Redis publish/subscribe feature. It cannot be fully supported due to the stateless nature of REST and HTTP model. 578 | 579 | Consider the following example: 580 | 581 | ```nginx 582 | 583 | location = /redis { 584 | redis2_raw_queries 2 "subscribe /foo/bar\r\n"; 585 | redis2_pass 127.0.0.1:6379; 586 | } 587 | ``` 588 | 589 | And then publish a message for the key `/foo/bar` in the `redis-cli` command line. And then you'll receive two multi-bulk replies from the `/redis` location. 590 | 591 | You can surely parse the replies with the [lua-redis-parser](http://github.com/openresty/lua-redis-parser) library if you're using Lua to access this module's location. 592 | 593 | [Back to TOC](#table-of-contents) 594 | 595 | Limitations For Redis Publish/Subscribe 596 | --------------------------------------- 597 | 598 | If you want to use the [Redis pub/sub](http://redis.io/topics/pubsub) feature with this module, then you must note the following limitations: 599 | 600 | * You cannot use [HttpUpstreamKeepaliveModule](http://wiki.nginx.org/HttpUpstreamKeepaliveModule) with this Redis upstream. Only short Redis connections will work. 601 | * There may be some race conditions that produce the harmless `Redis server returned extra bytes` warnings in your nginx's error.log. Such warnings might be rare but just be prepared for it. 602 | * You should tune the various timeout settings provided by this module like [redis2_connect_timeout](#redis2_connect_timeout) and [redis2_read_timeout](#redis2_read_timeout). 603 | 604 | If you cannot stand these limitations, then you are highly recommended to switch to the [lua-resty-redis](https://github.com/openresty/lua-resty-redis) library for [lua-nginx-module](http://github.com/openresty/lua-nginx-module). 605 | 606 | [Back to TOC](#table-of-contents) 607 | 608 | Performance Tuning 609 | ================== 610 | 611 | * When you're using this module, please ensure you're using a TCP connection pool (provided by [HttpUpstreamKeepaliveModule](http://wiki.nginx.org/HttpUpstreamKeepaliveModule)) and Redis pipelining wherever possible. These features will significantly improve performance. 612 | * Using multiple instance of Redis servers on your multi-core machines also help a lot due to the sequential processing nature of a single Redis server instance. 613 | * When you're benchmarking performance using something like `ab` or `http_load`, please ensure that your error log level is high enough (like `warn`) to prevent Nginx workers spend too much cycles on flushing the `error.log` file, which is always non-buffered and blocking and thus very expensive. 614 | 615 | [Back to TOC](#table-of-contents) 616 | 617 | Installation 618 | ============ 619 | 620 | You are recommended to install this module (as well as the Nginx core and many many other goodies) via the [ngx_openresty bundle](http://openresty.org). Check out the [installation instructions](http://openresty.org/#Installation) for setting up [ngx_openresty](http://openresty.org). 621 | 622 | Alternatively, you can install this module manually by recompiling the standard Nginx core as follows: 623 | 624 | * Grab the nginx source code from [nginx.org](http://nginx.org), for example, the version 1.11.2 (see nginx compatibility), 625 | * and then download the latest version of the release tarball of this module from ngx_redis2's [file list](http://github.com/openresty/redis2-nginx-module/tags). 626 | * and finally build the source with this module: 627 | ```bash 628 | 629 | wget 'http://nginx.org/download/nginx-1.11.2.tar.gz' 630 | tar -xzvf nginx-1.11.2.tar.gz 631 | cd nginx-1.11.2/ 632 | 633 | # Here we assume you would install you nginx under /opt/nginx/. 634 | ./configure --prefix=/opt/nginx \ 635 | --add-module=/path/to/redis2-nginx-module 636 | 637 | make -j2 638 | make install 639 | ``` 640 | 641 | Starting from NGINX 1.9.11, you can also compile this module as a dynamic module, by using the `--add-dynamic-module=PATH` option instead of `--add-module=PATH` on the 642 | `./configure` command line above. And then you can explicitly load the module in your `nginx.conf` via the [load_module](http://nginx.org/en/docs/ngx_core_module.html#load_module) 643 | directive, for example, 644 | 645 | ```nginx 646 | load_module /path/to/modules/ngx_http_redis2_module.so; 647 | ``` 648 | 649 | [Back to TOC](#table-of-contents) 650 | 651 | Compatibility 652 | ============= 653 | 654 | Redis 2.0, 2.2, 2.4, and above should work with this module without any issues. So is the [Alchemy Database](http://code.google.com/p/alchemydatabase/) (aka redisql in its early days). 655 | 656 | The following versions of Nginx should work with this module: 657 | 658 | * **1.17.x** (last tested: 1.17.4) 659 | * **1.16.x** 660 | * **1.15.x** (last tested: 1.15.8) 661 | * **1.14.x** 662 | * **1.13.x** (last tested: 1.13.6) 663 | * **1.12.x** 664 | * 1.11.x (last tested: 1.11.2) 665 | * 1.10.x 666 | * 1.9.x (last tested: 1.9.15) 667 | * 1.8.x 668 | * 1.7.x (last tested: 1.7.10) 669 | * 1.6.x 670 | * 1.5.x (last tested: 1.5.12) 671 | * 1.4.x (last tested: 1.4.3) 672 | * 1.3.x (last tested: 1.3.7) 673 | * 1.2.x (last tested: 1.2.7) 674 | * 1.1.x (last tested: 1.1.5) 675 | * 1.0.x (last tested: 1.0.10) 676 | * 0.9.x (last tested: 0.9.4) 677 | * 0.8.x >= 0.8.31 (last tested: 0.8.54) 678 | 679 | Earlier versions of Nginx will *not* work. 680 | 681 | If you find that any particular version of Nginx above 0.8.31 does not work with this module, please consider reporting a bug. 682 | 683 | [Back to TOC](#table-of-contents) 684 | 685 | Community 686 | ========= 687 | 688 | [Back to TOC](#table-of-contents) 689 | 690 | English Mailing List 691 | -------------------- 692 | 693 | The [openresty-en](https://groups.google.com/group/openresty-en) mailing list is for English speakers. 694 | 695 | [Back to TOC](#table-of-contents) 696 | 697 | Chinese Mailing List 698 | -------------------- 699 | 700 | The [openresty](https://groups.google.com/group/openresty) mailing list is for Chinese speakers. 701 | 702 | [Back to TOC](#table-of-contents) 703 | 704 | Bugs and Patches 705 | ================ 706 | 707 | Please submit bug reports, wishlists, or patches by 708 | 709 | 1. creating a ticket on the [GitHub Issue Tracker](http://github.com/openresty/redis2-nginx-module/issues), 710 | 1. or posting to the [OpenResty community](#community). 711 | 712 | [Back to TOC](#table-of-contents) 713 | 714 | Source Repository 715 | ================= 716 | 717 | Available on github at [openresty/redis2-nginx-module](http://github.com/openresty/redis2-nginx-module). 718 | 719 | [Back to TOC](#table-of-contents) 720 | 721 | TODO 722 | ==== 723 | * Add the `redis2_as_json` directive to allow emitting JSON directly. 724 | 725 | [Back to TOC](#table-of-contents) 726 | 727 | Author 728 | ====== 729 | 730 | Yichun "agentzh" Zhang (章亦春) , OpenResty Inc. 731 | 732 | [Back to TOC](#table-of-contents) 733 | 734 | Getting involved 735 | ================ 736 | 737 | You'll be very welcomed to submit patches to the author or just ask for 738 | a commit bit to the source repository on GitHub. 739 | 740 | [Back to TOC](#table-of-contents) 741 | 742 | Copyright & License 743 | =================== 744 | 745 | This module is licenced under the BSD license. 746 | 747 | Copyright (C) 2010-2018, by Yichun "agentzh" Zhang (章亦春) , OpenResty Inc. 748 | 749 | All rights reserved. 750 | 751 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 752 | 753 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 754 | 755 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 756 | 757 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 758 | 759 | [Back to TOC](#table-of-contents) 760 | 761 | SEE ALSO 762 | ======== 763 | * The [Redis](http://redis.io/) server homepage. 764 | * The Redis wire protocol: 765 | * a redis response parser and a request constructor for Lua: [lua-redis-parser](http://github.com/openresty/lua-redis-parser). 766 | * [lua-nginx-module](http://github.com/openresty/lua-nginx-module) 767 | * The [ngx_openresty bundle](http://openresty.org). 768 | * The [lua-resty-redis](https://github.com/openresty/lua-resty-redis) library based on the [lua-nginx-module](http://github.com/openresty/lua-nginx-module) cosocket API. 769 | 770 | [Back to TOC](#table-of-contents) 771 | 772 | --------------------------------------------------------------------------------