├── .gitattributes ├── config ├── src ├── ngx_http_rds.h ├── ngx_http_rds_csv_util.h ├── ngx_http_rds_csv_processor.h ├── ngx_http_rds_csv_output.h ├── ddebug.h ├── resty_dbd_stream.h ├── ngx_http_rds_csv_util.c ├── ngx_http_rds_csv_filter_module.h ├── ngx_http_rds_utils.h ├── ngx_http_rds_csv_processor.c ├── ngx_http_rds_csv_filter_module.c └── ngx_http_rds_csv_output.c ├── t ├── form.t ├── unused.t ├── pg.t ├── 000_init.t ├── buf.t ├── escape.t ├── sanity.t └── sanity-stream.t ├── .gitignore ├── util └── build.sh ├── .travis.yml ├── valgrind.suppress └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | *.t linguist-language=Text 2 | -------------------------------------------------------------------------------- /config: -------------------------------------------------------------------------------- 1 | ngx_addon_name=ngx_http_rds_csv_filter_module 2 | HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES ngx_http_rds_csv_filter_module" 3 | NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/src/ngx_http_rds_csv_filter_module.c $ngx_addon_dir/src/ngx_http_rds_csv_processor.c $ngx_addon_dir/src/ngx_http_rds_csv_util.c $ngx_addon_dir/src/ngx_http_rds_csv_output.c" 4 | NGX_ADDON_DEPS="$NGX_ADDON_DEPS $ngx_addon_dir/src/ddebug.h $ngx_addon_dir/src/resty_dbd_stream.h $ngx_addon_dir/src/ngx_http_rds_csv_filter_module.h $ngx_addon_dir/src/ngx_http_rds_csv_processor.h $ngx_addon_dir/src/ngx_http_rds_csv_util.h $ngx_addon_dir/src/ngx_http_rds.h $ngx_addon_dir/src/resty_dbd_stream.h $ngx_addon_dir/src/ngx_http_rds_csv_output.h $ngx_addon_dir/src/ngx_http_rds_utils.h" 5 | 6 | -------------------------------------------------------------------------------- /src/ngx_http_rds.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) agentzh 4 | */ 5 | 6 | #ifndef NGX_HTTP_RDS_H 7 | #define NGX_HTTP_RDS_H 8 | 9 | 10 | #include "resty_dbd_stream.h" 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | typedef struct { 17 | uint16_t std_errcode; 18 | uint16_t drv_errcode; 19 | ngx_str_t errstr; 20 | 21 | uint64_t affected_rows; 22 | uint64_t insert_id; 23 | uint16_t col_count; 24 | 25 | } ngx_http_rds_header_t; 26 | 27 | 28 | typedef struct ngx_http_rds_column_s { 29 | rds_col_type_t std_type; 30 | uint16_t drv_type; 31 | 32 | ngx_str_t name; 33 | 34 | } ngx_http_rds_column_t; 35 | 36 | 37 | #endif /* NGX_HTTP_RDS_H */ 38 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_util.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) Yichun Zhang (agentzh) 4 | */ 5 | 6 | 7 | #ifndef NGX_HTTP_RDS_CSV_UTIL_H 8 | #define NGX_HTTP_RDS_CSV_UTIL_H 9 | 10 | 11 | #include 12 | #include 13 | 14 | 15 | #ifndef NGX_UINT64_LEN 16 | #define NGX_UINT64_LEN (sizeof("18446744073709551615") - 1) 17 | #endif 18 | 19 | #ifndef NGX_UINT16_LEN 20 | #define NGX_UINT16_LEN (sizeof("65535") - 1) 21 | #endif 22 | 23 | #ifndef ngx_copy_literal 24 | #define ngx_copy_literal(p, s) ngx_copy(p, s, sizeof(s) - 1) 25 | #endif 26 | 27 | 28 | uintptr_t ngx_http_rds_csv_escape_csv_str(u_char field_sep, u_char *dst, 29 | u_char *src, size_t size, unsigned *need_quotes); 30 | ngx_int_t ngx_http_rds_csv_test_content_type(ngx_http_request_t *r); 31 | void ngx_http_rds_csv_discard_bufs(ngx_pool_t *pool, ngx_chain_t *in); 32 | 33 | 34 | #endif /* NGX_HTTP_RDS_CSV_UTIL_H */ 35 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_processor.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) Yichun Zhang (agentzh) 4 | */ 5 | 6 | 7 | #ifndef NGX_HTTP_RDS_CSV_PROCESSOR_H 8 | #define NGX_HTTP_RDS_CSV_PROCESSOR_H 9 | 10 | 11 | #include "ngx_http_rds_csv_filter_module.h" 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | ngx_int_t ngx_http_rds_csv_process_header(ngx_http_request_t *r, 19 | ngx_chain_t *in, ngx_http_rds_csv_ctx_t *ctx); 20 | 21 | ngx_int_t ngx_http_rds_csv_process_col(ngx_http_request_t *r, 22 | ngx_chain_t *in, ngx_http_rds_csv_ctx_t *ctx); 23 | 24 | ngx_int_t ngx_http_rds_csv_process_row(ngx_http_request_t *r, 25 | ngx_chain_t *in, ngx_http_rds_csv_ctx_t *ctx); 26 | 27 | ngx_int_t ngx_http_rds_csv_process_field(ngx_http_request_t *r, 28 | ngx_chain_t *in, ngx_http_rds_csv_ctx_t *ctx); 29 | 30 | ngx_int_t ngx_http_rds_csv_process_more_field_data(ngx_http_request_t *r, 31 | ngx_chain_t *in, ngx_http_rds_csv_ctx_t *ctx); 32 | 33 | 34 | #endif /* NGX_HTTP_RDS_CSV_PROCESSOR_H */ 35 | -------------------------------------------------------------------------------- /t/form.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * 2 * blocks(); 9 | 10 | $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; 11 | 12 | our $http_config = <<'_EOC_'; 13 | upstream backend { 14 | drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql 15 | dbname=ngx_test user=ngx_test password=ngx_test; 16 | } 17 | _EOC_ 18 | 19 | no_diff(); 20 | 21 | run_tests(); 22 | 23 | __DATA__ 24 | 25 | === TEST 1: sanity 26 | --- http_config eval: $::http_config 27 | --- config 28 | location /mysql { 29 | set_form_input $sql 'sql'; 30 | set_unescape_uri $sql; 31 | #echo $sql; 32 | drizzle_query $sql; 33 | drizzle_pass backend; 34 | rds_csv on; 35 | } 36 | --- more_headers 37 | Content-Type: application/x-www-form-urlencoded 38 | --- request 39 | POST /mysql 40 | sql=select%20*%20from%20cats; 41 | --- response_body eval 42 | "id,name\r 43 | 2,\r 44 | 3,bob\r 45 | " 46 | 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.mobi 2 | genmobi.sh 3 | .libs 4 | *.swp 5 | *.slo 6 | *.la 7 | *.swo 8 | *.lo 9 | *~ 10 | *.o 11 | print.txt 12 | .rsync 13 | *.tar.gz 14 | dist 15 | build[789] 16 | build 17 | tags 18 | update-readme 19 | *.tmp 20 | test/Makefile 21 | test/blib 22 | test.sh 23 | go 24 | t/t.sh 25 | test/t/servroot/ 26 | releng 27 | reset 28 | *.t_ 29 | src/handler.h 30 | src/util.c 31 | src/module.h 32 | src/module.c 33 | src/drizzle.c 34 | src/processor.h 35 | src/handler.c 36 | src/util.h 37 | src/drizzle.h 38 | src/processor.c 39 | src/output.c 40 | src/output.h 41 | libdrizzle 42 | ctags 43 | src/stream.h 44 | nginx 45 | keepalive 46 | reindex 47 | src/keepalive.c 48 | src/keepalive.h 49 | src/checker.h 50 | src/checker.c 51 | src/quoting.h 52 | src/quoting.c 53 | src/module.h 54 | src/module.c 55 | src/util.h 56 | src/util.c 57 | src/processor.h 58 | src/processor.c 59 | src/rds.h 60 | src/utils.h 61 | src/handler.c 62 | src/handler.h 63 | util/bench 64 | pack 65 | restart 66 | misc/ 67 | t/servroot/ 68 | all 69 | build1[0-9] 70 | buildroot/ 71 | a.pl 72 | *.plist 73 | Makefile 74 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_output.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) Yichun Zhang (agentzh) 4 | */ 5 | 6 | 7 | #ifndef NGX_HTTP_RDS_CSV_OUTPUT_H 8 | #define NGX_HTTP_RDS_CSV_OUTPUT_H 9 | 10 | 11 | #include "ngx_http_rds_csv_filter_module.h" 12 | #include "ngx_http_rds.h" 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | 19 | ngx_int_t ngx_http_rds_csv_output_header(ngx_http_request_t *r, 20 | ngx_http_rds_csv_ctx_t *ctx, ngx_http_rds_header_t *header); 21 | 22 | ngx_int_t ngx_http_rds_csv_output_field_names(ngx_http_request_t *r, 23 | ngx_http_rds_csv_ctx_t *ctx); 24 | 25 | ngx_int_t ngx_http_rds_csv_output_literal(ngx_http_request_t *r, 26 | ngx_http_rds_csv_ctx_t *ctx, u_char *data, size_t len, int last_buf); 27 | 28 | ngx_int_t ngx_http_rds_csv_output_bufs(ngx_http_request_t *r, 29 | ngx_http_rds_csv_ctx_t *ctx); 30 | 31 | ngx_int_t ngx_http_rds_csv_output_field(ngx_http_request_t *r, 32 | ngx_http_rds_csv_ctx_t *ctx, u_char *data, size_t len, int is_null); 33 | 34 | ngx_int_t ngx_http_rds_csv_output_more_field_data(ngx_http_request_t *r, 35 | ngx_http_rds_csv_ctx_t *ctx, u_char *data, size_t len); 36 | 37 | 38 | #endif /* NGX_HTTP_RDS_CSV_OUTPUT_H */ 39 | -------------------------------------------------------------------------------- /util/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this file is mostly meant to be used by the author himself. 4 | 5 | root=`pwd` 6 | version=$1 7 | home=~ 8 | force=$2 9 | 10 | ngx-build $force $version \ 11 | --with-cc-opt="-O1" \ 12 | --with-ld-opt="-Wl,-rpath,/opt/drizzle/lib:/opt/pg9/lib" \ 13 | --without-mail_pop3_module \ 14 | --without-mail_imap_module \ 15 | --without-mail_smtp_module \ 16 | --without-http_upstream_ip_hash_module \ 17 | --without-http_empty_gif_module \ 18 | --without-http_memcached_module \ 19 | --without-http_referer_module \ 20 | --without-http_autoindex_module \ 21 | --without-http_auth_basic_module \ 22 | --without-http_userid_module \ 23 | --add-module=$root/../eval-nginx-module \ 24 | --add-module=$root/../echo-nginx-module \ 25 | --add-module=$root/../xss-nginx-module \ 26 | --add-module=$root/../ndk-nginx-module \ 27 | --add-module=$root/../set-misc-nginx-module \ 28 | --add-module=$root/../array-var-nginx-module \ 29 | --add-module=$root $opts \ 30 | --add-module=$root/../drizzle-nginx-module \ 31 | --add-module=$root/../form-input-nginx-module \ 32 | --add-module=$root/../postgres-nginx-module \ 33 | --with-debug 34 | #--add-module=$root/../lua-nginx-module \ 35 | #--add-module=$home/work/ngx_http_auth_request-0.1 #\ 36 | #--with-rtsig_module 37 | #--with-cc-opt="-g3 -O0" 38 | #--add-module=$root/../echo-nginx-module \ 39 | #--without-http_ssi_module # we cannot disable ssi because echo_location_async depends on it (i dunno why?!) 40 | 41 | -------------------------------------------------------------------------------- /t/unused.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | #repeat_each(10); 7 | no_shuffle(); 8 | 9 | repeat_each(2); 10 | 11 | plan tests => repeat_each() * (4 * blocks()); 12 | 13 | run_tests(); 14 | 15 | __DATA__ 16 | 17 | === TEST 1: used 18 | --- http_config eval: $::http_config 19 | --- config 20 | location = /t { 21 | echo ok; 22 | rds_csv on; 23 | } 24 | --- request 25 | GET /t 26 | --- stap 27 | F(ngx_http_rds_csv_header_filter) { 28 | println("rds csv header filter") 29 | } 30 | 31 | F(ngx_http_rds_csv_body_filter) { 32 | println("rds csv body filter") 33 | } 34 | 35 | --- stap_out 36 | rds csv header filter 37 | rds csv body filter 38 | rds csv body filter 39 | 40 | --- response_body 41 | ok 42 | --- no_error_log 43 | [error] 44 | 45 | 46 | 47 | === TEST 2: unused 48 | --- http_config eval: $::http_config 49 | --- config 50 | location = /t { 51 | echo ok; 52 | #rds_csv on; 53 | } 54 | --- request 55 | GET /t 56 | --- stap 57 | F(ngx_http_rds_csv_header_filter) { 58 | println("rds csv header filter") 59 | } 60 | 61 | F(ngx_http_rds_csv_body_filter) { 62 | println("rds csv body filter") 63 | } 64 | 65 | --- stap_out 66 | --- response_body 67 | ok 68 | --- no_error_log 69 | [error] 70 | 71 | 72 | 73 | === TEST 3: used (multi http {} blocks) 74 | This test case won't run with nginx 1.9.3+ since duplicate http {} blocks 75 | have been prohibited since then. 76 | --- SKIP 77 | --- http_config eval: $::http_config 78 | --- config 79 | location = /t { 80 | echo ok; 81 | rds_csv on; 82 | } 83 | 84 | --- post_main_config 85 | http { 86 | } 87 | 88 | --- request 89 | GET /t 90 | --- stap 91 | F(ngx_http_rds_csv_header_filter) { 92 | println("rds csv header filter") 93 | } 94 | 95 | F(ngx_http_rds_csv_body_filter) { 96 | println("rds csv body filter") 97 | } 98 | 99 | --- stap_out 100 | rds csv header filter 101 | rds csv body filter 102 | rds csv body filter 103 | 104 | --- response_body 105 | ok 106 | --- no_error_log 107 | [error] 108 | 109 | -------------------------------------------------------------------------------- /src/ddebug.h: -------------------------------------------------------------------------------- 1 | #ifndef DDEBUG_H 2 | #define DDEBUG_H 3 | 4 | #include 5 | #include 6 | 7 | #if defined(DDEBUG) && (DDEBUG) 8 | 9 | # define dd_dump_chain_size() { \ 10 | int n; \ 11 | ngx_chain_t *cl; \ 12 | \ 13 | for (n = 0, cl = ctx->out; cl; cl = cl->next, n++) { \ 14 | } \ 15 | \ 16 | dd("chain size: %d", n); \ 17 | } 18 | 19 | # if (NGX_HAVE_VARIADIC_MACROS) 20 | 21 | # define dd(...) fprintf(stderr, "rds-csv *** %s: ", __func__); \ 22 | fprintf(stderr, __VA_ARGS__); \ 23 | fprintf(stderr, " at %s line %d.\n", __FILE__, __LINE__) 24 | 25 | # else 26 | 27 | #include 28 | #include 29 | 30 | #include 31 | 32 | static ngx_inline void 33 | dd(const char * fmt, ...) { 34 | } 35 | 36 | # endif 37 | 38 | #else 39 | 40 | # define dd_dump_chain_size() 41 | 42 | # if (NGX_HAVE_VARIADIC_MACROS) 43 | 44 | # define dd(...) 45 | 46 | # else 47 | 48 | #include 49 | 50 | static ngx_inline void 51 | dd(const char * fmt, ...) { 52 | } 53 | 54 | # endif 55 | 56 | #endif 57 | 58 | #if defined(DDEBUG) && (DDEBUG) 59 | 60 | #define dd_check_read_event_handler(r) \ 61 | dd("r->read_event_handler = %s", \ 62 | r->read_event_handler == ngx_http_block_reading ? \ 63 | "ngx_http_block_reading" : \ 64 | r->read_event_handler == ngx_http_test_reading ? \ 65 | "ngx_http_test_reading" : \ 66 | r->read_event_handler == ngx_http_request_empty_handler ? \ 67 | "ngx_http_request_empty_handler" : "UNKNOWN") 68 | 69 | #define dd_check_write_event_handler(r) \ 70 | dd("r->write_event_handler = %s", \ 71 | r->write_event_handler == ngx_http_handler ? \ 72 | "ngx_http_handler" : \ 73 | r->write_event_handler == ngx_http_core_run_phases ? \ 74 | "ngx_http_core_run_phases" : \ 75 | r->write_event_handler == ngx_http_request_empty_handler ? \ 76 | "ngx_http_request_empty_handler" : "UNKNOWN") 77 | 78 | #else 79 | 80 | #define dd_check_read_event_handler(r) 81 | #define dd_check_write_event_handler(r) 82 | 83 | #endif 84 | 85 | #endif /* DDEBUG_H */ 86 | -------------------------------------------------------------------------------- /src/resty_dbd_stream.h: -------------------------------------------------------------------------------- 1 | #ifndef RESTY_DBD_STREAME_H 2 | #define RESTY_DBD_STREAME_H 3 | 4 | #define resty_dbd_stream_version 3 5 | #define resty_dbd_stream_version_string "0.0.3" 6 | 7 | #define rds_content_type \ 8 | "application/x-resty-dbd-stream" 9 | 10 | #define rds_content_type_len \ 11 | (sizeof(rds_content_type) - 1) 12 | 13 | 14 | typedef enum { 15 | rds_rough_col_type_int = 0 << 14, 16 | rds_rough_col_type_float = 1 << 14, 17 | rds_rough_col_type_str = 2 << 14, 18 | rds_rough_col_type_bool = 3 << 14 19 | 20 | } rds_rough_col_type_t; 21 | 22 | 23 | /* The following types (or spellings thereof) are specified 24 | * by SQL: 25 | * bigint, bit, bit varying, boolean, char, character varying, 26 | * character, varchar, date, double precision, integer, 27 | * interval, numeric, decimal, real, smallint, 28 | * time (with or without time zone), 29 | * timestamp (with or without time zone), xml */ 30 | 31 | typedef enum { 32 | rds_col_type_unknown = 0 | rds_rough_col_type_str, 33 | rds_col_type_bigint = 1 | rds_rough_col_type_int, 34 | rds_col_type_bit = 2 | rds_rough_col_type_str, 35 | rds_col_type_bit_varying = 3 | rds_rough_col_type_str, 36 | 37 | rds_col_type_bool = 4 | rds_rough_col_type_bool, 38 | rds_col_type_char = 5 | rds_rough_col_type_str, 39 | rds_col_type_varchar = 6 | rds_rough_col_type_str, 40 | rds_col_type_date = 7 | rds_rough_col_type_str, 41 | rds_col_type_double = 8 | rds_rough_col_type_float, 42 | rds_col_type_integer = 9 | rds_rough_col_type_int, 43 | rds_col_type_interval = 10 | rds_rough_col_type_float, 44 | rds_col_type_decimal = 11 | rds_rough_col_type_float, 45 | rds_col_type_real = 12 | rds_rough_col_type_float, 46 | rds_col_type_smallint = 13 | rds_rough_col_type_int, 47 | rds_col_type_time_with_time_zone = 14 | rds_rough_col_type_str, 48 | rds_col_type_time = 15 | rds_rough_col_type_str, 49 | rds_col_type_timestamp_with_time_zone = 16 | rds_rough_col_type_str, 50 | rds_col_type_timestamp = 17 | rds_rough_col_type_str, 51 | rds_col_type_xml = 18 | rds_rough_col_type_str, 52 | 53 | /* our additions */ 54 | rds_col_type_blob = 19 | rds_rough_col_type_str 55 | 56 | } rds_col_type_t; 57 | 58 | #endif /* RESTY_DBD_STREAME_H */ 59 | 60 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_util.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) Yichun Zhang (agentzh) 4 | */ 5 | 6 | 7 | #ifndef DDEBUG 8 | #define DDEBUG 0 9 | #endif 10 | #include "ddebug.h" 11 | 12 | 13 | #include "resty_dbd_stream.h" 14 | #include "ngx_http_rds_csv_util.h" 15 | 16 | 17 | uintptr_t 18 | ngx_http_rds_csv_escape_csv_str(u_char field_sep, u_char *dst, u_char *src, 19 | size_t size, unsigned *need_quotes) 20 | { 21 | ngx_uint_t n; 22 | 23 | if (dst == NULL) { 24 | *need_quotes = 0; 25 | 26 | /* find the number of characters to be escaped */ 27 | 28 | n = 0; 29 | 30 | while (size) { 31 | switch (*src) { 32 | case '"': 33 | n++; 34 | /* fallthrough */ 35 | 36 | case '\r': 37 | case '\n': 38 | *need_quotes = 1; 39 | break; 40 | 41 | default: 42 | if (*src == field_sep) { 43 | *need_quotes = 1; 44 | } 45 | break; 46 | } 47 | 48 | src++; 49 | size--; 50 | } 51 | 52 | return (uintptr_t) n; 53 | } 54 | 55 | while (size) { 56 | if (*src == '"') { 57 | *dst++ = '"'; 58 | *dst++ = '"'; 59 | src++; 60 | 61 | } else { 62 | *dst++ = *src++; 63 | } 64 | 65 | size--; 66 | } 67 | 68 | return (uintptr_t) dst; 69 | } 70 | 71 | 72 | ngx_int_t 73 | ngx_http_rds_csv_test_content_type(ngx_http_request_t *r) 74 | { 75 | ngx_str_t *type; 76 | 77 | type = &r->headers_out.content_type; 78 | if (type->len != rds_content_type_len 79 | || ngx_strncmp(type->data, rds_content_type, rds_content_type_len) 80 | != 0) 81 | { 82 | return NGX_DECLINED; 83 | } 84 | 85 | return NGX_OK; 86 | } 87 | 88 | 89 | void 90 | ngx_http_rds_csv_discard_bufs(ngx_pool_t *pool, ngx_chain_t *in) 91 | { 92 | ngx_chain_t *cl; 93 | 94 | for (cl = in; cl; cl = cl->next) { 95 | #if 0 96 | if (cl->buf->temporary 97 | && ngx_buf_size(cl->buf) > 0) 98 | { 99 | ngx_pfree(pool, cl->buf->start); 100 | } 101 | #endif 102 | 103 | cl->buf->pos = cl->buf->last; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_filter_module.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) agentzh 4 | */ 5 | 6 | 7 | #ifndef NGX_HTTP_RDS_CSV_FILTER_MODULE_H 8 | #define NGX_HTTP_RDS_CSV_FILTER_MODULE_H 9 | 10 | 11 | #include "ngx_http_rds.h" 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | #ifndef NGX_HTTP_RESET_CONTENT 19 | #define NGX_HTTP_RESET_CONTENT 205 20 | #endif 21 | 22 | 23 | extern ngx_module_t ngx_http_rds_csv_filter_module; 24 | 25 | extern ngx_http_output_header_filter_pt ngx_http_rds_csv_next_header_filter; 26 | extern ngx_http_output_body_filter_pt ngx_http_rds_csv_next_body_filter; 27 | 28 | 29 | typedef struct { 30 | ngx_flag_t enabled; 31 | ngx_str_t row_term; 32 | ngx_uint_t field_sep; 33 | size_t buf_size; 34 | ngx_flag_t field_name_header; 35 | ngx_str_t content_type; 36 | } ngx_http_rds_csv_loc_conf_t; 37 | 38 | 39 | typedef struct { 40 | ngx_int_t requires_filter; 41 | } ngx_http_rds_csv_main_conf_t; 42 | 43 | 44 | typedef enum { 45 | state_expect_header, 46 | state_expect_col, 47 | state_expect_row, 48 | state_expect_field, 49 | state_expect_more_field_data, 50 | state_done 51 | 52 | } ngx_http_rds_csv_state_t; 53 | 54 | 55 | typedef struct { 56 | ngx_http_rds_csv_state_t state; 57 | 58 | ngx_str_t *col_name; 59 | ngx_uint_t col_count; 60 | ngx_uint_t cur_col; 61 | 62 | ngx_http_rds_column_t *cols; 63 | size_t row; 64 | 65 | uint32_t field_offset; 66 | uint32_t field_total; 67 | 68 | ngx_buf_tag_t tag; 69 | 70 | ngx_chain_t *out; 71 | ngx_chain_t **last_out; 72 | ngx_chain_t *busy_bufs; 73 | ngx_chain_t *free_bufs; 74 | 75 | ngx_buf_t *out_buf; 76 | ngx_buf_t cached; 77 | ngx_buf_t postponed; 78 | 79 | size_t avail_out; 80 | 81 | uint32_t field_data_rest; 82 | 83 | uint32_t header_sent:1; 84 | uint32_t seen_stream_end:1; 85 | uint32_t generated_col_names:1; 86 | } ngx_http_rds_csv_ctx_t; 87 | 88 | 89 | #endif /* NGX_HTTP_RDS_CSV_FILTER_MODULE_H */ 90 | -------------------------------------------------------------------------------- /t/pg.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * 2 * blocks(); 9 | 10 | $ENV{TEST_NGINX_POSTGRESQL_PORT} ||= 5432; 11 | 12 | no_long_string(); 13 | 14 | no_diff(); 15 | 16 | run_tests(); 17 | 18 | __DATA__ 19 | 20 | === TEST 1: bool blob field (keepalive off) 21 | --- http_config 22 | upstream backend { 23 | postgres_server 127.0.0.1:$TEST_NGINX_POSTGRESQL_PORT 24 | dbname=ngx_test user=ngx_test password=ngx_test; 25 | postgres_keepalive off; 26 | } 27 | --- config 28 | location /test { 29 | echo_location /pgignore "drop table if exists foo"; 30 | echo_location /pg "create table foo (id serial, flag bool);"; 31 | echo_location /pg "insert into foo (flag) values (true);"; 32 | echo_location /pg "insert into foo (flag) values (false);"; 33 | echo_location /pg "select * from foo order by id;"; 34 | } 35 | location /pg { 36 | postgres_pass backend; 37 | postgres_query $query_string; 38 | rds_csv on; 39 | } 40 | location = /pgignore { 41 | postgres_pass backend; 42 | postgres_query $query_string; 43 | rds_csv on; 44 | error_page 500 = /ignore; 45 | } 46 | location /ignore { echo "ignore"; } 47 | --- request 48 | GET /test 49 | --- response_body eval 50 | qq{errcode,errstr,insert_id,affected_rows\r 51 | 0,,0,0\r 52 | errcode,errstr,insert_id,affected_rows\r 53 | 0,,0,0\r 54 | errcode,errstr,insert_id,affected_rows\r 55 | 0,,0,1\r 56 | errcode,errstr,insert_id,affected_rows\r 57 | 0,,0,1\r 58 | id,flag\r 59 | 1,t\r 60 | 2,f\r 61 | } 62 | --- skip_nginx: 2: < 0.7.46 63 | 64 | 65 | 66 | === TEST 2: bool blob field (keepalive on) 67 | --- http_config 68 | upstream backend { 69 | postgres_server 127.0.0.1:$TEST_NGINX_POSTGRESQL_PORT 70 | dbname=ngx_test user=ngx_test password=ngx_test; 71 | } 72 | --- config 73 | location /test { 74 | echo_location /pg "drop table if exists foo"; 75 | echo_location /pg "create table foo (id serial, flag bool);"; 76 | echo_location /pg "insert into foo (flag) values (true);"; 77 | echo_location /pg "insert into foo (flag) values (false);"; 78 | echo_location /pg "select * from foo order by id;"; 79 | } 80 | location /pg { 81 | postgres_pass backend; 82 | postgres_query $query_string; 83 | rds_csv on; 84 | } 85 | location = /pgignore { 86 | postgres_pass backend; 87 | postgres_query $query_string; 88 | rds_csv on; 89 | error_page 500 = /ignore; 90 | } 91 | location /ignore { echo "ignore"; } 92 | --- request 93 | GET /test 94 | --- response_body eval 95 | qq{errcode,errstr,insert_id,affected_rows\r 96 | 0,,0,0\r 97 | errcode,errstr,insert_id,affected_rows\r 98 | 0,,0,0\r 99 | errcode,errstr,insert_id,affected_rows\r 100 | 0,,0,1\r 101 | errcode,errstr,insert_id,affected_rows\r 102 | 0,,0,1\r 103 | id,flag\r 104 | 1,t\r 105 | 2,f\r 106 | } 107 | --- skip_nginx: 2: < 0.7.46 108 | 109 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: focal 3 | 4 | branches: 5 | only: 6 | - "master" 7 | 8 | os: linux 9 | 10 | language: c 11 | 12 | cache: 13 | directories: 14 | - download-cache 15 | 16 | addons: 17 | apt: 18 | packages: 19 | - axel 20 | - cpanminus 21 | - libtest-base-perl 22 | - libtext-diff-perl 23 | - liburi-perl 24 | - libwww-perl 25 | - libtest-longstring-perl 26 | - liblist-moreutils-perl 27 | - libgd-dev 28 | postgresql: "13" 29 | 30 | compiler: 31 | - gcc 32 | 33 | env: 34 | global: 35 | - LUAJIT_PREFIX=/opt/luajit21 36 | - LUAJIT_LIB=$LUAJIT_PREFIX/lib 37 | - LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH 38 | - LUAJIT_INC=$LUAJIT_PREFIX/include/luajit-2.1 39 | - LUA_INCLUDE_DIR=$LUAJIT_INC 40 | - LUA_CMODULE_DIR=/lib 41 | - JOBS=3 42 | - NGX_BUILD_JOBS=$JOBS 43 | - TEST_NGINX_SLEEP=0.006 44 | matrix: 45 | - NGINX_VERSION=1.29.2 46 | 47 | services: 48 | - mysql 49 | - postgresql 50 | 51 | install: 52 | - if [ ! -f download-cache/drizzle7-2011.07.21.tar.gz ]; then wget -P download-cache https://github.com/openresty/openresty-deps-prebuild/releases/download/v20230902/drizzle7-2011.07.21.tar.gz; fi 53 | - git clone https://github.com/openresty/nginx-devel-utils.git 54 | - git clone https://github.com/openresty/openresty.git ../openresty 55 | - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx 56 | - git clone https://github.com/simpl/ngx_devel_kit.git ../ndk-nginx-module 57 | - git clone https://github.com/openresty/test-nginx.git 58 | - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git 59 | - git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module 60 | - git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module 61 | - git clone https://github.com/openresty/nginx-eval-module.git ../eval-nginx-module 62 | - git clone https://github.com/openresty/xss-nginx-module.git ../xss-nginx-module 63 | - git clone https://github.com/openresty/set-misc-nginx-module.git ../set-misc-nginx-module 64 | - git clone https://github.com/openresty/array-var-nginx-module.git ../array-var-nginx-module 65 | - git clone https://github.com/openresty/drizzle-nginx-module.git ../drizzle-nginx-module 66 | - git clone https://github.com/calio/form-input-nginx-module.git ../form-input-nginx-module 67 | - git clone https://github.com/openresty/ngx_postgres.git ../postgres-nginx-module 68 | - git clone https://github.com/openresty/openresty.git ../ngx_openresty 69 | 70 | before_script: 71 | - mysql -uroot -e "create database ngx_test; CREATE USER 'ngx_test'@'%' IDENTIFIED WITH mysql_native_password BY 'ngx_test'; grant all on ngx_test.* to 'ngx_test'@'%'; flush privileges;" 72 | - mysql -uroot -e 'alter database ngx_test character set utf8mb4 collate utf8mb4_unicode_ci;' 73 | - psql -c "create database ngx_test;" -U postgres 74 | - psql -c "create user ngx_test with password 'ngx_test';" -U postgres 75 | - psql -c "grant all privileges on database ngx_test to ngx_test;" -U postgres 76 | 77 | script: 78 | - tar xzf download-cache/drizzle7-2011.07.21.tar.gz && cd drizzle7-2011.07.21 79 | - ./configure --prefix=/usr --without-server > build.log 2>&1 || (cat build.log && exit 1) 80 | - sudo PATH=$PATH make libdrizzle-1.0 install-libdrizzle-1.0 > build.log 2>&1 || (cat build.log && exit 1) 81 | - cd ../luajit2 82 | - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' > build.log 2>&1 || (cat build.log && exit 1) 83 | - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) 84 | - cd ../test-nginx && sudo cpanm . && cd .. 85 | - export PATH=$PWD/work/nginx/sbin:$PWD/nginx-devel-utils:$PATH 86 | - export NGX_BUILD_CC=$CC 87 | - sh util/build.sh $NGINX_VERSION > build.log 2>&1 || (cat build.log && exit 1) 88 | - nginx -V 89 | - prove -I. -r t 90 | -------------------------------------------------------------------------------- /src/ngx_http_rds_utils.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) Yichun Zhang (agentzh) 4 | */ 5 | 6 | 7 | #ifndef NGX_HTTP_RDS_UTILS_H 8 | #define NGX_HTTP_RDS_UTILS_H 9 | 10 | 11 | #include 12 | 13 | 14 | static ngx_inline ngx_int_t 15 | ngx_http_rds_parse_header(ngx_http_request_t *r, ngx_buf_t *b, 16 | ngx_http_rds_header_t *header) 17 | { 18 | ssize_t rest; 19 | 20 | rest = sizeof(uint8_t) /* endian type */ 21 | + sizeof(uint32_t) /* format version */ 22 | + sizeof(uint8_t) /* result type */ 23 | 24 | + sizeof(uint16_t) /* standard error code */ 25 | + sizeof(uint16_t) /* driver-specific error code */ 26 | 27 | + sizeof(uint16_t) /* driver-specific errstr len */ 28 | + 0 /* driver-specific errstr data */ 29 | + sizeof(uint64_t) /* affected rows */ 30 | + sizeof(uint64_t) /* insert id */ 31 | + sizeof(uint16_t) /* column count */ 32 | ; 33 | 34 | if (b->last - b->pos < rest) { 35 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 36 | "rds: header is incomplete in the buf"); 37 | return NGX_ERROR; 38 | } 39 | 40 | /* check endian type */ 41 | 42 | if (*(uint8_t *) b->pos != 43 | #if (NGX_HAVE_LITTLE_ENDIAN) 44 | 0 45 | #else /* big endian */ 46 | 1 47 | #endif 48 | ) 49 | { 50 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 51 | "rds: endian type in the header differ"); 52 | return NGX_ERROR; 53 | } 54 | 55 | b->pos += sizeof(uint8_t); 56 | 57 | /* check RDS format version number */ 58 | 59 | if (*(uint32_t *) b->pos != (uint32_t) resty_dbd_stream_version) { 60 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 61 | "rds: RDS format version differ"); 62 | return NGX_ERROR; 63 | } 64 | 65 | dd("RDS format version: %d", (int) *(uint32_t *) b->pos); 66 | 67 | b->pos += sizeof(uint32_t); 68 | 69 | /* check RDS result type */ 70 | 71 | if (*b->pos != 0) { 72 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 73 | "rds: RDS result type must be 0 for now"); 74 | return NGX_ERROR; 75 | } 76 | 77 | b->pos++; 78 | 79 | /* save the standard error code */ 80 | 81 | header->std_errcode = *(uint16_t *) b->pos; 82 | 83 | b->pos += sizeof(uint16_t); 84 | 85 | /* save the driver-specific error code */ 86 | 87 | header->drv_errcode = *(uint16_t *) b->pos; 88 | 89 | b->pos += sizeof(uint16_t); 90 | 91 | /* save the error string length */ 92 | 93 | header->errstr.len = *(uint16_t *) b->pos; 94 | 95 | b->pos += sizeof(uint16_t); 96 | 97 | dd("errstr len: %d", (int) header->errstr.len); 98 | 99 | /* check the rest data's size */ 100 | 101 | rest = header->errstr.len 102 | + sizeof(uint64_t) /* affected rows */ 103 | + sizeof(uint64_t) /* insert id */ 104 | + sizeof(uint16_t) /* column count */ 105 | ; 106 | 107 | if (b->last - b->pos < rest) { 108 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 109 | "rds: header is incomplete in the buf"); 110 | return NGX_ERROR; 111 | } 112 | 113 | /* save the error string data */ 114 | 115 | header->errstr.data = b->pos; 116 | 117 | b->pos += header->errstr.len; 118 | 119 | /* save affected rows */ 120 | 121 | header->affected_rows = *(uint64_t *) b->pos; 122 | 123 | b->pos += sizeof(uint64_t); 124 | 125 | /* save insert id */ 126 | 127 | header->insert_id = *(uint64_t *) b->pos; 128 | 129 | b->pos += sizeof(uint64_t); 130 | 131 | /* save column count */ 132 | 133 | header->col_count = *(uint16_t *) b->pos; 134 | 135 | b->pos += sizeof(uint16_t); 136 | 137 | dd("saved column count: %d", (int) header->col_count); 138 | 139 | return NGX_OK; 140 | } 141 | 142 | 143 | static ngx_inline ngx_int_t 144 | ngx_http_rds_parse_col(ngx_http_request_t *r, ngx_buf_t *b, 145 | ngx_http_rds_column_t *col) 146 | { 147 | ssize_t rest; 148 | 149 | rest = sizeof(uint16_t) /* std col type */ 150 | + sizeof(uint16_t) /* driver col type */ 151 | + sizeof(uint16_t) /* col name str len */ 152 | ; 153 | 154 | if (b->last - b->pos < rest) { 155 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 156 | "rds: column spec is incomplete in the buf"); 157 | return NGX_ERROR; 158 | } 159 | 160 | /* save standard column type */ 161 | col->std_type = *(uint16_t *) b->pos; 162 | b->pos += sizeof(uint16_t); 163 | 164 | /* save driver-specific column type */ 165 | col->drv_type = *(uint16_t *) b->pos; 166 | b->pos += sizeof(uint16_t); 167 | 168 | /* read column name string length */ 169 | 170 | col->name.len = *(uint16_t *) b->pos; 171 | b->pos += sizeof(uint16_t); 172 | 173 | if (col->name.len == 0) { 174 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 175 | "rds_csv: column name empty"); 176 | return NGX_ERROR; 177 | } 178 | 179 | rest = col->name.len; 180 | 181 | if (b->last - b->pos < rest) { 182 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 183 | "rds: column name string is incomplete in the buf"); 184 | return NGX_ERROR; 185 | } 186 | 187 | /* save the column name string data */ 188 | 189 | col->name.data = ngx_palloc(r->pool, col->name.len); 190 | if (col->name.data == NULL) { 191 | return NGX_ERROR; 192 | } 193 | 194 | ngx_memcpy(col->name.data, b->pos, col->name.len); 195 | b->pos += col->name.len; 196 | 197 | dd("saved column name \"%.*s\" (len %d, offset %d)", 198 | (int) col->name.len, col->name.data, 199 | (int) col->name.len, (int) (b->pos - b->start)); 200 | 201 | return NGX_OK; 202 | } 203 | 204 | 205 | #endif /* NGX_HTTP_RDS_UTILS_H */ 206 | -------------------------------------------------------------------------------- /t/000_init.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | repeat_each(1); 7 | 8 | plan tests => repeat_each() * blocks(); 9 | 10 | $ENV{TEST_NGINX_POSTGRESQL_PORT} ||= 5432; 11 | $ENV{TEST_NGINX_POSTGRESQL_HOST} ||= '127.0.0.1'; 12 | 13 | $ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; 14 | $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; 15 | 16 | our $http_config = <<'_EOC_'; 17 | upstream database { 18 | drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql 19 | dbname=ngx_test user=ngx_test password=ngx_test; 20 | } 21 | _EOC_ 22 | 23 | our $http_config2 = <<'_EOC_'; 24 | upstream database { 25 | postgres_server $TEST_NGINX_POSTGRESQL_HOST:$TEST_NGINX_POSTGRESQL_PORT 26 | dbname=ngx_test user=ngx_test password=ngx_test; 27 | } 28 | _EOC_ 29 | 30 | worker_connections(128); 31 | no_shuffle(); 32 | run_tests(); 33 | 34 | no_diff(); 35 | 36 | __DATA__ 37 | 38 | === TEST 1: cats - drop table 39 | --- http_config eval: $::http_config 40 | --- config 41 | location = /init { 42 | drizzle_pass database; 43 | drizzle_query "DROP TABLE IF EXISTS cats"; 44 | } 45 | --- request 46 | GET /init 47 | --- error_code: 200 48 | --- timeout: 10 49 | 50 | 51 | 52 | === TEST 2: cats - create table 53 | --- http_config eval: $::http_config 54 | --- config 55 | location = /init { 56 | drizzle_pass database; 57 | drizzle_query "CREATE TABLE cats (id integer, name text)"; 58 | } 59 | --- request 60 | GET /init 61 | --- error_code: 200 62 | --- timeout: 10 63 | 64 | 65 | 66 | === TEST 3: cats - insert value 67 | --- http_config eval: $::http_config 68 | --- config 69 | location = /init { 70 | drizzle_pass database; 71 | drizzle_query "INSERT INTO cats (id) VALUES (2)"; 72 | } 73 | --- request 74 | GET /init 75 | --- error_code: 200 76 | --- timeout: 10 77 | 78 | 79 | 80 | === TEST 4: cats - insert value 81 | --- http_config eval: $::http_config 82 | --- config 83 | location = /init { 84 | drizzle_pass database; 85 | drizzle_query "INSERT INTO cats (id, name) VALUES (3, 'bob')"; 86 | } 87 | --- request 88 | GET /init 89 | --- error_code: 200 90 | --- timeout: 10 91 | 92 | 93 | 94 | === TEST 5: dogs - drop table 95 | --- http_config eval: $::http_config 96 | --- config 97 | location = /init { 98 | drizzle_pass database; 99 | drizzle_query "DROP TABLE IF EXISTS dogs"; 100 | } 101 | --- request 102 | GET /init 103 | --- error_code: 200 104 | --- timeout: 10 105 | 106 | 107 | 108 | === TEST 6: dogs - create table 109 | --- http_config eval: $::http_config 110 | --- config 111 | location = /init { 112 | drizzle_pass database; 113 | drizzle_query "CREATE TABLE dogs (male boolean, name text, height real)"; 114 | } 115 | --- request 116 | GET /init 117 | --- error_code: 200 118 | --- timeout: 10 119 | 120 | 121 | 122 | === TEST 7: dogs - insert values 123 | --- http_config eval: $::http_config 124 | --- config 125 | location = /init { 126 | drizzle_pass database; 127 | drizzle_query " 128 | INSERT INTO dogs (male, name, height) 129 | VALUES 130 | (false, 'hello \"tom', 3.14), 131 | (true, 'hi,ya', -3), 132 | (false, 'hey\\ndad', 7), 133 | (false, '\\rkay', 0.005), 134 | (false, 'ab;c', 0.005), 135 | (false, 'foo\\tbar', 21);"; 136 | } 137 | --- request 138 | GET /init 139 | --- error_code: 200 140 | --- timeout: 10 141 | 142 | 143 | 144 | === TEST 8: birds - drop table 145 | --- http_config eval: $::http_config 146 | --- config 147 | location = /init { 148 | drizzle_pass database; 149 | drizzle_query "DROP TABLE IF EXISTS birds"; 150 | } 151 | --- request 152 | GET /init 153 | --- error_code: 200 154 | --- timeout: 10 155 | 156 | 157 | 158 | === TEST 9: birds - create table 159 | --- http_config eval: $::http_config 160 | --- config 161 | location = /init { 162 | drizzle_pass database; 163 | drizzle_query "CREATE TABLE birds (`\"name\"` text, height real)"; 164 | } 165 | --- request 166 | GET /init 167 | --- error_code: 200 168 | --- timeout: 10 169 | 170 | 171 | 172 | === TEST 10: birds - insert values 173 | --- http_config eval: $::http_config 174 | --- config 175 | location = /init { 176 | drizzle_pass database; 177 | drizzle_query " 178 | INSERT INTO birds (`\"name\"`, height) 179 | VALUES 180 | ('hello \"tom', 3.14), 181 | ('hi,ya', -3), 182 | ('hey\\ndad', 7), 183 | ('\\rkay', 0.005), 184 | ('ab;c', 0.005), 185 | ('foo\\tbar', 21);"; 186 | } 187 | --- request 188 | GET /init 189 | --- error_code: 200 190 | --- timeout: 10 191 | 192 | 193 | 194 | === TEST 11: cats - drop table 195 | --- http_config eval: $::http_config2 196 | --- config 197 | location = /init { 198 | postgres_pass database; 199 | postgres_query "DROP TABLE cats"; 200 | error_page 500 = /ignore; 201 | } 202 | 203 | location /ignore { echo "ignore"; } 204 | --- request 205 | GET /init 206 | --- error_code: 200 207 | --- timeout: 10 208 | 209 | 210 | 211 | === TEST 12: cats - create table 212 | --- http_config eval: $::http_config2 213 | --- config 214 | location = /init { 215 | postgres_pass database; 216 | postgres_query "CREATE TABLE cats (id integer, name text)"; 217 | } 218 | --- request 219 | GET /init 220 | --- error_code: 200 221 | --- timeout: 10 222 | 223 | 224 | 225 | === TEST 13: cats - insert value 226 | --- http_config eval: $::http_config2 227 | --- config 228 | location = /init { 229 | postgres_pass database; 230 | postgres_query "INSERT INTO cats (id) VALUES (2)"; 231 | } 232 | --- request 233 | GET /init 234 | --- error_code: 200 235 | --- timeout: 10 236 | 237 | 238 | 239 | === TEST 14: cats - insert value 240 | --- http_config eval: $::http_config2 241 | --- config 242 | location = /init { 243 | postgres_pass database; 244 | postgres_query "INSERT INTO cats (id, name) VALUES (3, 'bob')"; 245 | } 246 | --- request 247 | GET /init 248 | --- error_code: 200 249 | --- timeout: 10 250 | 251 | -------------------------------------------------------------------------------- /t/buf.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | repeat_each(1); 7 | 8 | plan tests => repeat_each() * 2 * blocks(); 9 | 10 | no_long_string(); 11 | 12 | run_tests(); 13 | 14 | #no_diff(); 15 | 16 | __DATA__ 17 | 18 | === TEST 1: rds in a single buf (empty result set) 19 | --- config 20 | location = /single { 21 | default_type 'application/x-resty-dbd-stream'; 22 | set_unescape_uri $rds $arg_rds; 23 | echo_duplicate 1 $rds; 24 | rds_csv on; 25 | } 26 | --- request eval 27 | my $rds = "\x{00}". # endian 28 | "\x{03}\x{00}\x{00}\x{00}". # format version 0.0.3 29 | "\x{00}". # result type 30 | "\x{00}\x{00}". # std errcode 31 | "\x{00}\x{00}" . # driver errcode 32 | "\x{00}\x{00}". # driver errstr len 33 | "". # driver errstr data 34 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # rows affected 35 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # insert id 36 | "\x{02}\x{00}". # col count 37 | "\x{01}\x{00}". # std col type (bigint/int) 38 | "\x{03}\x{00}". # drizzle col type 39 | "\x{02}\x{00}". # col name len 40 | "id". # col name data 41 | "\x{13}\x{80}". # std col type (blob/str) 42 | "\x{fc}\x{00}". # drizzle col type 43 | "\x{04}\x{00}". # col name len 44 | "name". # col name data 45 | "\x{00}"; # row list terminator 46 | 47 | use URI::Escape; 48 | $rds = uri_escape($rds); 49 | "GET /single?rds=$rds" 50 | --- response_body eval 51 | "id,name\r 52 | " 53 | 54 | 55 | 56 | === TEST 2: rds in a single buf (non-empty result set) 57 | --- config 58 | location = /single { 59 | default_type 'application/x-resty-dbd-stream'; 60 | set_unescape_uri $rds $arg_rds; 61 | echo_duplicate 1 $rds; 62 | rds_csv on; 63 | } 64 | --- request eval 65 | my $rds = 66 | "\x{00}". # endian 67 | "\x{03}\x{00}\x{00}\x{00}". # format version 0.0.3 68 | "\x{00}". # result type 69 | "\x{00}\x{00}". # std errcode 70 | "\x{00}\x{00}" . # driver errcode 71 | "\x{00}\x{00}". # driver errstr len 72 | "". # driver errstr data 73 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # rows affected 74 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # insert id 75 | "\x{02}\x{00}". # col count 76 | "\x{01}\x{00}". # std col type (bigint/int) 77 | "\x{03}\x{00}". # drizzle col type 78 | "\x{02}\x{00}". # col name len 79 | "id". # col name data 80 | "\x{13}\x{80}". # std col type (blob/str) 81 | "\x{fc}\x{00}". # drizzle col type 82 | "\x{04}\x{00}". # col name len 83 | "name". # col name data 84 | "\x{01}". # valid row flag 85 | "\x{01}\x{00}\x{00}\x{00}". # field len 86 | "2". # field data 87 | "\x{ff}\x{ff}\x{ff}\x{ff}". # field len 88 | "". # field data 89 | "\x{01}". # valid row flag 90 | "\x{01}\x{00}\x{00}\x{00}". # field len 91 | "3". # field data 92 | "\x{03}\x{00}\x{00}\x{00}". # field len 93 | "bob". # field data 94 | "\x{00}"; # row list terminator 95 | 96 | use URI::Escape; 97 | $rds = uri_escape($rds); 98 | "GET /single?rds=$rds" 99 | --- response_body eval 100 | "id,name\r 101 | 2,\r 102 | 3,bob\r 103 | " 104 | 105 | 106 | 107 | === TEST 3: rds in a single buf (non-empty result set, and each row in a single buf) 108 | --- config 109 | location = /single { 110 | default_type 'application/x-resty-dbd-stream'; 111 | 112 | set_unescape_uri $a $arg_a; 113 | set_unescape_uri $b $arg_b; 114 | set_unescape_uri $c $arg_c; 115 | set_unescape_uri $d $arg_d; 116 | 117 | echo_duplicate 1 $a; 118 | echo_duplicate 1 $b; 119 | echo_duplicate 1 $c; 120 | echo_duplicate 1 $d; 121 | 122 | rds_csv on; 123 | } 124 | --- request eval 125 | my $a = 126 | "\x{00}". # endian 127 | "\x{03}\x{00}\x{00}\x{00}". # format version 0.0.3 128 | "\x{00}". # result type 129 | "\x{00}\x{00}". # std errcode 130 | "\x{00}\x{00}" . # driver errcode 131 | "\x{00}\x{00}". # driver errstr len 132 | "". # driver errstr data 133 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # rows affected 134 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # insert id 135 | "\x{02}\x{00}". # col count 136 | "\x{01}\x{00}". # std col type (bigint/int) 137 | "\x{03}\x{00}". # drizzle col type 138 | "\x{02}\x{00}". # col name len 139 | "id". # col name data 140 | "\x{13}\x{80}". # std col type (blob/str) 141 | "\x{fc}\x{00}". # drizzle col type 142 | "\x{04}\x{00}". # col name len 143 | "name"; # col name data 144 | 145 | my $b = 146 | "\x{01}". # valid row flag 147 | "\x{01}\x{00}\x{00}\x{00}". # field len 148 | "2". # field data 149 | "\x{ff}\x{ff}\x{ff}\x{ff}". # field len 150 | ""; # field data 151 | 152 | my $c = 153 | "\x{01}". # valid row flag 154 | "\x{01}\x{00}\x{00}\x{00}". # field len 155 | "3". # field data 156 | "\x{03}\x{00}\x{00}\x{00}". # field len 157 | "bob"; # field data 158 | 159 | my $d = 160 | "\x{00}"; # row list terminator 161 | 162 | use URI::Escape; 163 | 164 | $a = uri_escape($a); 165 | $b = uri_escape($b); 166 | $c = uri_escape($c); 167 | $d = uri_escape($d); 168 | 169 | "GET /single?a=$a&b=$b&c=$c&d=$d" 170 | --- response_body eval 171 | "id,name\r 172 | 2,\r 173 | 3,bob\r 174 | " 175 | 176 | 177 | 178 | === TEST 4: rds in a single buf (non-empty result set, and each row in a single buf) 179 | --- config 180 | location = /single { 181 | default_type 'application/x-resty-dbd-stream'; 182 | 183 | set_unescape_uri $a $arg_a; 184 | set_unescape_uri $b $arg_b; 185 | set_unescape_uri $c $arg_c; 186 | 187 | echo -n $a; 188 | echo -n $b; 189 | echo -n $c; 190 | 191 | rds_csv on; 192 | } 193 | --- request eval 194 | my $a = 195 | "\x{00}". # endian 196 | "\x{03}\x{00}\x{00}\x{00}". # format version 0.0.3 197 | "\x{00}". # result type 198 | "\x{00}\x{00}". # std errcode 199 | "\x{00}\x{00}" . # driver errcode 200 | "\x{00}\x{00}". # driver errstr len 201 | "". # driver errstr data 202 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # rows affected 203 | "\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}\x{00}". # insert id 204 | "\x{02}\x{00}". # col count 205 | "\x{01}\x{00}". # std col type (bigint/int) 206 | "\x{03}\x{00}". # drizzle col type 207 | "\x{02}\x{00}". # col name len 208 | "id". # col name data 209 | "\x{13}\x{80}". # std col type (blob/str) 210 | "\x{fc}\x{00}". # drizzle col type 211 | "\x{04}\x{00}". # col name len 212 | "name"; # col name data 213 | 214 | my $b = 215 | "\x{01}". # valid row flag 216 | "\x{01}\x{00}\x{00}\x{00}". # field len 217 | "2". # field data 218 | "\x{ff}\x{ff}\x{ff}\x{ff}". # field len 219 | ""; # field data 220 | 221 | my $c = 222 | "\x{01}". # valid row flag 223 | "\x{01}\x{00}\x{00}\x{00}". # field len 224 | "3". # field data 225 | "\x{03}\x{00}\x{00}\x{00}". # field len 226 | "bob". # field data 227 | "\x{00}"; # row list terminator 228 | 229 | use URI::Escape; 230 | 231 | $a = uri_escape($a); 232 | $b = uri_escape($b); 233 | $c = uri_escape($c); 234 | 235 | "GET /single?a=$a&b=$b&c=$c" 236 | --- response_body eval 237 | "id,name\r 238 | 2,\r 239 | 3,bob\r 240 | " 241 | 242 | -------------------------------------------------------------------------------- /valgrind.suppress: -------------------------------------------------------------------------------- 1 | { 2 | 3 | Memcheck:Addr4 4 | fun:ngx_init_cycle 5 | fun:ngx_master_process_cycle 6 | fun:main 7 | } 8 | { 9 | 10 | Memcheck:Leak 11 | fun:malloc 12 | fun:ngx_alloc 13 | obj:* 14 | } 15 | { 16 | 17 | Memcheck:Param 18 | epoll_ctl(event) 19 | fun:epoll_ctl 20 | } 21 | { 22 | 23 | Memcheck:Leak 24 | fun:malloc 25 | fun:ngx_alloc 26 | fun:ngx_event_process_init 27 | } 28 | { 29 | nginx-core-process-init 30 | Memcheck:Leak 31 | fun:malloc 32 | fun:ngx_alloc 33 | fun:ngx_event_process_init 34 | } 35 | { 36 | nginx-core-crc32-init 37 | Memcheck:Leak 38 | fun:malloc 39 | fun:ngx_alloc 40 | fun:ngx_crc32_table_init 41 | fun:main 42 | } 43 | { 44 | palloc_large_for_init_request 45 | Memcheck:Leak 46 | fun:malloc 47 | fun:ngx_alloc 48 | fun:ngx_palloc_large 49 | fun:ngx_palloc 50 | fun:ngx_pcalloc 51 | fun:ngx_http_init_request 52 | fun:ngx_epoll_process_events 53 | fun:ngx_process_events_and_timers 54 | } 55 | { 56 | palloc_large_for_create_temp_buf 57 | Memcheck:Leak 58 | fun:malloc 59 | fun:ngx_alloc 60 | fun:ngx_palloc_large 61 | fun:ngx_palloc 62 | fun:ngx_create_temp_buf 63 | fun:ngx_http_init_request 64 | fun:ngx_epoll_process_events 65 | fun:ngx_process_events_and_timers 66 | } 67 | { 68 | accept_create_pool 69 | Memcheck:Leak 70 | fun:memalign 71 | fun:posix_memalign 72 | fun:ngx_memalign 73 | fun:ngx_create_pool 74 | fun:ngx_event_accept 75 | fun:ngx_epoll_process_events 76 | fun:ngx_process_events_and_timers 77 | } 78 | { 79 | create_pool_for_init_req 80 | Memcheck:Leak 81 | fun:memalign 82 | fun:posix_memalign 83 | fun:ngx_memalign 84 | fun:ngx_create_pool 85 | fun:ngx_http_init_request 86 | fun:ngx_epoll_process_events 87 | fun:ngx_process_events_and_timers 88 | } 89 | { 90 | create_pool_posix_memalign 91 | Memcheck:Leak 92 | fun:memalign 93 | fun:posix_memalign 94 | fun:ngx_memalign 95 | fun:ngx_create_pool 96 | fun:main 97 | } 98 | { 99 | spawn_process_alloc 100 | Memcheck:Leak 101 | fun:malloc 102 | fun:ngx_alloc 103 | fun:ngx_event_process_init 104 | fun:ngx_worker_process_init 105 | fun:ngx_worker_process_cycle 106 | fun:ngx_spawn_process 107 | fun:ngx_start_worker_processes 108 | fun:ngx_master_process_cycle 109 | fun:main 110 | } 111 | { 112 | 113 | Memcheck:Leak 114 | fun:malloc 115 | fun:ngx_alloc 116 | fun:ngx_palloc_large 117 | fun:ngx_palloc 118 | fun:ngx_array_push 119 | fun:ngx_hash_add_key 120 | fun:ngx_http_variables_add_core_vars 121 | fun:ngx_http_core_preconfiguration 122 | fun:ngx_http_block 123 | fun:ngx_conf_parse 124 | fun:ngx_init_cycle 125 | fun:main 126 | } 127 | { 128 | 129 | Memcheck:Leak 130 | fun:malloc 131 | fun:ngx_alloc 132 | fun:ngx_palloc_large 133 | fun:ngx_palloc 134 | fun:ngx_pcalloc 135 | fun:ngx_hash_init 136 | fun:ngx_http_variables_init_vars 137 | fun:ngx_http_block 138 | fun:ngx_conf_parse 139 | fun:ngx_init_cycle 140 | fun:main 141 | } 142 | { 143 | 144 | Memcheck:Leak 145 | fun:malloc 146 | fun:ngx_alloc 147 | fun:ngx_palloc_large 148 | fun:ngx_palloc 149 | fun:ngx_pcalloc 150 | fun:ngx_hash_keys_array_init 151 | fun:ngx_http_variables_add_core_vars 152 | fun:ngx_http_core_preconfiguration 153 | fun:ngx_http_block 154 | fun:ngx_conf_parse 155 | fun:ngx_init_cycle 156 | fun:main 157 | } 158 | { 159 | 160 | Memcheck:Leak 161 | fun:malloc 162 | fun:ngx_alloc 163 | fun:ngx_palloc_large 164 | fun:ngx_palloc 165 | fun:ngx_array_push 166 | fun:ngx_hash_add_key 167 | fun:ngx_http_add_variable 168 | fun:ngx_http_ssi_preconfiguration 169 | fun:ngx_http_block 170 | fun:ngx_conf_parse 171 | fun:ngx_init_cycle 172 | fun:main 173 | } 174 | { 175 | 176 | Memcheck:Leak 177 | fun:malloc 178 | fun:ngx_alloc 179 | fun:ngx_palloc_large 180 | fun:ngx_palloc 181 | fun:ngx_pcalloc 182 | fun:ngx_http_upstream_drizzle_create_srv_conf 183 | fun:ngx_http_core_server 184 | fun:ngx_conf_parse 185 | fun:ngx_http_block 186 | fun:ngx_conf_parse 187 | fun:ngx_init_cycle 188 | fun:main 189 | } 190 | { 191 | 192 | Memcheck:Leak 193 | fun:malloc 194 | fun:ngx_alloc 195 | fun:ngx_palloc_large 196 | fun:ngx_palloc 197 | fun:ngx_pcalloc 198 | fun:ngx_http_upstream_drizzle_create_srv_conf 199 | fun:ngx_http_upstream 200 | fun:ngx_conf_parse 201 | fun:ngx_http_block 202 | fun:ngx_conf_parse 203 | fun:ngx_init_cycle 204 | fun:main 205 | } 206 | { 207 | 208 | Memcheck:Leak 209 | fun:malloc 210 | fun:ngx_alloc 211 | fun:ngx_palloc_large 212 | fun:ngx_palloc 213 | fun:ngx_pcalloc 214 | fun:ngx_http_upstream_drizzle_create_srv_conf 215 | fun:ngx_http_block 216 | fun:ngx_conf_parse 217 | fun:ngx_init_cycle 218 | fun:main 219 | } 220 | { 221 | 222 | Memcheck:Leak 223 | fun:malloc 224 | fun:ngx_alloc 225 | fun:ngx_palloc_large 226 | fun:ngx_palloc 227 | fun:ngx_pcalloc 228 | fun:ngx_init_cycle 229 | fun:main 230 | } 231 | { 232 | 233 | Memcheck:Leak 234 | fun:malloc 235 | fun:ngx_alloc 236 | fun:ngx_palloc_large 237 | fun:ngx_palloc 238 | fun:ngx_hash_init 239 | fun:ngx_http_upstream_init_main_conf 240 | fun:ngx_http_block 241 | fun:ngx_conf_parse 242 | fun:ngx_init_cycle 243 | fun:main 244 | } 245 | { 246 | 247 | Memcheck:Leak 248 | fun:malloc 249 | fun:ngx_alloc 250 | fun:ngx_palloc_large 251 | fun:ngx_palloc 252 | fun:ngx_array_push 253 | fun:ngx_http_get_variable_index 254 | fun:ngx_http_script_compile 255 | fun:ngx_http_rewrite_value 256 | fun:ndk_set_var_value_core 257 | fun:ndk_set_var_value 258 | fun:ngx_conf_parse 259 | fun:ngx_http_core_location 260 | } 261 | { 262 | 263 | Memcheck:Leak 264 | fun:malloc 265 | fun:ngx_alloc 266 | fun:ngx_palloc_large 267 | fun:ngx_palloc 268 | fun:ngx_array_push_n 269 | fun:ngx_http_script_add_code 270 | fun:ngx_http_script_compile 271 | fun:ngx_http_rewrite_value 272 | fun:ndk_set_var_value_core 273 | fun:ndk_set_var_value 274 | fun:ngx_conf_parse 275 | fun:ngx_http_core_location 276 | } 277 | { 278 | 279 | Memcheck:Leak 280 | fun:malloc 281 | fun:ngx_alloc 282 | fun:ngx_palloc_large 283 | fun:ngx_palloc 284 | fun:ngx_array_push 285 | fun:ngx_http_get_variable_index 286 | fun:ngx_http_rewrite_set 287 | fun:ngx_conf_parse 288 | fun:ngx_http_core_location 289 | fun:ngx_conf_parse 290 | fun:ngx_http_core_server 291 | fun:ngx_conf_parse 292 | } 293 | { 294 | 295 | Memcheck:Cond 296 | fun:index 297 | fun:expand_dynamic_string_token 298 | fun:_dl_map_object 299 | fun:map_doit 300 | fun:_dl_catch_error 301 | fun:do_preload 302 | fun:dl_main 303 | } 304 | { 305 | 306 | Memcheck:Leak 307 | match-leak-kinds: definite 308 | fun:malloc 309 | fun:ngx_alloc 310 | fun:ngx_set_environment 311 | fun:ngx_single_process_cycle 312 | } 313 | { 314 | 315 | Memcheck:Leak 316 | match-leak-kinds: definite 317 | fun:malloc 318 | fun:ngx_alloc 319 | fun:ngx_set_environment 320 | fun:ngx_worker_process_init 321 | fun:ngx_worker_process_cycle 322 | } 323 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Name 2 | ==== 3 | 4 | ngx_rds_csv - Nginx output filter module to convert Resty-DBD-Streams 5 | (RDS) to Comma-Separated Values (CSV) 6 | 7 | Table of Contents 8 | ================= 9 | 10 | * [Name](#name) 11 | * [Status](#status) 12 | * [Synopsis](#synopsis) 13 | * [Description](#description) 14 | * [Directives](#directives) 15 | * [rds_csv](#rds_csv) 16 | * [rds_csv_row_terminator](#rds_csv_row_terminator) 17 | * [rds_csv_field_separator](#rds_csv_field_separator) 18 | * [rds_csv_field_name_header](#rds_csv_field_name_header) 19 | * [rds_csv_content_type](#rds_csv_content_type) 20 | * [rds_csv_buffer_size](#rds_csv_buffer_size) 21 | * [Installation](#installation) 22 | * [Compatibility](#compatibility) 23 | * [Author](#author) 24 | * [Copyright & License](#copyright--license) 25 | * [See Also](#see-also) 26 | 27 | Status 28 | ====== 29 | 30 | This module is considered production ready. 31 | 32 | Synopsis 33 | ======== 34 | 35 | ```nginx 36 | location /foo { 37 | # drizzle_pass/postgres_pass/... 38 | 39 | rds_csv on; 40 | rds_csv_row_terminator "\n"; # default to "\r\n" 41 | } 42 | ``` 43 | 44 | Description 45 | =========== 46 | 47 | This module implements an efficient output filter that converts 48 | Resty-DBD-Streams (RDS) generated by [ngx_drizzle](https://github.com/openresty/drizzle-nginx-module) 49 | and [ngx_postgres](https://github.com/FRiCKLE/ngx_postgres/) 50 | to Comma-Separated Values (CSV) format in a streaming fashion. 51 | By default, the CSV format is in compliance with [RFC 4180](http://tools.ietf.org/html/rfc4180): 52 | 53 | http://tools.ietf.org/html/rfc4180 54 | 55 | SQL NULL values will be converted to empty field value, just like 56 | empty string values. 57 | 58 | [Back to TOC](#table-of-contents) 59 | 60 | Directives 61 | ========== 62 | 63 | [Back to TOC](#table-of-contents) 64 | 65 | rds_csv 66 | ------- 67 | **syntax:** *rds_csv on|off* 68 | 69 | **default:** *rds_csv off* 70 | 71 | **context:** *http, server, location, location if* 72 | 73 | Enables this output filter when on and disables otherwise. 74 | 75 | [Back to TOC](#table-of-contents) 76 | 77 | rds_csv_row_terminator 78 | ---------------------- 79 | 80 | **syntax:** *rds_csv_row_terminator <str>* 81 | 82 | **default:** *rds_csv_row_terminator "\r\n"* 83 | 84 | **context:** *http, server, location, location if* 85 | 86 | Specifies the row terminator used by the CSV format. 87 | Only `"\r\n"` and `"\n"` are allowed. 88 | 89 | Defaults to `"\r\n"`, i.e., CR LF, according to [RFC 4180](http://tools.ietf.org/html/rfc4180). 90 | 91 | [Back to TOC](#table-of-contents) 92 | 93 | rds_csv_field_separator 94 | ----------------------- 95 | 96 | **syntax:** *rds_csv_field_separator <char>* 97 | 98 | **default:** *rds_csv_field_separator ","* 99 | 100 | **context:** *http, server, location, location if* 101 | 102 | Specifies the field seperator used by the CSV format. 103 | Only `","`, `";"`, and `"\t"` are allowed. 104 | 105 | Defaults to `","` according to [RFC 4180](http://tools.ietf.org/html/rfc4180). 106 | 107 | [Back to TOC](#table-of-contents) 108 | 109 | rds_csv_field_name_header 110 | ------------------------- 111 | 112 | **syntax:** *rds_csv_field_name_header on|off* 113 | 114 | **default:** *rds_csv_field_name_header off* 115 | 116 | **context:** *http, server, location, location if* 117 | 118 | Emits the first line of field names when this directive is set on, 119 | and none otherwise. 120 | 121 | [Back to TOC](#table-of-contents) 122 | 123 | rds_csv_content_type 124 | -------------------- 125 | **syntax:** *rds_csv_content_type <str>* 126 | 127 | **default:** *rds_csv_content_type "text/csv; header=<present|absence>"* 128 | 129 | **context:** *http, server, location, location if* 130 | 131 | Specifies the `Content-Type` response header generated by this module. 132 | 133 | Defaults to `"text/csv; header=present"` or `"text/csv; header=absence"`, 134 | depending on whether [rds_csv_field_name_header](#rds_csv_field_name_header) is on or off. 135 | 136 | [Back to TOC](#table-of-contents) 137 | 138 | rds_csv_buffer_size 139 | ------------------- 140 | **syntax:** *rds_csv_buffer_size <size>* 141 | 142 | **default:** *rds_csv_buffer_size 4k/8k* 143 | 144 | **context:** *http, server, location, location if* 145 | 146 | The lager this buffer size setting, the less streammy the output 147 | will be. 148 | 149 | [Back to TOC](#table-of-contents) 150 | 151 | Installation 152 | ============ 153 | 154 | You're recommended to install this module (as well as the Nginx core and many other goodies) via the [ngx_openresty bundle](http://openresty.org). See [the detailed instructions](http://openresty.org/#Installation) for downloading and installing ngx_openresty into your system. This is the easiest and most safe way to set things up. 155 | 156 | Alternatively, you can install this module manually with the Nginx source: 157 | 158 | Grab the nginx source code from [nginx.org](http://nginx.org/), for example, 159 | the version 1.13.6 (see [nginx compatibility](#compatibility)), and then build the source with this module: 160 | 161 | ```bash 162 | 163 | $ wget 'http://nginx.org/download/nginx-1.13.6.tar.gz' 164 | $ tar -xzvf nginx-1.13.6.tar.gz 165 | $ cd nginx-1.13.6/ 166 | 167 | # Here we assume you would install you nginx under /opt/nginx/. 168 | $ ./configure --prefix=/opt/nginx \ 169 | --add-module=/path/to/rds-csv-nginx-module 170 | 171 | $ make -j2 172 | $ make install 173 | ``` 174 | 175 | Download the latest version of the release tarball of this module from [rds-csv-nginx-module file list](https://github.com/openresty/rds-csv-nginx-module/tags). 176 | 177 | Also, this module is included and enabled by default in the [ngx_openresty bundle](http://openresty.org). 178 | 179 | [Back to TOC](#table-of-contents) 180 | 181 | Compatibility 182 | ============= 183 | 184 | This module is compatible with the following versions of Nginx: 185 | 186 | * **1.13.x** (last tested: 1.13.6) 187 | * **1.12.x** 188 | * **1.11.x** (last tested: 1.11.2) 189 | * **1.10.x** 190 | * **1.9.x** (last tested: 1.9.7) 191 | * **1.8.x** 192 | * **1.7.x** (last tested: 1.7.10) 193 | * **1.6.x** 194 | * **1.5.x** 195 | * **1.4.x** (last tested: 1.4.3) 196 | * **1.2.x** (last tested: 1.2.9) 197 | * **1.1.x** (last tested: 1.1.5) 198 | * **1.0.x** (last tested: 1.0.8) 199 | 200 | [Back to TOC](#table-of-contents) 201 | 202 | Author 203 | ====== 204 | Yichun "agentzh" Zhang <agentzh@gmail.com>, OpenResty Inc. 205 | 206 | [Back to TOC](#table-of-contents) 207 | 208 | Copyright & License 209 | =================== 210 | This module is licenced under the BSD license. 211 | 212 | Copyright (C) 2011-2018, Yichun "agentzh" Zhang (章亦春) <agentzh@gmail.com>, OpenResty Inc. 213 | 214 | All rights reserved. 215 | 216 | Redistribution and use in source and binary forms, with or without 217 | modification, are permitted provided that the following conditions 218 | are met: 219 | 220 | * Redistributions of source code must retain the above copyright 221 | notice, this list of conditions and the following disclaimer. 222 | * Redistributions in binary form must reproduce the above copyright 223 | notice, this list of conditions and the following disclaimer in the 224 | documentation and/or other materials provided with the distribution. 225 | 226 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 227 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 228 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 229 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 230 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 231 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 232 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 233 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 234 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 235 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 236 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 237 | 238 | [Back to TOC](#table-of-contents) 239 | 240 | See Also 241 | ======== 242 | 243 | * [ngx_drizzle](https://github.com/openresty/drizzle-nginx-module) 244 | * [ngx_postgres](https://github.com/FRiCKLE/ngx_postgres/) 245 | * [ngx_rds_json](https://github.com/openresty/rds-json-nginx-module) 246 | 247 | [Back to TOC](#table-of-contents) 248 | 249 | -------------------------------------------------------------------------------- /t/escape.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * 3 * blocks(); 9 | 10 | $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; 11 | 12 | our $http_config = <<'_EOC_'; 13 | upstream backend { 14 | drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql 15 | dbname=ngx_test user=ngx_test password=ngx_test; 16 | } 17 | _EOC_ 18 | 19 | #no_diff(); 20 | no_long_string(); 21 | 22 | run_tests(); 23 | 24 | __DATA__ 25 | 26 | === TEST 1: sanity 27 | --- http_config eval: $::http_config 28 | --- config 29 | location /mysql { 30 | drizzle_query " 31 | select * from dogs order by name asc; 32 | "; 33 | drizzle_pass backend; 34 | rds_csv on; 35 | } 36 | --- request 37 | POST /mysql 38 | sql=select%20*%20from%20cats; 39 | --- response_headers 40 | Content-Type: text/csv; header=presence 41 | --- response_body eval 42 | qq{male,name,height\r 43 | 0,"\rkay",0.005\r 44 | 0,ab;c,0.005\r 45 | 0,foo\tbar,21\r 46 | 0,"hello ""tom",3.14\r 47 | 0,"hey 48 | dad",7\r 49 | 1,"hi,ya",-3\r 50 | } 51 | 52 | 53 | 54 | === TEST 2: using ; as the field separator 55 | --- http_config eval: $::http_config 56 | --- config 57 | location /mysql { 58 | drizzle_query " 59 | select * from dogs order by name asc; 60 | "; 61 | drizzle_pass backend; 62 | rds_csv_field_separator ';'; 63 | rds_csv on; 64 | } 65 | --- request 66 | POST /mysql 67 | sql=select%20*%20from%20cats; 68 | --- response_headers 69 | Content-Type: text/csv; header=presence 70 | --- response_body eval 71 | qq{male;name;height\r 72 | 0;"\rkay";0.005\r 73 | 0;"ab;c";0.005\r 74 | 0;foo\tbar;21\r 75 | 0;"hello ""tom";3.14\r 76 | 0;"hey 77 | dad";7\r 78 | 1;hi,ya;-3\r 79 | } 80 | 81 | 82 | 83 | === TEST 3: using tab as the field separator 84 | --- http_config eval: $::http_config 85 | --- config 86 | location /mysql { 87 | drizzle_query " 88 | select * from dogs order by name asc; 89 | "; 90 | drizzle_pass backend; 91 | rds_csv_field_separator '\t'; 92 | rds_csv on; 93 | } 94 | --- request 95 | POST /mysql 96 | sql=select%20*%20from%20cats; 97 | --- response_headers 98 | Content-Type: text/csv; header=presence 99 | --- response_body eval 100 | qq{male\tname\theight\r 101 | 0\t"\rkay"\t0.005\r 102 | 0\tab;c\t0.005\r 103 | 0\t"foo\tbar"\t21\r 104 | 0\t"hello ""tom"\t3.14\r 105 | 0\t"hey 106 | dad"\t7\r 107 | 1\thi,ya\t-3\r 108 | } 109 | 110 | 111 | 112 | === TEST 4: explicitly using \r\n as the row terminator 113 | --- http_config eval: $::http_config 114 | --- config 115 | location /mysql { 116 | drizzle_query " 117 | select * from dogs order by name asc; 118 | "; 119 | drizzle_pass backend; 120 | rds_csv_field_separator '\t'; 121 | rds_csv_row_terminator '\r\n'; 122 | rds_csv on; 123 | } 124 | --- request 125 | POST /mysql 126 | sql=select%20*%20from%20cats; 127 | --- response_headers 128 | Content-Type: text/csv; header=presence 129 | --- response_body eval 130 | qq{male\tname\theight\r 131 | 0\t"\rkay"\t0.005\r 132 | 0\tab;c\t0.005\r 133 | 0\t"foo\tbar"\t21\r 134 | 0\t"hello ""tom"\t3.14\r 135 | 0\t"hey 136 | dad"\t7\r 137 | 1\thi,ya\t-3\r 138 | } 139 | 140 | 141 | 142 | === TEST 5: using \n as the row terminator 143 | --- http_config eval: $::http_config 144 | --- config 145 | location /mysql { 146 | drizzle_query " 147 | select * from dogs order by name asc; 148 | "; 149 | drizzle_pass backend; 150 | rds_csv_field_separator '\t'; 151 | rds_csv_row_terminator '\n'; 152 | rds_csv on; 153 | } 154 | --- request 155 | POST /mysql 156 | sql=select%20*%20from%20cats; 157 | --- response_headers 158 | Content-Type: text/csv; header=presence 159 | --- response_body eval 160 | qq{male\tname\theight 161 | 0\t"\rkay"\t0.005 162 | 0\tab;c\t0.005 163 | 0\t"foo\tbar"\t21 164 | 0\t"hello ""tom"\t3.14 165 | 0\t"hey 166 | dad"\t7 167 | 1\thi,ya\t-3 168 | } 169 | 170 | 171 | 172 | === TEST 6: explicitly field name header on 173 | --- http_config eval: $::http_config 174 | --- config 175 | location /mysql { 176 | drizzle_query " 177 | select * from dogs order by name asc; 178 | "; 179 | drizzle_pass backend; 180 | rds_csv_field_separator '\t'; 181 | rds_csv_row_terminator '\n'; 182 | rds_csv_field_name_header on; 183 | rds_csv on; 184 | } 185 | --- request 186 | POST /mysql 187 | sql=select%20*%20from%20cats; 188 | --- response_headers 189 | Content-Type: text/csv; header=presence 190 | --- response_body eval 191 | qq{male\tname\theight 192 | 0\t"\rkay"\t0.005 193 | 0\tab;c\t0.005 194 | 0\t"foo\tbar"\t21 195 | 0\t"hello ""tom"\t3.14 196 | 0\t"hey 197 | dad"\t7 198 | 1\thi,ya\t-3 199 | } 200 | 201 | 202 | 203 | === TEST 7: explicitly field name header off 204 | --- http_config eval: $::http_config 205 | --- config 206 | location /mysql { 207 | drizzle_query " 208 | select * from dogs order by name asc; 209 | "; 210 | drizzle_pass backend; 211 | rds_csv_field_separator '\t'; 212 | rds_csv_row_terminator '\n'; 213 | rds_csv_field_name_header off; 214 | rds_csv on; 215 | } 216 | --- request 217 | POST /mysql 218 | sql=select%20*%20from%20cats; 219 | --- response_headers 220 | Content-Type: text/csv; header=absence 221 | --- response_body eval 222 | qq{0\t"\rkay"\t0.005 223 | 0\tab;c\t0.005 224 | 0\t"foo\tbar"\t21 225 | 0\t"hello ""tom"\t3.14 226 | 0\t"hey 227 | dad"\t7 228 | 1\thi,ya\t-3 229 | } 230 | 231 | 232 | 233 | === TEST 8: the "charset" directive does not affect us 234 | --- http_config eval: $::http_config 235 | --- config 236 | location /mysql { 237 | charset "gbk"; 238 | drizzle_query " 239 | select * from dogs order by name asc; 240 | "; 241 | drizzle_pass backend; 242 | rds_csv_field_separator '\t'; 243 | rds_csv_row_terminator '\n'; 244 | rds_csv_field_name_header off; 245 | rds_csv on; 246 | } 247 | --- request 248 | POST /mysql 249 | sql=select%20*%20from%20cats; 250 | --- response_headers 251 | Content-Type: text/csv; header=absence 252 | --- response_body eval 253 | qq{0\t"\rkay"\t0.005 254 | 0\tab;c\t0.005 255 | 0\t"foo\tbar"\t21 256 | 0\t"hello ""tom"\t3.14 257 | 0\t"hey 258 | dad"\t7 259 | 1\thi,ya\t-3 260 | } 261 | 262 | 263 | 264 | === TEST 9: set content type 265 | --- http_config eval: $::http_config 266 | --- config 267 | location /mysql { 268 | drizzle_query " 269 | select * from dogs order by name asc; 270 | "; 271 | drizzle_pass backend; 272 | rds_csv_field_separator '\t'; 273 | rds_csv_row_terminator '\n'; 274 | rds_csv_field_name_header off; 275 | rds_csv_content_type "text/tab-separated-values"; 276 | rds_csv on; 277 | } 278 | --- request 279 | GET /mysql 280 | --- response_headers 281 | Content-Type: text/tab-separated-values 282 | --- response_body eval 283 | qq{0\t"\rkay"\t0.005 284 | 0\tab;c\t0.005 285 | 0\t"foo\tbar"\t21 286 | 0\t"hello ""tom"\t3.14 287 | 0\t"hey 288 | dad"\t7 289 | 1\thi,ya\t-3 290 | } 291 | 292 | 293 | 294 | === TEST 10: set content type (bigger buffer size) 295 | --- http_config eval: $::http_config 296 | --- config 297 | location /mysql { 298 | drizzle_query " 299 | select * from dogs order by name asc; 300 | "; 301 | drizzle_pass backend; 302 | rds_csv_field_separator '\t'; 303 | rds_csv_row_terminator '\n'; 304 | rds_csv_field_name_header off; 305 | rds_csv_content_type "text/tab-separated-values"; 306 | rds_csv on; 307 | rds_csv_buffer_size 8k; 308 | } 309 | --- request 310 | GET /mysql 311 | --- response_headers 312 | Content-Type: text/tab-separated-values 313 | --- response_body eval 314 | qq{0\t"\rkay"\t0.005 315 | 0\tab;c\t0.005 316 | 0\t"foo\tbar"\t21 317 | 0\t"hello ""tom"\t3.14 318 | 0\t"hey 319 | dad"\t7 320 | 1\thi,ya\t-3 321 | } 322 | 323 | 324 | 325 | === TEST 11: escaping column names 326 | --- http_config eval: $::http_config 327 | --- config 328 | location /mysql { 329 | drizzle_query " 330 | select `\"name\"`, height from birds order by height limit 1; 331 | "; 332 | drizzle_pass backend; 333 | rds_csv on; 334 | rds_csv_row_terminator "\n"; 335 | } 336 | --- request 337 | GET /mysql 338 | --- response_headers 339 | Content-Type: text/csv; header=presence 340 | --- response_body 341 | """name""",height 342 | "hi,ya",-3 343 | 344 | -------------------------------------------------------------------------------- /t/sanity.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | #repeat_each(10); 7 | no_shuffle(); 8 | 9 | repeat_each(2); 10 | 11 | plan tests => repeat_each() * (2 * blocks() + 7); 12 | 13 | $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; 14 | 15 | our $http_config = <<'_EOC_'; 16 | upstream backend { 17 | drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql 18 | dbname=ngx_test user=ngx_test password=ngx_test; 19 | } 20 | _EOC_ 21 | 22 | no_long_string(); 23 | no_diff(); 24 | 25 | run_tests(); 26 | 27 | __DATA__ 28 | 29 | === TEST 1: sanity 30 | --- http_config eval: $::http_config 31 | --- config 32 | location /mysql { 33 | drizzle_pass backend; 34 | #drizzle_dbname $dbname; 35 | drizzle_query 'select * from cats'; 36 | rds_csv on; 37 | rds_csv_row_terminator "\n"; 38 | rds_csv_field_name_header off; 39 | } 40 | --- request 41 | GET /mysql 42 | --- response_headers_like 43 | X-Resty-DBD-Module: ngx_drizzle \d+\.\d+\.\d+ 44 | Content-Type: text/csv; header=absence 45 | --- response_body 46 | 2, 47 | 3,bob 48 | --- timeout: 15 49 | 50 | 51 | 52 | === TEST 2: keep-alive 53 | --- http_config eval: $::http_config 54 | --- config 55 | location /mysql { 56 | drizzle_pass backend; 57 | #drizzle_dbname $dbname; 58 | drizzle_query 'select * from cats'; 59 | rds_csv on; 60 | rds_csv_row_terminator "\n"; 61 | } 62 | --- request 63 | GET /mysql 64 | --- response_body 65 | id,name 66 | 2, 67 | 3,bob 68 | 69 | 70 | 71 | === TEST 3: update 72 | --- http_config eval: $::http_config 73 | --- config 74 | location /mysql { 75 | drizzle_pass backend; 76 | #drizzle_dbname $dbname; 77 | drizzle_query "update cats set name='bob' where name='bob'"; 78 | rds_csv on; 79 | rds_csv_row_terminator "\n"; 80 | } 81 | --- request 82 | GET /mysql 83 | --- response_body 84 | errcode,errstr,insert_id,affected_rows 85 | 0,Rows matched: 1 Changed: 0 Warnings: 0,0,0 86 | 87 | 88 | 89 | === TEST 4: select empty result 90 | --- http_config eval: $::http_config 91 | --- config 92 | location /mysql { 93 | drizzle_pass backend; 94 | drizzle_query "select * from cats where name='tom'"; 95 | rds_csv on; 96 | } 97 | --- request 98 | GET /mysql 99 | --- response_body eval 100 | "id,name\r 101 | " 102 | 103 | 104 | 105 | === TEST 5: update & no module header 106 | --- http_config eval: $::http_config 107 | --- config 108 | location /mysql { 109 | if ($arg_name ~ '[^A-Za-z0-9]') { 110 | return 400; 111 | } 112 | 113 | drizzle_pass backend; 114 | drizzle_module_header off; 115 | drizzle_query "update cats set name='$arg_name' where name='$arg_name'"; 116 | 117 | rds_csv on; 118 | } 119 | --- request 120 | GET /mysql?name=bob 121 | --- response_headers 122 | X-Resty-DBD-Module: 123 | Content-Type: text/csv; header=presence 124 | --- response_body eval 125 | qq{errcode,errstr,insert_id,affected_rows\r 126 | 0,Rows matched: 1 Changed: 0 Warnings: 0,0,0\r 127 | } 128 | --- no_error_log 129 | [error] 130 | 131 | 132 | 133 | === TEST 6: invalid SQL 134 | --- http_config eval: $::http_config 135 | --- config 136 | location /mysql { 137 | drizzle_pass backend; 138 | drizzle_module_header off; 139 | drizzle_query "select '32"; 140 | rds_csv on; 141 | rds_csv_row_terminator "\n"; 142 | } 143 | --- response_headers 144 | X-Resty-DBD-Module: 145 | Content-Type: text/html 146 | --- request 147 | GET /mysql 148 | --- error_code: 500 149 | --- response_body_like: 500 Internal Server Error 150 | 151 | 152 | 153 | === TEST 7: single row, single col 154 | --- http_config eval: $::http_config 155 | --- config 156 | location /test { 157 | echo_location /mysql "drop table if exists singles"; 158 | echo_location /mysql "create table singles (name varchar(15));"; 159 | echo_location /mysql "insert into singles values ('marry');"; 160 | echo_location /mysql "select * from singles;"; 161 | } 162 | location /mysql { 163 | drizzle_pass backend; 164 | drizzle_module_header off; 165 | drizzle_query $query_string; 166 | rds_csv on; 167 | rds_csv_row_terminator "\n"; 168 | } 169 | --- request 170 | GET /test 171 | --- response_body 172 | errcode,errstr,insert_id,affected_rows 173 | 0,,0,0 174 | errcode,errstr,insert_id,affected_rows 175 | 0,,0,0 176 | errcode,errstr,insert_id,affected_rows 177 | 0,,0,1 178 | name 179 | marry 180 | --- skip_nginx: 2: < 0.7.46 181 | --- timeout: 5 182 | 183 | 184 | 185 | === TEST 8: floating number and insert id 186 | --- http_config eval: $::http_config 187 | --- config 188 | location /test { 189 | echo_location /mysql "drop table if exists foo"; 190 | echo_location /mysql "create table foo (id serial not null, primary key (id), val real);"; 191 | echo_location /mysql "insert into foo (val) values (3.1415926);"; 192 | echo_location /mysql "select * from foo;"; 193 | } 194 | location /mysql { 195 | drizzle_pass backend; 196 | drizzle_module_header off; 197 | drizzle_query $query_string; 198 | rds_csv on; 199 | rds_csv_row_terminator "\n"; 200 | } 201 | --- request 202 | GET /test 203 | --- response_body 204 | errcode,errstr,insert_id,affected_rows 205 | 0,,0,0 206 | errcode,errstr,insert_id,affected_rows 207 | 0,,0,0 208 | errcode,errstr,insert_id,affected_rows 209 | 0,,1,1 210 | id,val 211 | 1,3.1415926 212 | --- skip_nginx: 2: < 0.7.46 213 | 214 | 215 | 216 | === TEST 9: text blob field 217 | --- http_config eval: $::http_config 218 | --- config 219 | location /test { 220 | echo_location /mysql "drop table if exists foo"; 221 | echo_location /mysql "create table foo (id serial, body text);"; 222 | echo_location /mysql "insert into foo (body) values ('hello');"; 223 | echo_location /mysql "select * from foo;"; 224 | } 225 | location /mysql { 226 | drizzle_pass backend; 227 | drizzle_module_header off; 228 | drizzle_query $query_string; 229 | rds_csv on; 230 | rds_csv_row_terminator "\n"; 231 | } 232 | --- request 233 | GET /test 234 | --- response_body 235 | errcode,errstr,insert_id,affected_rows 236 | 0,,0,0 237 | errcode,errstr,insert_id,affected_rows 238 | 0,,0,0 239 | errcode,errstr,insert_id,affected_rows 240 | 0,,1,1 241 | id,body 242 | 1,hello 243 | --- skip_nginx: 2: < 0.7.46 244 | 245 | 246 | 247 | === TEST 10: bool blob field 248 | --- http_config eval: $::http_config 249 | --- config 250 | location /test { 251 | echo_location /mysql "drop table if exists foo"; 252 | echo_location /mysql "create table foo (id serial, flag bool);"; 253 | echo_location /mysql "insert into foo (flag) values (true);"; 254 | echo_location /mysql "insert into foo (flag) values (false);"; 255 | echo_location /mysql "select * from foo order by id;"; 256 | } 257 | location /mysql { 258 | drizzle_pass backend; 259 | drizzle_module_header off; 260 | drizzle_query $query_string; 261 | rds_csv on; 262 | rds_csv_row_terminator "\n"; 263 | } 264 | --- request 265 | GET /test 266 | --- response_body eval 267 | qq{errcode,errstr,insert_id,affected_rows 268 | 0,,0,0 269 | errcode,errstr,insert_id,affected_rows 270 | 0,,0,0 271 | errcode,errstr,insert_id,affected_rows 272 | 0,,1,1 273 | errcode,errstr,insert_id,affected_rows 274 | 0,,2,1 275 | id,flag 276 | 1,1 277 | 2,0 278 | } 279 | --- skip_nginx: 2: < 0.7.46 280 | --- timeout: 10 281 | 282 | 283 | 284 | === TEST 11: bit field 285 | --- http_config eval: $::http_config 286 | --- config 287 | location /test { 288 | echo_location /mysql "drop table if exists foo"; 289 | echo_location /mysql "create table foo (id serial, flag bit);"; 290 | echo_location /mysql "insert into foo (flag) values (1);"; 291 | echo_location /mysql "insert into foo (flag) values (0);"; 292 | echo_location /mysql "select * from foo order by id;"; 293 | } 294 | location /mysql { 295 | drizzle_pass backend; 296 | drizzle_module_header off; 297 | drizzle_query $query_string; 298 | rds_csv on; 299 | rds_csv_row_terminator "\n"; 300 | } 301 | --- request 302 | GET /test 303 | --- response_body eval 304 | qq{errcode,errstr,insert_id,affected_rows 305 | 0,,0,0 306 | errcode,errstr,insert_id,affected_rows 307 | 0,,0,0 308 | errcode,errstr,insert_id,affected_rows 309 | 0,,1,1 310 | errcode,errstr,insert_id,affected_rows 311 | 0,,2,1 312 | id,flag 313 | 1,\x01 314 | 2,\x00 315 | } 316 | --- skip_nginx: 2: < 0.7.46 317 | --- timeout: 10 318 | 319 | 320 | 321 | === TEST 12: date type 322 | --- http_config eval: $::http_config 323 | --- config 324 | location /test { 325 | echo_location /mysql "drop table if exists foo"; 326 | echo_location /mysql "create table foo (id serial, created date);"; 327 | echo_location /mysql "insert into foo (created) values ('2007-05-24');"; 328 | echo_location /mysql "select * from foo"; 329 | } 330 | location /mysql { 331 | drizzle_pass backend; 332 | drizzle_module_header off; 333 | drizzle_query $query_string; 334 | rds_csv on; 335 | } 336 | --- request 337 | GET /test 338 | --- response_body eval 339 | qq{errcode,errstr,insert_id,affected_rows\r 340 | 0,,0,0\r 341 | errcode,errstr,insert_id,affected_rows\r 342 | 0,,0,0\r 343 | errcode,errstr,insert_id,affected_rows\r 344 | 0,,1,1\r 345 | id,created\r 346 | 1,2007-05-24\r 347 | } 348 | 349 | 350 | 351 | === TEST 13: strings need to be escaped (forcing utf8) 352 | --- http_config 353 | upstream backend { 354 | drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql 355 | dbname=ngx_test user=ngx_test password=ngx_test 356 | charset=utf8mb4; 357 | } 358 | 359 | --- config 360 | location /test { 361 | echo_location /mysql "drop table if exists foo"; 362 | echo_location /mysql "create table foo (id serial, body char(25));"; 363 | echo_location /mysql "insert into foo (body) values ('a\\r\\nb\\b你好\Z');"; 364 | echo_location /mysql "select * from foo"; 365 | } 366 | location /mysql { 367 | drizzle_pass backend; 368 | drizzle_module_header off; 369 | drizzle_query $query_string; 370 | rds_csv on; 371 | } 372 | --- request 373 | GET /test 374 | --- response_body eval 375 | qq{errcode,errstr,insert_id,affected_rows\r 376 | 0,,0,0\r 377 | errcode,errstr,insert_id,affected_rows\r 378 | 0,,0,0\r 379 | errcode,errstr,insert_id,affected_rows\r 380 | 0,,1,1\r 381 | id,body\r 382 | 1,"a\r 383 | b\b你好\cZ"\r 384 | } 385 | --- timeout: 5 386 | 387 | 388 | 389 | === TEST 14: strings need to be escaped 390 | --- SKIP 391 | --- http_config eval: $::http_config 392 | --- config 393 | location /test { 394 | echo_location /mysql "drop table if exists foo"; 395 | echo_location /mysql "create table foo (id serial, body char(25));"; 396 | echo_location /mysql "insert into foo (body) values ('a\\r\\nb\\b你好\Z');"; 397 | echo_location /mysql "select * from foo"; 398 | } 399 | location /mysql { 400 | drizzle_pass backend; 401 | drizzle_module_header off; 402 | drizzle_query $query_string; 403 | rds_csv on; 404 | } 405 | --- request 406 | GET /test 407 | --- response_body eval 408 | qq{errcode,errstr,insert_id,affected_rows\r 409 | 0,,0,0\r 410 | errcode,errstr,insert_id,affected_rows\r 411 | 0,,0,0\r 412 | errcode,errstr,insert_id,affected_rows\r 413 | 0,,1,1\r 414 | id,body\r 415 | 1,"a\r 416 | b\b你好\x1a"\r 417 | } 418 | 419 | 420 | 421 | === TEST 15: null values 422 | --- http_config eval: $::http_config 423 | --- config 424 | location /test { 425 | echo_location /mysql "drop table if exists foo"; 426 | echo_location /mysql "create table foo (id serial, name char(10), age integer);"; 427 | echo_location /mysql "insert into foo (name, age) values ('', null);"; 428 | echo_location /mysql "insert into foo (name, age) values (null, 0);"; 429 | echo_location /mysql "select * from foo order by id"; 430 | } 431 | location /mysql { 432 | drizzle_pass backend; 433 | drizzle_module_header off; 434 | drizzle_query $query_string; 435 | rds_csv on; 436 | } 437 | --- request 438 | GET /test 439 | --- response_body eval 440 | qq{errcode,errstr,insert_id,affected_rows\r 441 | 0,,0,0\r 442 | errcode,errstr,insert_id,affected_rows\r 443 | 0,,0,0\r 444 | errcode,errstr,insert_id,affected_rows\r 445 | 0,,1,1\r 446 | errcode,errstr,insert_id,affected_rows\r 447 | 0,,2,1\r 448 | id,name,age\r 449 | 1,,\r 450 | 2,,0\r 451 | } 452 | --- timeout: 10 453 | 454 | -------------------------------------------------------------------------------- /t/sanity-stream.t: -------------------------------------------------------------------------------- 1 | # vi:filetype= 2 | 3 | use lib 'lib'; 4 | use Test::Nginx::Socket; 5 | 6 | #repeat_each(10); 7 | no_shuffle(); 8 | 9 | repeat_each(1); 10 | 11 | plan tests => repeat_each() * 2 * blocks() + 2 * repeat_each() * 3; 12 | 13 | $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; 14 | 15 | our $http_config = <<'_EOC_'; 16 | upstream backend { 17 | drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql 18 | dbname=ngx_test user=ngx_test password=ngx_test; 19 | } 20 | _EOC_ 21 | 22 | no_long_string(); 23 | #no_diff(); 24 | 25 | run_tests(); 26 | 27 | __DATA__ 28 | 29 | === TEST 1: sanity 30 | --- http_config eval: $::http_config 31 | --- config 32 | location /mysql { 33 | drizzle_pass backend; 34 | #drizzle_dbname $dbname; 35 | drizzle_query 'select * from cats'; 36 | rds_csv on; 37 | rds_csv_row_terminator "\n"; 38 | rds_csv_field_name_header off; 39 | drizzle_buffer_size 1; 40 | } 41 | --- request 42 | GET /mysql 43 | --- response_headers_like 44 | X-Resty-DBD-Module: ngx_drizzle \d+\.\d+\.\d+ 45 | Content-Type: text/csv; header=absence 46 | --- response_body 47 | 2, 48 | 3,bob 49 | --- timeout: 15 50 | 51 | 52 | 53 | === TEST 2: keep-alive 54 | --- http_config eval: $::http_config 55 | --- config 56 | location /mysql { 57 | drizzle_pass backend; 58 | #drizzle_dbname $dbname; 59 | drizzle_query 'select * from cats'; 60 | rds_csv on; 61 | rds_csv_row_terminator "\n"; 62 | drizzle_buffer_size 1; 63 | } 64 | --- request 65 | GET /mysql 66 | --- response_body 67 | id,name 68 | 2, 69 | 3,bob 70 | 71 | 72 | 73 | === TEST 3: update 74 | --- http_config eval: $::http_config 75 | --- config 76 | location /mysql { 77 | drizzle_pass backend; 78 | #drizzle_dbname $dbname; 79 | drizzle_query "update cats set name='bob' where name='bob'"; 80 | rds_csv on; 81 | rds_csv_row_terminator "\n"; 82 | drizzle_buffer_size 1; 83 | } 84 | --- request 85 | GET /mysql 86 | --- response_body 87 | errcode,errstr,insert_id,affected_rows 88 | 0,Rows matched: 1 Changed: 0 Warnings: 0,0,0 89 | 90 | 91 | 92 | === TEST 4: select empty result 93 | --- http_config eval: $::http_config 94 | --- config 95 | location /mysql { 96 | drizzle_pass backend; 97 | drizzle_query "select * from cats where name='tom'"; 98 | rds_csv on; 99 | drizzle_buffer_size 1; 100 | } 101 | --- request 102 | GET /mysql 103 | --- response_body eval 104 | "id,name\r 105 | " 106 | 107 | 108 | 109 | === TEST 5: update & no module header 110 | --- http_config eval: $::http_config 111 | --- config 112 | location /mysql { 113 | if ($arg_name ~ '[^A-Za-z0-9]') { 114 | return 400; 115 | } 116 | 117 | drizzle_pass backend; 118 | drizzle_module_header off; 119 | drizzle_query "update cats set name='$arg_name' where name='$arg_name'"; 120 | 121 | rds_csv on; 122 | drizzle_buffer_size 1; 123 | } 124 | --- request 125 | GET /mysql?name=bob 126 | --- response_headers 127 | X-Resty-DBD-Module: 128 | Content-Type: text/csv; header=presence 129 | --- response_body eval 130 | qq{errcode,errstr,insert_id,affected_rows\r 131 | 0,Rows matched: 1 Changed: 0 Warnings: 0,0,0\r 132 | } 133 | 134 | 135 | 136 | === TEST 6: invalid SQL 137 | --- http_config eval: $::http_config 138 | --- config 139 | location /mysql { 140 | drizzle_pass backend; 141 | drizzle_module_header off; 142 | drizzle_query "select '32"; 143 | rds_csv on; 144 | rds_csv_row_terminator "\n"; 145 | drizzle_buffer_size 1; 146 | } 147 | --- response_headers 148 | X-Resty-DBD-Module: 149 | Content-Type: text/html 150 | --- request 151 | GET /mysql 152 | --- error_code: 500 153 | --- response_body_like: 500 Internal Server Error 154 | 155 | 156 | 157 | === TEST 7: single row, single col 158 | --- http_config eval: $::http_config 159 | --- config 160 | location /test { 161 | echo_location /mysql "drop table if exists singles"; 162 | echo_location /mysql "create table singles (name varchar(15));"; 163 | echo_location /mysql "insert into singles values ('marry');"; 164 | echo_location /mysql "select * from singles;"; 165 | } 166 | location /mysql { 167 | drizzle_pass backend; 168 | drizzle_module_header off; 169 | drizzle_query $query_string; 170 | rds_csv on; 171 | rds_csv_row_terminator "\n"; 172 | drizzle_buffer_size 1; 173 | } 174 | --- request 175 | GET /test 176 | --- response_body 177 | errcode,errstr,insert_id,affected_rows 178 | 0,,0,0 179 | errcode,errstr,insert_id,affected_rows 180 | 0,,0,0 181 | errcode,errstr,insert_id,affected_rows 182 | 0,,0,1 183 | name 184 | marry 185 | --- skip_nginx: 2: < 0.7.46 186 | --- timeout: 5 187 | 188 | 189 | 190 | === TEST 8: floating number and insert id 191 | --- http_config eval: $::http_config 192 | --- config 193 | location /test { 194 | echo_location /mysql "drop table if exists foo"; 195 | echo_location /mysql "create table foo (id serial not null, primary key (id), val real);"; 196 | echo_location /mysql "insert into foo (val) values (3.1415926);"; 197 | echo_location /mysql "select * from foo;"; 198 | } 199 | location /mysql { 200 | drizzle_pass backend; 201 | drizzle_module_header off; 202 | drizzle_query $query_string; 203 | rds_csv on; 204 | rds_csv_row_terminator "\n"; 205 | drizzle_buffer_size 1; 206 | } 207 | --- request 208 | GET /test 209 | --- response_body 210 | errcode,errstr,insert_id,affected_rows 211 | 0,,0,0 212 | errcode,errstr,insert_id,affected_rows 213 | 0,,0,0 214 | errcode,errstr,insert_id,affected_rows 215 | 0,,1,1 216 | id,val 217 | 1,3.1415926 218 | --- skip_nginx: 2: < 0.7.46 219 | 220 | 221 | 222 | === TEST 9: text blob field 223 | --- http_config eval: $::http_config 224 | --- config 225 | location /test { 226 | echo_location /mysql "drop table if exists foo"; 227 | echo_location /mysql "create table foo (id serial, body text);"; 228 | echo_location /mysql "insert into foo (body) values ('hello');"; 229 | echo_location /mysql "select * from foo;"; 230 | } 231 | location /mysql { 232 | drizzle_pass backend; 233 | drizzle_module_header off; 234 | drizzle_query $query_string; 235 | rds_csv on; 236 | rds_csv_row_terminator "\n"; 237 | drizzle_buffer_size 1; 238 | } 239 | --- request 240 | GET /test 241 | --- response_body 242 | errcode,errstr,insert_id,affected_rows 243 | 0,,0,0 244 | errcode,errstr,insert_id,affected_rows 245 | 0,,0,0 246 | errcode,errstr,insert_id,affected_rows 247 | 0,,1,1 248 | id,body 249 | 1,hello 250 | --- skip_nginx: 2: < 0.7.46 251 | 252 | 253 | 254 | === TEST 10: bool blob field 255 | --- http_config eval: $::http_config 256 | --- config 257 | location /test { 258 | echo_location /mysql "drop table if exists foo"; 259 | echo_location /mysql "create table foo (id serial, flag bool);"; 260 | echo_location /mysql "insert into foo (flag) values (true);"; 261 | echo_location /mysql "insert into foo (flag) values (false);"; 262 | echo_location /mysql "select * from foo order by id;"; 263 | } 264 | location /mysql { 265 | drizzle_pass backend; 266 | drizzle_module_header off; 267 | drizzle_query $query_string; 268 | rds_csv on; 269 | rds_csv_row_terminator "\n"; 270 | drizzle_buffer_size 1; 271 | } 272 | --- request 273 | GET /test 274 | --- response_body eval 275 | qq{errcode,errstr,insert_id,affected_rows 276 | 0,,0,0 277 | errcode,errstr,insert_id,affected_rows 278 | 0,,0,0 279 | errcode,errstr,insert_id,affected_rows 280 | 0,,1,1 281 | errcode,errstr,insert_id,affected_rows 282 | 0,,2,1 283 | id,flag 284 | 1,1 285 | 2,0 286 | } 287 | --- skip_nginx: 2: < 0.7.46 288 | --- timeout: 10 289 | 290 | 291 | 292 | === TEST 11: bit field 293 | --- http_config eval: $::http_config 294 | --- config 295 | location /test { 296 | echo_location /mysql "drop table if exists foo"; 297 | echo_location /mysql "create table foo (id serial, flag bit);"; 298 | echo_location /mysql "insert into foo (flag) values (1);"; 299 | echo_location /mysql "insert into foo (flag) values (0);"; 300 | echo_location /mysql "select * from foo order by id;"; 301 | } 302 | location /mysql { 303 | drizzle_pass backend; 304 | drizzle_module_header off; 305 | drizzle_query $query_string; 306 | rds_csv on; 307 | rds_csv_row_terminator "\n"; 308 | drizzle_buffer_size 1; 309 | } 310 | --- request 311 | GET /test 312 | --- response_body eval 313 | qq{errcode,errstr,insert_id,affected_rows 314 | 0,,0,0 315 | errcode,errstr,insert_id,affected_rows 316 | 0,,0,0 317 | errcode,errstr,insert_id,affected_rows 318 | 0,,1,1 319 | errcode,errstr,insert_id,affected_rows 320 | 0,,2,1 321 | id,flag 322 | 1,\x01 323 | 2,\x00 324 | } 325 | --- skip_nginx: 2: < 0.7.46 326 | --- timeout: 10 327 | 328 | 329 | 330 | === TEST 12: date type 331 | --- http_config eval: $::http_config 332 | --- config 333 | location /test { 334 | echo_location /mysql "drop table if exists foo"; 335 | echo_location /mysql "create table foo (id serial, created date);"; 336 | echo_location /mysql "insert into foo (created) values ('2007-05-24');"; 337 | echo_location /mysql "select * from foo"; 338 | } 339 | location /mysql { 340 | drizzle_pass backend; 341 | drizzle_module_header off; 342 | drizzle_query $query_string; 343 | rds_csv on; 344 | drizzle_buffer_size 1; 345 | } 346 | --- request 347 | GET /test 348 | --- response_body eval 349 | qq{errcode,errstr,insert_id,affected_rows\r 350 | 0,,0,0\r 351 | errcode,errstr,insert_id,affected_rows\r 352 | 0,,0,0\r 353 | errcode,errstr,insert_id,affected_rows\r 354 | 0,,1,1\r 355 | id,created\r 356 | 1,2007-05-24\r 357 | } 358 | 359 | 360 | 361 | === TEST 13: strings need to be escaped (forcing utf8) 362 | --- http_config 363 | upstream backend { 364 | drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql 365 | dbname=ngx_test user=ngx_test password=ngx_test 366 | charset=utf8mb4; 367 | } 368 | 369 | --- config 370 | location /test { 371 | echo_location /mysql "drop table if exists foo"; 372 | echo_location /mysql "create table foo (id serial, body char(25));"; 373 | echo_location /mysql "insert into foo (body) values ('a\\r\\nb\\b你好\Z');"; 374 | echo_location /mysql "select * from foo"; 375 | } 376 | location /mysql { 377 | drizzle_pass backend; 378 | drizzle_module_header off; 379 | drizzle_query $query_string; 380 | rds_csv on; 381 | drizzle_buffer_size 1; 382 | } 383 | --- request 384 | GET /test 385 | --- response_body eval 386 | qq{errcode,errstr,insert_id,affected_rows\r 387 | 0,,0,0\r 388 | errcode,errstr,insert_id,affected_rows\r 389 | 0,,0,0\r 390 | errcode,errstr,insert_id,affected_rows\r 391 | 0,,1,1\r 392 | id,body\r 393 | 1,"a\r 394 | b\b你好\cZ"\r 395 | } 396 | --- timeout: 5 397 | 398 | 399 | 400 | === TEST 14: strings need to be escaped 401 | --- SKIP 402 | --- http_config eval: $::http_config 403 | --- config 404 | location /test { 405 | echo_location /mysql "drop table if exists foo"; 406 | echo_location /mysql "create table foo (id serial, body char(25));"; 407 | echo_location /mysql "insert into foo (body) values ('a\\r\\nb\\b你好\Z');"; 408 | echo_location /mysql "select * from foo"; 409 | } 410 | location /mysql { 411 | drizzle_pass backend; 412 | drizzle_module_header off; 413 | drizzle_query $query_string; 414 | rds_csv on; 415 | drizzle_buffer_size 1; 416 | } 417 | --- request 418 | GET /test 419 | --- response_body eval 420 | qq{errcode,errstr,insert_id,affected_rows\r 421 | 0,,0,0\r 422 | errcode,errstr,insert_id,affected_rows\r 423 | 0,,0,0\r 424 | errcode,errstr,insert_id,affected_rows\r 425 | 0,,1,1\r 426 | id,body\r 427 | 1,"a\r 428 | b\b你好\x1a"\r 429 | } 430 | 431 | 432 | 433 | === TEST 15: null values 434 | --- http_config eval: $::http_config 435 | --- config 436 | location /test { 437 | echo_location /mysql "drop table if exists foo"; 438 | echo_location /mysql "create table foo (id serial, name char(10), age integer);"; 439 | echo_location /mysql "insert into foo (name, age) values ('', null);"; 440 | echo_location /mysql "insert into foo (name, age) values (null, 0);"; 441 | echo_location /mysql "select * from foo order by id"; 442 | } 443 | location /mysql { 444 | drizzle_pass backend; 445 | drizzle_module_header off; 446 | drizzle_query $query_string; 447 | rds_csv on; 448 | drizzle_buffer_size 1; 449 | } 450 | --- request 451 | GET /test 452 | --- response_body eval 453 | qq{errcode,errstr,insert_id,affected_rows\r 454 | 0,,0,0\r 455 | errcode,errstr,insert_id,affected_rows\r 456 | 0,,0,0\r 457 | errcode,errstr,insert_id,affected_rows\r 458 | 0,,1,1\r 459 | errcode,errstr,insert_id,affected_rows\r 460 | 0,,2,1\r 461 | id,name,age\r 462 | 1,,\r 463 | 2,,0\r 464 | } 465 | --- timeout: 10 466 | 467 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_processor.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) agentzh 3 | */ 4 | 5 | 6 | #ifndef DDEBUG 7 | #define DDEBUG 0 8 | #endif 9 | #include "ddebug.h" 10 | 11 | 12 | #include "ngx_http_rds_csv_processor.h" 13 | #include "ngx_http_rds_csv_util.h" 14 | #include "ngx_http_rds_csv_output.h" 15 | #include "ngx_http_rds.h" 16 | #include "ngx_http_rds_utils.h" 17 | 18 | 19 | #include 20 | #include 21 | 22 | 23 | ngx_int_t 24 | ngx_http_rds_csv_process_header(ngx_http_request_t *r, ngx_chain_t *in, 25 | ngx_http_rds_csv_ctx_t *ctx) 26 | { 27 | ngx_buf_t *b; 28 | ngx_http_rds_header_t header; 29 | ngx_int_t rc; 30 | 31 | if (in == NULL) { 32 | return NGX_OK; 33 | } 34 | 35 | b = in->buf; 36 | 37 | if (!ngx_buf_in_memory(b)) { 38 | if (!ngx_buf_special(b)) { 39 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 40 | "rds_csv: process header: buf from " 41 | "upstream not in memory"); 42 | goto invalid; 43 | } 44 | 45 | in = in->next; 46 | 47 | if (in == NULL) { 48 | return NGX_OK; 49 | } 50 | 51 | b = in->buf; 52 | } 53 | 54 | rc = ngx_http_rds_parse_header(r, b, &header); 55 | 56 | if (rc != NGX_OK) { 57 | goto invalid; 58 | } 59 | 60 | dd("col count: %d", (int) header.col_count); 61 | 62 | if (header.col_count == 0) { 63 | /* for empty result set, just return the JSON 64 | * representation of the RDS header */ 65 | 66 | dd("col count == 0"); 67 | 68 | if (b->pos != b->last) { 69 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 70 | "rds_csv: header: there's unexpected remaining data " 71 | "in the buf"); 72 | 73 | goto invalid; 74 | } 75 | 76 | ctx->state = state_done; 77 | 78 | /* now we send the postponed response header */ 79 | if (!ctx->header_sent) { 80 | ctx->header_sent = 1; 81 | 82 | rc = ngx_http_rds_csv_next_header_filter(r); 83 | 84 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 85 | return rc; 86 | } 87 | } 88 | 89 | rc = ngx_http_rds_csv_output_header(r, ctx, &header); 90 | 91 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 92 | return rc; 93 | } 94 | 95 | ngx_http_rds_csv_discard_bufs(r->pool, in); 96 | 97 | return rc; 98 | } 99 | 100 | ctx->cols = ngx_palloc(r->pool, 101 | header.col_count * sizeof(ngx_http_rds_column_t)); 102 | 103 | if (ctx->cols == NULL) { 104 | goto invalid; 105 | } 106 | 107 | ctx->state = state_expect_col; 108 | ctx->cur_col = 0; 109 | ctx->col_count = header.col_count; 110 | 111 | /* now we send the postponed response header */ 112 | if (!ctx->header_sent) { 113 | ctx->header_sent = 1; 114 | 115 | rc = ngx_http_rds_csv_next_header_filter(r); 116 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 117 | return rc; 118 | } 119 | } 120 | 121 | return ngx_http_rds_csv_process_col(r, b->pos == b->last ? in->next : in, 122 | ctx); 123 | 124 | invalid: 125 | 126 | dd("return 500"); 127 | if (!ctx->header_sent) { 128 | ctx->header_sent = 1; 129 | 130 | r->headers_out.status = NGX_HTTP_INTERNAL_SERVER_ERROR; 131 | ngx_http_send_header(r); 132 | ngx_http_send_special(r, NGX_HTTP_LAST); 133 | 134 | return NGX_ERROR; 135 | } 136 | 137 | return NGX_ERROR; 138 | } 139 | 140 | 141 | ngx_int_t 142 | ngx_http_rds_csv_process_col(ngx_http_request_t *r, ngx_chain_t *in, 143 | ngx_http_rds_csv_ctx_t *ctx) 144 | { 145 | ngx_buf_t *b; 146 | ngx_int_t rc; 147 | ngx_http_rds_csv_loc_conf_t *conf; 148 | 149 | if (in == NULL) { 150 | return NGX_OK; 151 | } 152 | 153 | b = in->buf; 154 | 155 | if (!ngx_buf_in_memory(b)) { 156 | if (!ngx_buf_special(b)) { 157 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 158 | "rds_csv: process col: buf from upstream not in " 159 | "memory"); 160 | return NGX_ERROR; 161 | } 162 | 163 | in = in->next; 164 | 165 | if (in == NULL) { 166 | return NGX_OK; 167 | } 168 | 169 | b = in->buf; 170 | } 171 | 172 | dd("parsing rds column"); 173 | 174 | rc = ngx_http_rds_parse_col(r, b, &ctx->cols[ctx->cur_col]); 175 | 176 | dd("parse col returns %d (%d)", (int) rc, (int) NGX_OK); 177 | 178 | if (rc != NGX_OK) { 179 | return NGX_ERROR; 180 | } 181 | 182 | if (b->pos == b->last) { 183 | dd("parse col buf consumed"); 184 | in = in->next; 185 | } 186 | 187 | ctx->cur_col++; 188 | 189 | if (ctx->cur_col >= ctx->col_count) { 190 | dd("end of column list"); 191 | 192 | ctx->state = state_expect_row; 193 | ctx->row = 0; 194 | 195 | conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module); 196 | 197 | if (conf->field_name_header) { 198 | rc = ngx_http_rds_csv_output_field_names(r, ctx); 199 | 200 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 201 | return rc; 202 | } 203 | } 204 | 205 | dd("after output literal"); 206 | 207 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 208 | return rc; 209 | } 210 | 211 | 212 | dd("process col is entering process row..."); 213 | return ngx_http_rds_csv_process_row(r, in, ctx); 214 | } 215 | 216 | return ngx_http_rds_csv_process_col(r, in, ctx); 217 | } 218 | 219 | 220 | ngx_int_t 221 | ngx_http_rds_csv_process_row(ngx_http_request_t *r, ngx_chain_t *in, 222 | ngx_http_rds_csv_ctx_t *ctx) 223 | { 224 | ngx_buf_t *b; 225 | ngx_int_t rc; 226 | 227 | if (in == NULL) { 228 | return NGX_OK; 229 | } 230 | 231 | dd("process row"); 232 | 233 | b = in->buf; 234 | 235 | if (!ngx_buf_in_memory(b)) { 236 | if (!ngx_buf_special(b)) { 237 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 238 | "rds_csv: process row: buf from " 239 | "upstream not in memory"); 240 | return NGX_ERROR; 241 | } 242 | 243 | in = in->next; 244 | 245 | if (in == NULL) { 246 | return NGX_OK; 247 | } 248 | 249 | b = in->buf; 250 | } 251 | 252 | if (b->last - b->pos < (ssize_t) sizeof(uint8_t)) { 253 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 254 | "rds_csv: row flag is incomplete in the buf"); 255 | return NGX_ERROR; 256 | } 257 | 258 | dd("row flag: %d (offset %d)", (char) *b->pos, (int) (b->pos - b->start)); 259 | 260 | if (*b->pos++ == 0) { 261 | /* end of row list */ 262 | ctx->state = state_done; 263 | 264 | if (b->pos != b->last) { 265 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 266 | "rds_csv: row: there's unexpected remaining data " 267 | "in the buf"); 268 | return NGX_ERROR; 269 | } 270 | 271 | rc = ngx_http_rds_csv_output_literal(r, ctx, (u_char *) "", 0, 272 | 1 /* last buf*/); 273 | 274 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 275 | return rc; 276 | } 277 | 278 | return rc; 279 | } 280 | 281 | ctx->row++; 282 | ctx->cur_col = 0; 283 | ctx->state = state_expect_field; 284 | 285 | if (b->pos == b->last) { 286 | in = in->next; 287 | 288 | } else { 289 | dd("process row: buf not consumed completely"); 290 | } 291 | 292 | return ngx_http_rds_csv_process_field(r, in, ctx); 293 | } 294 | 295 | 296 | ngx_int_t 297 | ngx_http_rds_csv_process_field(ngx_http_request_t *r, ngx_chain_t *in, 298 | ngx_http_rds_csv_ctx_t *ctx) 299 | { 300 | size_t total, len; 301 | ngx_buf_t *b; 302 | ngx_int_t rc; 303 | 304 | for (;;) { 305 | if (in == NULL) { 306 | return NGX_OK; 307 | } 308 | 309 | b = in->buf; 310 | 311 | if (!ngx_buf_in_memory(b)) { 312 | dd("buf not in memory"); 313 | 314 | if (!ngx_buf_special(b)) { 315 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 316 | "rds_csv: process field: buf from " 317 | "upstream not in memory"); 318 | return NGX_ERROR; 319 | } 320 | 321 | in = in->next; 322 | 323 | if (in == NULL) { 324 | return NGX_OK; 325 | } 326 | 327 | b = in->buf; 328 | } 329 | 330 | dd("process field: buf size: %d", (int) ngx_buf_size(b)); 331 | 332 | if (b->last - b->pos < (ssize_t) sizeof(uint32_t)) { 333 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 334 | "rds_csv: field size is incomplete in the buf: %*s " 335 | "(len: %d)", b->last - b->pos, b->pos, 336 | (int) (b->last - b->pos)); 337 | 338 | return NGX_ERROR; 339 | } 340 | 341 | total = *(uint32_t *) b->pos; 342 | 343 | dd("total: %d", (int) total); 344 | 345 | b->pos += sizeof(uint32_t); 346 | 347 | if (total == (uint32_t) -1) { 348 | /* SQL NULL found */ 349 | total = 0; 350 | len = 0; 351 | ctx->field_data_rest = 0; 352 | 353 | rc = ngx_http_rds_csv_output_field(r, ctx, b->pos, len, 354 | 1 /* is null */); 355 | 356 | } else { 357 | len = (uint32_t) (b->last - b->pos); 358 | 359 | if (len >= total) { 360 | len = total; 361 | } 362 | 363 | ctx->field_data_rest = total - len; 364 | 365 | rc = ngx_http_rds_csv_output_field(r, ctx, b->pos, len, 366 | 0 /* not null */); 367 | } 368 | 369 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 370 | return rc; 371 | } 372 | 373 | b->pos += len; 374 | 375 | if (b->pos == b->last) { 376 | in = in->next; 377 | } 378 | 379 | if (len < total) { 380 | dd("process field: need to read more field data"); 381 | 382 | ctx->state = state_expect_more_field_data; 383 | 384 | return ngx_http_rds_csv_process_more_field_data(r, in, ctx); 385 | } 386 | 387 | ctx->cur_col++; 388 | 389 | if (ctx->cur_col >= ctx->col_count) { 390 | dd("reached the end of the current row"); 391 | 392 | ctx->state = state_expect_row; 393 | 394 | return ngx_http_rds_csv_process_row(r, in, ctx); 395 | } 396 | 397 | /* continue to process the next field (if any) */ 398 | } 399 | 400 | /* impossible to reach here */ 401 | 402 | return NGX_ERROR; 403 | } 404 | 405 | 406 | ngx_int_t 407 | ngx_http_rds_csv_process_more_field_data(ngx_http_request_t *r, 408 | ngx_chain_t *in, ngx_http_rds_csv_ctx_t *ctx) 409 | { 410 | ngx_int_t rc; 411 | ngx_buf_t *b; 412 | size_t len; 413 | 414 | for (;;) { 415 | if (in == NULL) { 416 | return NGX_OK; 417 | } 418 | 419 | b = in->buf; 420 | 421 | if (!ngx_buf_in_memory(b)) { 422 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 423 | "rds_csv: buf from upstream not in memory"); 424 | return NGX_ERROR; 425 | } 426 | 427 | len = b->last - b->pos; 428 | 429 | if (len >= ctx->field_data_rest) { 430 | len = ctx->field_data_rest; 431 | ctx->field_data_rest = 0; 432 | 433 | } else { 434 | ctx->field_data_rest -= len; 435 | } 436 | 437 | rc = ngx_http_rds_csv_output_more_field_data(r, ctx, b->pos, len); 438 | 439 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 440 | return rc; 441 | } 442 | 443 | b->pos += len; 444 | 445 | if (b->pos == b->last) { 446 | in = in->next; 447 | } 448 | 449 | if (ctx->field_data_rest) { 450 | dd("process more field data: still some data remaining"); 451 | continue; 452 | } 453 | 454 | dd("process more field data: reached the end of the current field"); 455 | 456 | ctx->cur_col++; 457 | 458 | if (ctx->cur_col >= ctx->col_count) { 459 | dd("process more field data: reached the end of the current row"); 460 | 461 | ctx->state = state_expect_row; 462 | 463 | return ngx_http_rds_csv_process_row(r, in, ctx); 464 | } 465 | 466 | dd("proces more field data: read the next field"); 467 | 468 | ctx->state = state_expect_field; 469 | 470 | return ngx_http_rds_csv_process_field(r, in, ctx); 471 | } 472 | 473 | /* impossible to reach here */ 474 | 475 | return NGX_ERROR; 476 | } 477 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_filter_module.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) Yichun Zhang (agentzh) 4 | */ 5 | 6 | 7 | #ifndef DDEBUG 8 | #define DDEBUG 0 9 | #endif 10 | #include "ddebug.h" 11 | 12 | 13 | #include "ngx_http_rds_csv_filter_module.h" 14 | #include "ngx_http_rds_csv_util.h" 15 | #include "ngx_http_rds_csv_processor.h" 16 | #include "ngx_http_rds_csv_output.h" 17 | 18 | #include 19 | 20 | 21 | #define ngx_http_rds_csv_content_type "text/csv" 22 | #define ngx_http_rds_csv_row_term "\r\n" 23 | 24 | 25 | static volatile ngx_cycle_t *ngx_http_rds_csv_prev_cycle = NULL; 26 | 27 | 28 | ngx_http_output_header_filter_pt ngx_http_rds_csv_next_header_filter; 29 | ngx_http_output_body_filter_pt ngx_http_rds_csv_next_body_filter; 30 | 31 | 32 | static void *ngx_http_rds_csv_create_loc_conf(ngx_conf_t *cf); 33 | static char *ngx_http_rds_csv_merge_loc_conf(ngx_conf_t *cf, void *parent, 34 | void *child); 35 | static ngx_int_t ngx_http_rds_csv_filter_init(ngx_conf_t *cf); 36 | static char *ngx_http_rds_csv_row_terminator(ngx_conf_t *cf, 37 | ngx_command_t *cmd, void *conf); 38 | static char *ngx_http_rds_csv_field_separator(ngx_conf_t *cf, 39 | ngx_command_t *cmd, void *conf); 40 | static char *ngx_http_rds_csv(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); 41 | static void *ngx_http_rds_csv_create_main_conf(ngx_conf_t *cf); 42 | 43 | 44 | static ngx_command_t ngx_http_rds_csv_commands[] = { 45 | 46 | { ngx_string("rds_csv"), 47 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF 48 | |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF 49 | |NGX_CONF_FLAG, 50 | ngx_http_rds_csv, 51 | NGX_HTTP_LOC_CONF_OFFSET, 52 | offsetof(ngx_http_rds_csv_loc_conf_t, enabled), 53 | NULL }, 54 | 55 | { ngx_string("rds_csv_row_terminator"), 56 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF 57 | |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF 58 | |NGX_CONF_TAKE1, 59 | ngx_http_rds_csv_row_terminator, 60 | NGX_HTTP_LOC_CONF_OFFSET, 61 | offsetof(ngx_http_rds_csv_loc_conf_t, row_term), 62 | NULL }, 63 | 64 | { ngx_string("rds_csv_field_separator"), 65 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF 66 | |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF 67 | |NGX_CONF_TAKE1, 68 | ngx_http_rds_csv_field_separator, 69 | NGX_HTTP_LOC_CONF_OFFSET, 70 | 0, 71 | NULL }, 72 | 73 | { ngx_string("rds_csv_field_name_header"), 74 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF 75 | |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF 76 | |NGX_CONF_FLAG, 77 | ngx_conf_set_flag_slot, 78 | NGX_HTTP_LOC_CONF_OFFSET, 79 | offsetof(ngx_http_rds_csv_loc_conf_t, field_name_header), 80 | NULL }, 81 | 82 | { ngx_string("rds_csv_content_type"), 83 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF 84 | |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF 85 | |NGX_CONF_TAKE1, 86 | ngx_conf_set_str_slot, 87 | NGX_HTTP_LOC_CONF_OFFSET, 88 | offsetof(ngx_http_rds_csv_loc_conf_t, content_type), 89 | NULL }, 90 | 91 | { ngx_string("rds_csv_buffer_size"), 92 | NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF 93 | |NGX_CONF_TAKE1, 94 | ngx_conf_set_size_slot, 95 | NGX_HTTP_LOC_CONF_OFFSET, 96 | offsetof(ngx_http_rds_csv_loc_conf_t, buf_size), 97 | NULL }, 98 | 99 | ngx_null_command 100 | }; 101 | 102 | 103 | static ngx_http_module_t ngx_http_rds_csv_filter_module_ctx = { 104 | NULL, /* preconfiguration */ 105 | ngx_http_rds_csv_filter_init, /* postconfiguration */ 106 | 107 | ngx_http_rds_csv_create_main_conf, /* create main configuration */ 108 | NULL, /* init main configuration */ 109 | 110 | NULL, /* create server configuration */ 111 | NULL, /* merge server configuration */ 112 | 113 | ngx_http_rds_csv_create_loc_conf, /* create location configuration */ 114 | ngx_http_rds_csv_merge_loc_conf /* merge location configuration */ 115 | }; 116 | 117 | 118 | ngx_module_t ngx_http_rds_csv_filter_module = { 119 | NGX_MODULE_V1, 120 | &ngx_http_rds_csv_filter_module_ctx, /* module context */ 121 | ngx_http_rds_csv_commands, /* module directives */ 122 | NGX_HTTP_MODULE, /* module type */ 123 | NULL, /* init master */ 124 | NULL, /* init module */ 125 | NULL, /* init process */ 126 | NULL, /* init thread */ 127 | NULL, /* exit thread */ 128 | NULL, /* exit process */ 129 | NULL, /* exit master */ 130 | NGX_MODULE_V1_PADDING 131 | }; 132 | 133 | 134 | static ngx_int_t 135 | ngx_http_rds_csv_header_filter(ngx_http_request_t *r) 136 | { 137 | ngx_http_rds_csv_ctx_t *ctx; 138 | ngx_http_rds_csv_loc_conf_t *conf; 139 | size_t len; 140 | u_char *p; 141 | 142 | /* XXX maybe we can generate stub JSON strings like 143 | * {"errcode":403,"error":"Permission denied"} 144 | * for HTTP error pages? */ 145 | if ((r->headers_out.status < NGX_HTTP_OK) 146 | || (r->headers_out.status >= NGX_HTTP_SPECIAL_RESPONSE) 147 | || (r->headers_out.status == NGX_HTTP_NO_CONTENT) 148 | || (r->headers_out.status == NGX_HTTP_RESET_CONTENT)) 149 | { 150 | ngx_http_set_ctx(r, NULL, ngx_http_rds_csv_filter_module); 151 | 152 | dd("status is not OK: %d, skipping", (int) r->headers_out.status); 153 | 154 | return ngx_http_rds_csv_next_header_filter(r); 155 | } 156 | 157 | /* r->headers_out.status = 0; */ 158 | 159 | conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module); 160 | 161 | if (!conf->enabled) { 162 | return ngx_http_rds_csv_next_header_filter(r); 163 | } 164 | 165 | if (ngx_http_rds_csv_test_content_type(r) != NGX_OK) { 166 | return ngx_http_rds_csv_next_header_filter(r); 167 | } 168 | 169 | if (conf->content_type.len == sizeof(ngx_http_rds_csv_content_type) - 1 170 | && ngx_strncmp(conf->content_type.data, ngx_http_rds_csv_content_type, 171 | sizeof(ngx_http_rds_csv_content_type) - 1) == 0) 172 | { 173 | /* MIME type is text/csv, we process Content-Type 174 | * according to RFC 4180 */ 175 | 176 | len = sizeof(ngx_http_rds_csv_content_type) - 1 177 | + sizeof("; header=") - 1; 178 | 179 | if (conf->field_name_header) { 180 | len += sizeof("presence") - 1; 181 | 182 | } else { 183 | len += sizeof("absence") - 1; 184 | } 185 | 186 | p = ngx_palloc(r->pool, len); 187 | if (p == NULL) { 188 | return NGX_ERROR; 189 | } 190 | 191 | r->headers_out.content_type.len = len; 192 | r->headers_out.content_type_len = len; 193 | 194 | r->headers_out.content_type.data = p; 195 | 196 | p = ngx_copy(p, conf->content_type.data, conf->content_type.len); 197 | 198 | if (conf->field_name_header) { 199 | p = ngx_copy_literal(p, "; header=presence"); 200 | 201 | } else { 202 | p = ngx_copy_literal(p, "; header=absence"); 203 | } 204 | 205 | if (p - r->headers_out.content_type.data != (ssize_t) len) { 206 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 207 | "rds_csv: content type buffer error: %uz != %uz", 208 | (size_t) (p - r->headers_out.content_type.data), 209 | len); 210 | 211 | return NGX_ERROR; 212 | } 213 | 214 | } else { 215 | /* custom MIME-type, we just pass it through */ 216 | 217 | r->headers_out.content_type = conf->content_type; 218 | r->headers_out.content_type_len = conf->content_type.len; 219 | } 220 | 221 | ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_rds_csv_ctx_t)); 222 | 223 | if (ctx == NULL) { 224 | return NGX_ERROR; 225 | } 226 | 227 | ctx->tag = (ngx_buf_tag_t) &ngx_http_rds_csv_filter_module; 228 | 229 | ctx->state = state_expect_header; 230 | 231 | ctx->header_sent = 0; 232 | 233 | ctx->last_out = &ctx->out; 234 | 235 | /* set by ngx_pcalloc 236 | * ctx->out = NULL 237 | * ctx->busy_bufs = NULL 238 | * ctx->free_bufs = NULL 239 | * ctx->cached = (ngx_buf_t) 0 240 | * ctx->postponed = (ngx_buf_t) 0 241 | * ctx->avail_out = 0 242 | * ctx->col_names = NULL 243 | * ctx->col_count = 0 244 | * ctx->cur_col = 0 245 | * ctx->field_offset = 0 246 | * ctx->field_total = 0 247 | * ctx->field_data_rest = 0 248 | */ 249 | 250 | ngx_http_set_ctx(r, ctx, ngx_http_rds_csv_filter_module); 251 | 252 | ngx_http_clear_content_length(r); 253 | 254 | r->filter_need_in_memory = 1; 255 | 256 | /* we do postpone the header sending to the body filter */ 257 | return NGX_OK; 258 | } 259 | 260 | 261 | static ngx_int_t 262 | ngx_http_rds_csv_body_filter(ngx_http_request_t *r, ngx_chain_t *in) 263 | { 264 | ngx_http_rds_csv_ctx_t *ctx; 265 | ngx_int_t rc; 266 | 267 | if (in == NULL || r->header_only) { 268 | return ngx_http_rds_csv_next_body_filter(r, in); 269 | } 270 | 271 | ctx = ngx_http_get_module_ctx(r, ngx_http_rds_csv_filter_module); 272 | 273 | if (ctx == NULL) { 274 | return ngx_http_rds_csv_next_body_filter(r, in); 275 | } 276 | 277 | switch (ctx->state) { 278 | 279 | case state_expect_header: 280 | rc = ngx_http_rds_csv_process_header(r, in, ctx); 281 | break; 282 | 283 | case state_expect_col: 284 | rc = ngx_http_rds_csv_process_col(r, in, ctx); 285 | break; 286 | 287 | case state_expect_row: 288 | rc = ngx_http_rds_csv_process_row(r, in, ctx); 289 | break; 290 | 291 | case state_expect_field: 292 | rc = ngx_http_rds_csv_process_field(r, in, ctx); 293 | break; 294 | 295 | case state_expect_more_field_data: 296 | rc = ngx_http_rds_csv_process_more_field_data(r, in, ctx); 297 | break; 298 | 299 | case state_done: 300 | 301 | /* mark the remaining bufs as consumed */ 302 | 303 | dd("discarding bufs"); 304 | 305 | ngx_http_rds_csv_discard_bufs(r->pool, in); 306 | 307 | return NGX_OK; 308 | break; 309 | default: 310 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 311 | "rds_csv: invalid internal state: %d", 312 | ctx->state); 313 | 314 | rc = NGX_ERROR; 315 | 316 | break; 317 | } 318 | 319 | dd("body filter rc: %d", (int) rc); 320 | 321 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 322 | ctx->state = state_done; 323 | 324 | if (!ctx->header_sent) { 325 | ctx->header_sent = 1; 326 | 327 | if (rc == NGX_ERROR) { 328 | rc = NGX_HTTP_INTERNAL_SERVER_ERROR; 329 | } 330 | 331 | r->headers_out.status = rc; 332 | 333 | dd("sending ERROR headers"); 334 | 335 | ngx_http_rds_csv_next_header_filter(r); 336 | ngx_http_send_special(r, NGX_HTTP_LAST); 337 | 338 | return NGX_ERROR; 339 | } 340 | 341 | return NGX_ERROR; 342 | } 343 | 344 | dd("output bufs"); 345 | 346 | return ngx_http_rds_csv_output_bufs(r, ctx); 347 | } 348 | 349 | 350 | static ngx_int_t 351 | ngx_http_rds_csv_filter_init(ngx_conf_t *cf) 352 | { 353 | int multi_http_blocks; 354 | ngx_http_rds_csv_main_conf_t *rmcf; 355 | 356 | rmcf = ngx_http_conf_get_module_main_conf(cf, 357 | ngx_http_rds_csv_filter_module); 358 | 359 | if (ngx_http_rds_csv_prev_cycle != ngx_cycle) { 360 | ngx_http_rds_csv_prev_cycle = ngx_cycle; 361 | multi_http_blocks = 0; 362 | 363 | } else { 364 | multi_http_blocks = 1; 365 | } 366 | 367 | if (multi_http_blocks || rmcf->requires_filter) { 368 | ngx_http_rds_csv_next_header_filter = ngx_http_top_header_filter; 369 | ngx_http_top_header_filter = ngx_http_rds_csv_header_filter; 370 | 371 | ngx_http_rds_csv_next_body_filter = ngx_http_top_body_filter; 372 | ngx_http_top_body_filter = ngx_http_rds_csv_body_filter; 373 | } 374 | 375 | return NGX_OK; 376 | } 377 | 378 | 379 | static void * 380 | ngx_http_rds_csv_create_loc_conf(ngx_conf_t *cf) 381 | { 382 | ngx_http_rds_csv_loc_conf_t *conf; 383 | 384 | conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_rds_csv_loc_conf_t)); 385 | if (conf == NULL) { 386 | return NULL; 387 | } 388 | 389 | /* 390 | * set by ngx_pcalloc(): 391 | * 392 | * conf->content_type = { 0, NULL }; 393 | * conf->row_term = { 0, NULL }; 394 | */ 395 | 396 | conf->enabled = NGX_CONF_UNSET; 397 | conf->field_sep = NGX_CONF_UNSET_UINT; 398 | conf->buf_size = NGX_CONF_UNSET_SIZE; 399 | conf->field_name_header = NGX_CONF_UNSET; 400 | 401 | return conf; 402 | } 403 | 404 | 405 | static char * 406 | ngx_http_rds_csv_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) 407 | { 408 | ngx_http_rds_csv_loc_conf_t *prev = parent; 409 | ngx_http_rds_csv_loc_conf_t *conf = child; 410 | 411 | ngx_conf_merge_value(conf->enabled, prev->enabled, 0); 412 | 413 | ngx_conf_merge_value(conf->field_name_header, prev->field_name_header, 1); 414 | 415 | ngx_conf_merge_uint_value(conf->field_sep, prev->field_sep, 416 | (ngx_uint_t) ','); 417 | 418 | ngx_conf_merge_str_value(conf->row_term, prev->row_term, 419 | ngx_http_rds_csv_row_term); 420 | 421 | ngx_conf_merge_str_value(conf->content_type, prev->content_type, 422 | ngx_http_rds_csv_content_type); 423 | 424 | ngx_conf_merge_size_value(conf->buf_size, prev->buf_size, 425 | (size_t) ngx_pagesize); 426 | 427 | return NGX_CONF_OK; 428 | } 429 | 430 | 431 | static char * 432 | ngx_http_rds_csv_row_terminator(ngx_conf_t *cf, ngx_command_t *cmd, 433 | void *conf) 434 | { 435 | ngx_http_rds_csv_loc_conf_t *rlcf = conf; 436 | ngx_str_t *value; 437 | ngx_str_t *term; 438 | 439 | if (rlcf->row_term.len != 0) { 440 | return "is duplicate"; 441 | } 442 | 443 | value = cf->args->elts; 444 | 445 | term = &value[1]; 446 | 447 | if (term->len == 0) { 448 | return "takes empty string value"; 449 | } 450 | 451 | if ((term->len == 1 && term->data[0] == '\n') 452 | || (term->len == 2 && term->data[0] == '\r' && term->data[1] == '\n')) 453 | { 454 | return ngx_conf_set_str_slot(cf, cmd, conf); 455 | } 456 | 457 | return "takes a value other than \"\\n\" and \"\\r\\n\""; 458 | } 459 | 460 | 461 | static char * 462 | ngx_http_rds_csv_field_separator(ngx_conf_t *cf, ngx_command_t *cmd, 463 | void *conf) 464 | { 465 | ngx_http_rds_csv_loc_conf_t *rlcf = conf; 466 | ngx_str_t *value; 467 | ngx_str_t *sep; 468 | 469 | if (rlcf->field_sep != NGX_CONF_UNSET_UINT) { 470 | return "is duplicate"; 471 | } 472 | 473 | value = cf->args->elts; 474 | 475 | sep = &value[1]; 476 | 477 | if (sep->len != 1) { 478 | return "takes a string value not of length 1"; 479 | } 480 | 481 | if (sep->data[0] == ',' || sep->data[0] == ';' || sep->data[0] == '\t') { 482 | rlcf->field_sep = (ngx_uint_t) (sep->data[0]); 483 | return NGX_CONF_OK; 484 | } 485 | 486 | return "takes a value other than \",\", \";\", and \"\\t\""; 487 | } 488 | 489 | 490 | static char * 491 | ngx_http_rds_csv(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) 492 | { 493 | ngx_http_rds_csv_main_conf_t *rmcf; 494 | 495 | rmcf = ngx_http_conf_get_module_main_conf(cf, 496 | ngx_http_rds_csv_filter_module); 497 | 498 | rmcf->requires_filter = 1; 499 | 500 | return ngx_conf_set_flag_slot(cf, cmd, conf); 501 | } 502 | 503 | 504 | static void * 505 | ngx_http_rds_csv_create_main_conf(ngx_conf_t *cf) 506 | { 507 | ngx_http_rds_csv_main_conf_t *rmcf; 508 | 509 | rmcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_rds_csv_main_conf_t)); 510 | if (rmcf == NULL) { 511 | return NULL; 512 | } 513 | 514 | /* set by ngx_pcalloc: 515 | * rmcf->requires_filter = 0; 516 | */ 517 | 518 | return rmcf; 519 | } 520 | -------------------------------------------------------------------------------- /src/ngx_http_rds_csv_output.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) Yichun Zhang (agentzh) 4 | */ 5 | 6 | 7 | #ifndef DDEBUG 8 | #define DDEBUG 0 9 | #endif 10 | #include "ddebug.h" 11 | 12 | 13 | #include "ngx_http_rds_csv_filter_module.h" 14 | #include "ngx_http_rds_csv_output.h" 15 | #include "ngx_http_rds_csv_util.h" 16 | #include "resty_dbd_stream.h" 17 | 18 | 19 | static u_char *ngx_http_rds_csv_request_mem(ngx_http_request_t *r, 20 | ngx_http_rds_csv_ctx_t *ctx, size_t len); 21 | static ngx_int_t ngx_http_rds_csv_get_buf(ngx_http_request_t *r, 22 | ngx_http_rds_csv_ctx_t *ctx); 23 | static u_char *ngx_http_rds_csv_get_postponed(ngx_http_request_t *r, 24 | ngx_http_rds_csv_ctx_t *ctx, size_t len); 25 | static ngx_int_t ngx_http_rds_csv_submit_mem(ngx_http_request_t *r, 26 | ngx_http_rds_csv_ctx_t *ctx, size_t len, unsigned last_buf); 27 | static size_t ngx_get_num_size(uint64_t i); 28 | 29 | 30 | ngx_int_t 31 | ngx_http_rds_csv_output_literal(ngx_http_request_t *r, 32 | ngx_http_rds_csv_ctx_t *ctx, u_char *data, size_t len, 33 | int last_buf) 34 | { 35 | u_char *pos; 36 | 37 | pos = ngx_http_rds_csv_request_mem(r, ctx, len); 38 | if (pos == NULL) { 39 | return NGX_ERROR; 40 | } 41 | 42 | ngx_memcpy(pos, data, len); 43 | 44 | dd("before output chain"); 45 | 46 | if (last_buf) { 47 | ctx->seen_stream_end = 1; 48 | 49 | if (r != r->main) { 50 | last_buf = 0; 51 | } 52 | } 53 | 54 | return ngx_http_rds_csv_submit_mem(r, ctx, len, (unsigned) last_buf); 55 | } 56 | 57 | 58 | ngx_int_t 59 | ngx_http_rds_csv_output_bufs(ngx_http_request_t *r, 60 | ngx_http_rds_csv_ctx_t *ctx) 61 | { 62 | ngx_int_t rc; 63 | ngx_chain_t *cl; 64 | 65 | dd("entered output chain"); 66 | 67 | if (ctx->seen_stream_end) { 68 | ctx->seen_stream_end = 0; 69 | 70 | if (ctx->avail_out) { 71 | cl = ngx_alloc_chain_link(r->pool); 72 | if (cl == NULL) { 73 | return NGX_ERROR; 74 | } 75 | 76 | cl->buf = ctx->out_buf; 77 | cl->next = NULL; 78 | *ctx->last_out = cl; 79 | ctx->last_out = &cl->next; 80 | 81 | ctx->avail_out = 0; 82 | } 83 | } 84 | 85 | dd_dump_chain_size(); 86 | 87 | for ( ;; ) { 88 | if (ctx->out == NULL) { 89 | /* fprintf(stderr, "\n"); */ 90 | return NGX_OK; 91 | } 92 | 93 | /* fprintf(stderr, "XXX Relooping..."); */ 94 | 95 | rc = ngx_http_rds_csv_next_body_filter(r, ctx->out); 96 | 97 | if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { 98 | return rc; 99 | } 100 | 101 | #if defined(nginx_version) && nginx_version >= 1001004 102 | ngx_chain_update_chains(r->pool, &ctx->free_bufs, &ctx->busy_bufs, 103 | &ctx->out, ctx->tag); 104 | #else 105 | ngx_chain_update_chains(&ctx->free_bufs, &ctx->busy_bufs, &ctx->out, 106 | ctx->tag); 107 | #endif 108 | 109 | ctx->last_out = &ctx->out; 110 | } 111 | 112 | /* impossible to reach here */ 113 | return NGX_ERROR; 114 | } 115 | 116 | 117 | ngx_int_t 118 | ngx_http_rds_csv_output_header(ngx_http_request_t *r, 119 | ngx_http_rds_csv_ctx_t *ctx, ngx_http_rds_header_t *header) 120 | { 121 | u_char *pos, *last; 122 | size_t size; 123 | uintptr_t escape; 124 | unsigned last_buf = 0; 125 | unsigned need_quotes = 0; 126 | u_char sep; 127 | 128 | ngx_http_rds_csv_loc_conf_t *conf; 129 | 130 | /* calculate the buffer size */ 131 | 132 | conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module); 133 | 134 | if (conf->field_name_header) { 135 | size = sizeof("errcode,errstr,insert_id,affected_rows") - 1 136 | + conf->row_term.len; 137 | 138 | } else { 139 | size = 0; 140 | } 141 | 142 | sep = (u_char) conf->field_sep; 143 | 144 | size += 3 /* field seperators */ + conf->row_term.len; 145 | 146 | size += ngx_get_num_size(header->std_errcode); 147 | 148 | escape = ngx_http_rds_csv_escape_csv_str(sep, NULL, header->errstr.data, 149 | header->errstr.len, 150 | &need_quotes); 151 | 152 | if (need_quotes) { 153 | size += sizeof("\"\"") - 1; 154 | } 155 | 156 | size += header->errstr.len + escape 157 | + ngx_get_num_size(header->insert_id) 158 | + ngx_get_num_size(header->affected_rows); 159 | 160 | /* create the buffer */ 161 | 162 | pos = ngx_http_rds_csv_request_mem(r, ctx, size); 163 | if (pos == NULL) { 164 | return NGX_ERROR; 165 | } 166 | 167 | last = pos; 168 | 169 | /* fill up the buffer */ 170 | 171 | last = ngx_sprintf(last, "errcode%cerrstr%cinsert_id%caffected_rows%V" 172 | "%uD%c", sep, sep, sep, &conf->row_term, 173 | (uint32_t) header->std_errcode, sep); 174 | 175 | if (need_quotes) { 176 | *last++ = '"'; 177 | } 178 | 179 | if (escape == 0) { 180 | last = ngx_copy(last, header->errstr.data, header->errstr.len); 181 | 182 | } else { 183 | last = (u_char *) 184 | ngx_http_rds_csv_escape_csv_str(sep, last, 185 | header->errstr.data, 186 | header->errstr.len, NULL); 187 | } 188 | 189 | if (need_quotes) { 190 | *last++ = '"'; 191 | } 192 | 193 | last = ngx_sprintf(last, "%c%uL%c%uL%V", sep, header->insert_id, sep, 194 | header->affected_rows, &conf->row_term); 195 | 196 | if ((size_t) (last - pos) != size) { 197 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 198 | "rds_csv: output header buffer error: %uz != %uz", 199 | (size_t) (last - pos), size); 200 | 201 | return NGX_ERROR; 202 | } 203 | 204 | if (r == r->main) { 205 | last_buf = 1; 206 | } 207 | 208 | ctx->seen_stream_end = 1; 209 | 210 | return ngx_http_rds_csv_submit_mem(r, ctx, size, last_buf); 211 | } 212 | 213 | 214 | ngx_int_t 215 | ngx_http_rds_csv_output_field_names(ngx_http_request_t *r, 216 | ngx_http_rds_csv_ctx_t *ctx) 217 | { 218 | ngx_uint_t i; 219 | ngx_http_rds_column_t *col; 220 | size_t size; 221 | u_char *pos, *last; 222 | uintptr_t escape = 0; 223 | unsigned need_quotes; 224 | u_char sep; 225 | ngx_http_rds_csv_loc_conf_t *conf; 226 | 227 | conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module); 228 | 229 | sep = (u_char) conf->field_sep; 230 | 231 | size = ctx->col_count - 1 /* field sep count */ 232 | + conf->row_term.len; 233 | 234 | for (i = 0; i < ctx->col_count; i++) { 235 | col = &ctx->cols[i]; 236 | escape = ngx_http_rds_csv_escape_csv_str(sep, NULL, col->name.data, 237 | col->name.len, &need_quotes); 238 | 239 | dd("field escape: %d", (int) escape); 240 | 241 | if (need_quotes) { 242 | size += sizeof("\"\"") - 1; 243 | } 244 | 245 | size += col->name.len + escape; 246 | } 247 | 248 | ctx->generated_col_names = 1; 249 | 250 | pos = ngx_http_rds_csv_request_mem(r, ctx, size); 251 | if (pos == NULL) { 252 | return NGX_ERROR; 253 | } 254 | 255 | last = pos; 256 | 257 | for (i = 0; i < ctx->col_count; i++) { 258 | col = &ctx->cols[i]; 259 | 260 | escape = ngx_http_rds_csv_escape_csv_str(sep, NULL, col->name.data, 261 | col->name.len, &need_quotes); 262 | 263 | if (need_quotes) { 264 | *last++ = '"'; 265 | } 266 | 267 | if (escape == 0) { 268 | last = ngx_copy(last, col->name.data, col->name.len); 269 | 270 | } else { 271 | last = (u_char *) 272 | ngx_http_rds_csv_escape_csv_str(sep, last, 273 | col->name.data, 274 | col->name.len, NULL); 275 | } 276 | 277 | if (need_quotes) { 278 | *last++ = '"'; 279 | } 280 | 281 | if (i != ctx->col_count - 1) { 282 | *last++ = sep; 283 | } 284 | } 285 | 286 | last = ngx_copy(last, conf->row_term.data, conf->row_term.len); 287 | 288 | if ((size_t) (last - pos) != size) { 289 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 290 | "rds_csv: output field names buffer error: %uz != %uz", 291 | (size_t) (last - pos), size); 292 | 293 | return NGX_ERROR; 294 | } 295 | 296 | return ngx_http_rds_csv_submit_mem(r, ctx, size, 0); 297 | } 298 | 299 | 300 | ngx_int_t 301 | ngx_http_rds_csv_output_field(ngx_http_request_t *r, 302 | ngx_http_rds_csv_ctx_t *ctx, u_char *data, size_t len, int is_null) 303 | { 304 | u_char *pos, *last; 305 | ngx_http_rds_column_t *col; 306 | size_t size; 307 | uintptr_t val_escape = 0; 308 | unsigned need_quotes = 0; 309 | u_char sep; 310 | ngx_http_rds_csv_loc_conf_t *conf; 311 | #if DDEBUG 312 | u_char *p; 313 | #endif 314 | 315 | conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module); 316 | 317 | sep = (u_char) conf->field_sep; 318 | 319 | dd("reading row %llu, col %d, len %d", 320 | (unsigned long long) ctx->row, 321 | (int) ctx->cur_col, (int) len); 322 | 323 | /* calculate the buffer size */ 324 | 325 | if (ctx->cur_col == 0) { 326 | size = 0; 327 | 328 | } else { 329 | size = 1 /* field sep */; 330 | } 331 | 332 | col = &ctx->cols[ctx->cur_col]; 333 | 334 | if (len == 0 && ctx->field_data_rest > 0) { 335 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 336 | "rds_csv: at least one octet should go with the field " 337 | "size in one buf"); 338 | 339 | return NGX_ERROR; 340 | } 341 | 342 | if (is_null) { 343 | /* SQL NULL is just empty in the CSV field */ 344 | 345 | } else if (len == 0) { 346 | /* empty string is also empty */ 347 | 348 | } else { 349 | switch (col->std_type & 0xc000) { 350 | case rds_rough_col_type_float: 351 | case rds_rough_col_type_int: 352 | case rds_rough_col_type_bool: 353 | size += len; 354 | break; 355 | 356 | default: 357 | dd("string field found"); 358 | 359 | val_escape = ngx_http_rds_csv_escape_csv_str(sep, NULL, data, len, 360 | &need_quotes); 361 | 362 | if (ctx->field_data_rest > 0 && !need_quotes) { 363 | need_quotes = 1; 364 | } 365 | 366 | if (need_quotes) { 367 | if (ctx->field_data_rest == 0) { 368 | size += sizeof("\"\"") - 1; 369 | 370 | } else { 371 | size += sizeof("\"") - 1; 372 | } 373 | } 374 | 375 | size += len + val_escape; 376 | break; 377 | } 378 | } 379 | 380 | if (ctx->field_data_rest == 0 && ctx->cur_col == ctx->col_count - 1) { 381 | /* last column in the row */ 382 | size += conf->row_term.len; 383 | } 384 | 385 | /* allocate the buffer */ 386 | 387 | pos = ngx_http_rds_csv_request_mem(r, ctx, size); 388 | if (pos == NULL) { 389 | return NGX_ERROR; 390 | } 391 | 392 | last = pos; 393 | 394 | /* fill up the buffer */ 395 | 396 | if (ctx->cur_col != 0) { 397 | *last++ = sep; 398 | } 399 | 400 | if (is_null || len == 0) { 401 | /* do nothing */ 402 | 403 | } else { 404 | switch (col->std_type & 0xc000) { 405 | case rds_rough_col_type_int: 406 | case rds_rough_col_type_float: 407 | case rds_rough_col_type_bool: 408 | last = ngx_copy(last, data, len); 409 | break; 410 | 411 | default: 412 | /* string */ 413 | if (need_quotes) { 414 | *last++ = '"'; 415 | } 416 | 417 | if (val_escape == 0) { 418 | last = ngx_copy(last, data, len); 419 | 420 | } else { 421 | dd("field: string value escape non-zero: %d", 422 | (int) val_escape); 423 | 424 | #if DDEBUG 425 | p = last; 426 | #endif 427 | 428 | last = (u_char *) 429 | ngx_http_rds_csv_escape_csv_str(sep, last, data, len, 430 | NULL); 431 | 432 | #if DDEBUG 433 | dd("escaped value \"%.*s\" (len %d, escape %d, escape2 %d)", 434 | (int) (len + val_escape), 435 | p, (int) (len + val_escape), 436 | (int) val_escape, 437 | (int) ((last - p) - len)); 438 | #endif 439 | } 440 | 441 | if (need_quotes && ctx->field_data_rest == 0) { 442 | *last++ = '"'; 443 | } 444 | 445 | break; 446 | } 447 | } 448 | 449 | if (ctx->field_data_rest == 0 && ctx->cur_col == ctx->col_count - 1) { 450 | last = ngx_copy(last, conf->row_term.data, conf->row_term.len); 451 | } 452 | 453 | if ((size_t) (last - pos) != size) { 454 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 455 | "rds_csv: output field: buffer error (%d left)", 456 | (int) size - (last - pos)); 457 | 458 | return NGX_ERROR; 459 | } 460 | 461 | return ngx_http_rds_csv_submit_mem(r, ctx, size, 0); 462 | } 463 | 464 | 465 | ngx_int_t 466 | ngx_http_rds_csv_output_more_field_data(ngx_http_request_t *r, 467 | ngx_http_rds_csv_ctx_t *ctx, u_char *data, size_t len) 468 | { 469 | u_char *pos, *last; 470 | size_t size = 0; 471 | ngx_http_rds_column_t *col; 472 | uintptr_t escape = 0; 473 | #if DDEBUG 474 | u_char *p; 475 | #endif 476 | unsigned need_quotes; 477 | u_char sep; 478 | ngx_http_rds_csv_loc_conf_t *conf; 479 | 480 | conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module); 481 | 482 | sep = (u_char) conf->field_sep; 483 | 484 | /* calculate the buffer size */ 485 | 486 | col = &ctx->cols[ctx->cur_col]; 487 | 488 | switch (col->std_type & 0xc000) { 489 | case rds_rough_col_type_int: 490 | case rds_rough_col_type_float: 491 | case rds_rough_col_type_bool: 492 | size += len; 493 | break; 494 | 495 | default: 496 | /* string */ 497 | 498 | escape = ngx_http_rds_csv_escape_csv_str(sep, NULL, data, len, 499 | &need_quotes); 500 | 501 | size = len + escape; 502 | 503 | if (ctx->field_data_rest == 0) { 504 | size += sizeof("\"") - 1; 505 | } 506 | 507 | break; 508 | } 509 | 510 | if (ctx->field_data_rest == 0 && ctx->cur_col == ctx->col_count - 1) { 511 | /* last column in the row */ 512 | size += conf->row_term.len; 513 | } 514 | 515 | /* allocate the buffer */ 516 | 517 | pos = ngx_http_rds_csv_request_mem(r, ctx, size); 518 | if (pos == NULL) { 519 | return NGX_ERROR; 520 | } 521 | 522 | last = pos; 523 | 524 | /* fill up the buffer */ 525 | 526 | switch (col->std_type & 0xc000) { 527 | case rds_rough_col_type_int: 528 | case rds_rough_col_type_float: 529 | case rds_rough_col_type_bool: 530 | last = ngx_copy(last, data, len); 531 | break; 532 | 533 | default: 534 | /* string */ 535 | if (escape == 0) { 536 | last = ngx_copy(last, data, len); 537 | 538 | } else { 539 | dd("more field data: string value escape non-zero: %d", 540 | (int) escape); 541 | 542 | #if DDEBUG 543 | p = last; 544 | #endif 545 | 546 | last = (u_char *) ngx_http_rds_csv_escape_csv_str(sep, last, data, 547 | len, NULL); 548 | 549 | #if DDEBUG 550 | dd("escaped value \"%.*s\" (len %d, escape %d, escape2 %d)", 551 | (int) (len + escape), 552 | p, (int) (len + escape), 553 | (int) escape, 554 | (int) ((last - p) - len)); 555 | #endif 556 | } 557 | 558 | if (ctx->field_data_rest == 0) { 559 | *last++ = '"'; 560 | } 561 | 562 | break; 563 | } /* switch */ 564 | 565 | if (ctx->field_data_rest == 0 && ctx->cur_col == ctx->col_count - 1) { 566 | /* last column in the row */ 567 | last = ngx_copy(last, conf->row_term.data, conf->row_term.len); 568 | } 569 | 570 | if ((size_t) (last - pos) != size) { 571 | ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, 572 | "rds_csv: output more field data: buffer error " 573 | "(%d left)", (int) (size - (last - pos))); 574 | return NGX_ERROR; 575 | } 576 | 577 | return ngx_http_rds_csv_submit_mem(r, ctx, size, 0); 578 | } 579 | 580 | 581 | static u_char * 582 | ngx_http_rds_csv_request_mem(ngx_http_request_t *r, 583 | ngx_http_rds_csv_ctx_t *ctx, size_t len) 584 | { 585 | ngx_int_t rc; 586 | u_char *p; 587 | 588 | rc = ngx_http_rds_csv_get_buf(r, ctx); 589 | if (rc != NGX_OK) { 590 | return NULL; 591 | } 592 | 593 | if (ctx->avail_out < len) { 594 | p = ngx_http_rds_csv_get_postponed(r, ctx, len); 595 | if (p == NULL) { 596 | return NULL; 597 | } 598 | 599 | ctx->postponed.pos = p; 600 | ctx->postponed.last = p + len; 601 | 602 | return p; 603 | } 604 | 605 | return ctx->out_buf->last; 606 | } 607 | 608 | 609 | static ngx_int_t 610 | ngx_http_rds_csv_get_buf(ngx_http_request_t *r, ngx_http_rds_csv_ctx_t *ctx) 611 | { 612 | ngx_http_rds_csv_loc_conf_t *conf; 613 | 614 | dd("MEM enter"); 615 | 616 | if (ctx->avail_out) { 617 | return NGX_OK; 618 | } 619 | 620 | conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_csv_filter_module); 621 | 622 | if (ctx->free_bufs) { 623 | dd("MEM reusing temp buf from free_bufs"); 624 | 625 | ctx->out_buf = ctx->free_bufs->buf; 626 | ctx->free_bufs = ctx->free_bufs->next; 627 | 628 | } else { 629 | dd("MEM creating temp buf with size: %d", (int) conf->buf_size); 630 | ctx->out_buf = ngx_create_temp_buf(r->pool, conf->buf_size); 631 | if (ctx->out_buf == NULL) { 632 | return NGX_ERROR; 633 | } 634 | 635 | ctx->out_buf->tag = (ngx_buf_tag_t) &ngx_http_rds_csv_filter_module; 636 | ctx->out_buf->recycled = 1; 637 | } 638 | 639 | ctx->avail_out = conf->buf_size; 640 | 641 | return NGX_OK; 642 | } 643 | 644 | 645 | static u_char * 646 | ngx_http_rds_csv_get_postponed(ngx_http_request_t *r, 647 | ngx_http_rds_csv_ctx_t *ctx, size_t len) 648 | { 649 | u_char *p; 650 | 651 | dd("MEM enter"); 652 | 653 | if (ctx->cached.start == NULL) { 654 | goto alloc; 655 | } 656 | 657 | if ((size_t) (ctx->cached.end - ctx->cached.start) < len) { 658 | ngx_pfree(r->pool, ctx->cached.start); 659 | goto alloc; 660 | } 661 | 662 | return ctx->cached.start; 663 | 664 | alloc: 665 | 666 | p = ngx_palloc(r->pool, len); 667 | if (p == NULL) { 668 | return NULL; 669 | } 670 | 671 | ctx->cached.start = p; 672 | ctx->cached.end = p + len; 673 | 674 | return p; 675 | } 676 | 677 | 678 | static ngx_int_t 679 | ngx_http_rds_csv_submit_mem(ngx_http_request_t *r, 680 | ngx_http_rds_csv_ctx_t *ctx, size_t len, unsigned last_buf) 681 | { 682 | ngx_chain_t *cl; 683 | ngx_int_t rc; 684 | 685 | if (ctx->postponed.pos != NULL) { 686 | dd("MEM copy postponed data over to ctx->out for len %d", (int) len); 687 | 688 | for ( ;; ) { 689 | len = ctx->postponed.last - ctx->postponed.pos; 690 | if (len > ctx->avail_out) { 691 | len = ctx->avail_out; 692 | } 693 | 694 | ctx->out_buf->last = ngx_copy(ctx->out_buf->last, 695 | ctx->postponed.pos, len); 696 | 697 | ctx->avail_out -= len; 698 | 699 | ctx->postponed.pos += len; 700 | 701 | if (ctx->postponed.pos == ctx->postponed.last) { 702 | ctx->postponed.pos = NULL; 703 | } 704 | 705 | if (ctx->avail_out > 0) { 706 | break; 707 | } 708 | 709 | dd("MEM save ctx->out_buf"); 710 | 711 | cl = ngx_alloc_chain_link(r->pool); 712 | if (cl == NULL) { 713 | return NGX_ERROR; 714 | } 715 | 716 | cl->buf = ctx->out_buf; 717 | cl->next = NULL; 718 | *ctx->last_out = cl; 719 | ctx->last_out = &cl->next; 720 | 721 | if (ctx->postponed.pos == NULL) { 722 | ctx->out_buf->last_buf = last_buf; 723 | break; 724 | } 725 | 726 | rc = ngx_http_rds_csv_get_buf(r, ctx); 727 | if (rc != NGX_OK) { 728 | return NGX_ERROR; 729 | } 730 | } 731 | 732 | return NGX_OK; 733 | } 734 | 735 | dd("MEM consuming out_buf for %d", (int) len); 736 | 737 | ctx->out_buf->last += len; 738 | ctx->avail_out -= len; 739 | ctx->out_buf->last_buf = last_buf; 740 | 741 | if (ctx->avail_out == 0) { 742 | dd("MEM save ctx->out_buf"); 743 | 744 | cl = ngx_alloc_chain_link(r->pool); 745 | if (cl == NULL) { 746 | return NGX_ERROR; 747 | } 748 | 749 | cl->buf = ctx->out_buf; 750 | cl->next = NULL; 751 | *ctx->last_out = cl; 752 | ctx->last_out = &cl->next; 753 | } 754 | 755 | return NGX_OK; 756 | } 757 | 758 | 759 | static size_t 760 | ngx_get_num_size(uint64_t i) 761 | { 762 | size_t n = 0; 763 | 764 | do { 765 | i = i / 10; 766 | n++; 767 | } while (i > 0); 768 | 769 | return n; 770 | } 771 | --------------------------------------------------------------------------------