├── .github └── workflows │ ├── lint.yml │ ├── sast.yml │ └── tests.yml ├── .gitignore ├── .gitmodules ├── .luacheckrc ├── LICENSE ├── Makefile ├── README.md ├── config ├── config.make ├── lib └── resty │ ├── lmdb.lua │ └── lmdb │ ├── cdefs.lua │ ├── prefix.lua │ ├── status.lua │ └── transaction.lua ├── src ├── ngx_lua_resty_lmdb_module.c ├── ngx_lua_resty_lmdb_module.h ├── ngx_lua_resty_lmdb_prefix.c ├── ngx_lua_resty_lmdb_status.c └── ngx_lua_resty_lmdb_transaction.c ├── t ├── 00-init_mdb.t ├── 01-sanity.t ├── 02-multiple_dbs.t ├── 03-transactions.t ├── 04-not_enabled.t ├── 07-status.t ├── 08-max-key-size.t ├── 09-check-env-1.t ├── 09-check-env-2.t ├── 10-prefix.t ├── 11-validation-tag-plain.t └── 12-max-readers.t └── valgrind.suppress /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | pull_request: {} 5 | workflow_dispatch: {} 6 | push: 7 | branches: 8 | - main 9 | - master 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: ${{ github.event_name == 'pull_request' }} 14 | 15 | jobs: 16 | lua-check: 17 | name: Lua Check 18 | runs-on: ubuntu-latest 19 | permissions: 20 | contents: read 21 | issues: read 22 | checks: write 23 | pull-requests: write 24 | if: (github.actor != 'dependabot[bot]') 25 | 26 | steps: 27 | - name: Checkout source code 28 | uses: actions/checkout@v3 29 | 30 | # Optional step to run on only changed files 31 | - name: Get changed files 32 | id: changed-files 33 | uses: kong/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v37 34 | with: 35 | files: | 36 | **.lua 37 | 38 | - name: Lua Check 39 | if: steps.changed-files.outputs.any_changed == 'true' 40 | uses: Kong/public-shared-actions/code-check-actions/lua-lint@ac8939f0382827fbb43ce4e0028066a5ea4db01d #v4.1.2 41 | with: 42 | additional_args: '--no-default-config --config .luacheckrc' 43 | files: ${{ steps.changed-files.outputs.all_changed_files }} 44 | -------------------------------------------------------------------------------- /.github/workflows/sast.yml: -------------------------------------------------------------------------------- 1 | name: SAST 2 | 3 | on: 4 | pull_request: {} 5 | push: 6 | branches: 7 | - master 8 | - main 9 | workflow_dispatch: {} 10 | 11 | 12 | jobs: 13 | semgrep: 14 | name: Semgrep SAST 15 | runs-on: ubuntu-latest 16 | permissions: 17 | # required for all workflows 18 | security-events: write 19 | # only required for workflows in private repositories 20 | actions: read 21 | contents: read 22 | 23 | if: (github.actor != 'dependabot[bot]') 24 | 25 | steps: 26 | - uses: actions/checkout@v3 27 | - uses: Kong/public-shared-actions/security-actions/semgrep@ac8939f0382827fbb43ce4e0028066a5ea4db01d #v4.1.2 28 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: {} 5 | push: 6 | branches: [master] 7 | 8 | concurrency: 9 | group: ${{ github.workflow }}-${{ github.ref }} 10 | cancel-in-progress: ${{ github.event_name == 'pull_request' }} 11 | 12 | jobs: 13 | tests: 14 | name: Tests 15 | runs-on: ubuntu-20.04 16 | 17 | strategy: 18 | matrix: 19 | include: 20 | # TODO: arm64 21 | # latest and one version older for valgrind 22 | - nginx: "1.21.4" 23 | openssl: "1.1.1s" 24 | valgrind: "valgrind" 25 | lua_nginx_module: "v0.10.21" 26 | lua_resty_core: "v0.1.23" 27 | - nginx: "1.21.4" 28 | openssl: "1.1.1w" 29 | valgrind: "valgrind" 30 | lua_nginx_module: "v0.10.25" 31 | lua_resty_core: "v0.1.27" 32 | - nginx: "1.25.3" 33 | openssl: "1.1.1w" 34 | valgrind: "valgrind" 35 | lua_nginx_module: "v0.10.26" 36 | lua_resty_core: "v0.1.28" 37 | 38 | 39 | env: 40 | JOBS: 1 # must be 1 as LMDB tests interfere with each other 41 | SH: bash 42 | NGX_BUILD_JOBS: 3 43 | BASE_PATH: /home/runner/work/cache 44 | LUAJIT_PREFIX: /home/runner/work/cache/luajit21 45 | LUAJIT_LIB: /home/runner/work/cache/luajit21/lib 46 | LUAJIT_INC: /home/runner/work/cache/luajit21/include/luajit-2.1 47 | LUA_INCLUDE_DIR: /home/runner/work/cache/luajit21/include/luajit-2.1 48 | OPENSSL_PREFIX: /home/runner/work/cache/ssl 49 | # lib64 since openssl 3.0 50 | OPENSSL_LIB: /home/runner/work/cache/ssl/lib 51 | OPENSSL_INC: /home/runner/work/cache/ssl/include 52 | TEST_NGINX_SLEEP: 0.005 53 | TEST_NGINX_RANDOMIZE: 1 54 | LUACHECK_VER: 1.20.0 55 | CC: gcc 56 | NGX_BUILD_CC: gcc 57 | 58 | steps: 59 | - name: Checkout source code 60 | uses: actions/checkout@v2 61 | with: 62 | submodules: 'true' 63 | 64 | - name: Setup cache 65 | uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 66 | with: 67 | path: | 68 | /home/runner/work/cache 69 | key: ${{ runner.os }}-${{ hashFiles('**/tests.yml') }}-${{ hashFiles('**/*.c', '**/*.h') }}-nginx-${{ matrix.nginx }}-openssl-${{ matrix.openssl }} 70 | 71 | - name: Setup tools 72 | run: | 73 | sudo apt-get update 74 | sudo apt-get install -qq -y cpanminus axel ca-certificates valgrind haveged 75 | mkdir -p $OPENSSL_PREFIX $LUAJIT_PREFIX 76 | # perl cache 77 | pushd /home/runner/work/cache 78 | if [ ! -e perl ]; then sudo cpanm --notest Test::Nginx > build.log 2>&1 || (cat build.log && exit 1); cp -r /usr/local/share/perl/ .; else sudo cp -r perl /usr/local/share; fi 79 | # build tools at parent directory of cache 80 | cd .. 81 | git clone https://github.com/openresty/openresty.git ./openresty 82 | git clone https://github.com/openresty/nginx-devel-utils.git 83 | git clone https://github.com/simpl/ngx_devel_kit.git ./ndk-nginx-module 84 | git clone https://github.com/openresty/lua-nginx-module.git ./lua-nginx-module -b ${{ matrix.lua_nginx_module }} 85 | git clone https://github.com/openresty/no-pool-nginx.git ./no-pool-nginx 86 | git clone https://github.com/fffonion/lua-resty-openssl ../lua-resty-openssl 87 | # lua libraries at parent directory of current repository 88 | popd 89 | mkdir ../lib 90 | git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core -b ${{ matrix.lua_resty_core }} 91 | git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache 92 | git clone https://github.com/openresty/lua-resty-string.git ../lua-resty-string 93 | cp -r ../lua-resty-lrucache/lib/* ../lib/ 94 | cp -r ../lua-resty-string/lib/* ../lua-resty-core/lib/ 95 | find ../lib 96 | 97 | - name: Build OpenSSL 98 | run: | 99 | if [ "X$OPENSSL_HASH" != "X" ]; then wget https://github.com/openssl/openssl/archive/$OPENSSL_HASH.tar.gz -O - | tar zxf ; pushd openssl-$OPENSSL_HASH/; fi 100 | if [ "X$OPENSSL_HASH" = "X" ] ; then wget https://www.openssl.org/source/openssl-${{ matrix.openssl }}.tar.gz -O - | tar zxf -; pushd openssl-${{ matrix.openssl }}/; fi 101 | if [ ! -e $OPENSSL_PREFIX/include ]; then ./config shared -d --prefix=$OPENSSL_PREFIX -DPURIFY > build.log 2>&1 || (cat build.log && exit 1); fi 102 | if [ ! -e $OPENSSL_PREFIX/include ]; then make -j$JOBS > build.log 2>&1 || (cat build.log && exit 1); fi 103 | if [ ! -e $OPENSSL_PREFIX/include ]; then sudo make PATH=$PATH install_sw > build.log 2>&1 || (cat build.log && exit 1); fi 104 | 105 | - name: Build LuaJIT 106 | env: 107 | LUAJIT_CC_OPTS: ${{ matrix.luajit_cc_opts }} 108 | run: | 109 | if [ "X${{ matrix.valgrind }}" != "X" ]; then LUAJIT_CC_OPTS="$LUAJIT_CC_OPTS -DLUAJIT_NUMMODE=2 -DLUAJIT_${{ matrix.valgrind }} -DLUAJIT_USE_SYSMALLOC -O0"; fi 110 | export 111 | cd $LUAJIT_PREFIX 112 | if [ ! -e luajit2 ]; then git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git; fi 113 | cd luajit2 114 | make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS="-DLUA_USE_APICHECK -DLUA_USE_ASSERT -DLUAJIT_ENABLE_LUA52COMPAT ${{ matrix.luajit_cc_opts }}" > build.log 2>&1 || (cat build.log && exit 1) 115 | make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) 116 | 117 | - name: Build lua-cjson 118 | run: | 119 | if [ ! -e lua-cjson ]; then git clone https://github.com/openresty/lua-cjson.git ./lua-cjson; fi 120 | pushd ./lua-cjson && make && sudo PATH=$PATH make install && popd 121 | 122 | - name: Build Nginx 123 | env: 124 | NGINX_CC_OPTS: ${{ matrix.nginx_cc_opts }} 125 | run: | 126 | if [ "X${{ matrix.valgrind }}" != "X" ]; then NGINX_CC_OPTS="$NGINX_CC_OPTS -O0"; fi 127 | export PATH=$BASE_PATH/work/nginx/sbin:$BASE_PATH/../nginx-devel-utils:$PATH 128 | export LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH 129 | export NGX_LUA_LOC=$BASE_PATH/../lua-nginx-module 130 | export NGX_STREAM_LUA_LOC=$BASE_PATH/../stream-lua-nginx-module 131 | export 132 | cd $BASE_PATH 133 | if [ ! -e work ]; then ngx-build ${{ matrix.nginx }} --add-module=../ndk-nginx-module --add-module=../lua-nginx-module --add-module=../lua-resty-lmdb/lua-resty-lmdb --with-http_ssl_module --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-cc-opt="-I$OPENSSL_INC $NGINX_CC_OPTS" --with-ld-opt="-L$OPENSSL_LIB -Wl,-rpath,$OPENSSL_LIB" --with-debug > build.log 2>&1 || (cat build.log && exit 1); fi 134 | nginx -V 135 | ldd `which nginx`|grep -E 'luajit|ssl|pcre' 136 | 137 | - name: Run Test 138 | run: | 139 | export LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH 140 | export PATH=$BASE_PATH/work/nginx/sbin:$PATH 141 | TEST_NGINX_TIMEOUT=20 prove -j$JOBS -r t/ 142 | 143 | - name: Run Valgrind 144 | if: matrix.valgrind != '' 145 | run: | 146 | export LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH 147 | export TEST_NGINX_VALGRIND='--num-callers=100 -q --tool=memcheck --leak-check=full --show-possibly-lost=no --gen-suppressions=all --suppressions=valgrind.suppress --track-origins=yes' TEST_NGINX_TIMEOUT=60 TEST_NGINX_SLEEP=1 148 | export PATH=$BASE_PATH/work/nginx/sbin:$PATH 149 | stdbuf -o 0 -e 0 prove -j$JOBS -r t/ 2>&1 | grep -v "Connection refused" | grep -v "Retry connecting after" | tee output.log 150 | if grep -q 'insert_a_suppression_name_here' output.log; then echo "Valgrind found problems"; exit 1; fi 151 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | t/servroot 2 | 3 | *.swp 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "lmdb"] 2 | path = lmdb 3 | url = https://git.openldap.org/openldap/openldap.git 4 | branch = LMDB_0.9.33 5 | -------------------------------------------------------------------------------- /.luacheckrc: -------------------------------------------------------------------------------- 1 | std = "ngx_lua" 2 | unused_args = false 3 | redefined = false 4 | max_line_length = false 5 | 6 | 7 | not_globals = { 8 | "string.len", 9 | "table.getn", 10 | } 11 | 12 | 13 | ignore = { 14 | "6.", -- ignore whitespace warnings 15 | } 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021-2022 Kong Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | OPENRESTY_PREFIX=/usr/local/openresty 2 | 3 | #LUA_VERSION := 5.1 4 | PREFIX ?= /usr/local 5 | LUA_INCLUDE_DIR ?= $(PREFIX)/include 6 | LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION) 7 | INSTALL ?= install 8 | 9 | .PHONY: all test install 10 | 11 | all: ; 12 | 13 | install: all 14 | $(INSTALL) -d $(DESTDIR)$(LUA_LIB_DIR)/resty/lmdb/ 15 | $(INSTALL) -m 664 lib/resty/lmdb/*.lua $(DESTDIR)$(LUA_LIB_DIR)/resty/lmdb/ 16 | $(INSTALL) -m 664 lib/resty/lmdb.lua $(DESTDIR)$(LUA_LIB_DIR)/resty/ 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lua-resty-lmdb 2 | 3 | This module allows OpenResty applications to use the LMDB (Lightning Memory-Mapped Database) 4 | inside the Nginx worker process. It has two parts, a core module built into Nginx that 5 | controls the life cycle of the database environment, and a FFI based Lua binding for 6 | interacting with the module to access/change data. 7 | 8 | # Table of Contents 9 | 10 | * [lua-resty-lmdb](#lua-resty-lmdb) 11 | * [APIs](#apis) 12 | * [resty.lmdb](#restylmdb) 13 | * [get](#get) 14 | * [set](#set) 15 | * [get_env_info](#get_env_info) 16 | * [db\_drop](#db_drop) 17 | * [prefix](#prefix) 18 | * [resty.lmdb.transaction](#restylmdbtransaction) 19 | * [reset](#reset) 20 | * [get](#get) 21 | * [set](#set) 22 | * [db\_open](#db_open) 23 | * [db\_drop](#db_drop) 24 | * [commit](#commit) 25 | * [resty.lmdb.prefix](#restylmdbprefix) 26 | * [page](#page) 27 | * [Directives](#directives) 28 | * [lmdb_environment_path](#lmdb_environment_path) 29 | * [lmdb_max_databases](#lmdb_max_databases) 30 | * [lmdb_map_size](#lmdb_map_size) 31 | * [lmdb_validation_tag](#lmdb_validation_tag) 32 | * [Copyright and license](#copyright-and-license) 33 | 34 | ## APIs 35 | 36 | ### resty.lmdb 37 | 38 | #### get 39 | 40 | **syntax:** *value, err = lmdb.get(key, db?)* 41 | 42 | **context:** *any context **except** init_by_lua** 43 | 44 | Gets the value corresponding to `key` from LMDB database `db`. If `db` is omitted, 45 | it defaults to `"_default"`. 46 | 47 | If the key does not exist, `nil` will be returned. 48 | 49 | In case of error, `nil` and a string describing the error will be returned instead. 50 | 51 | [Back to TOC](#table-of-contents) 52 | 53 | #### set 54 | 55 | **syntax:** *ok, err = lmdb.set(key, value, db?)* 56 | 57 | **context:** *any context **except** init_by_lua** 58 | 59 | Sets the value corresponding to `key` to `value` inside LMDB database `db`. If `db` is omitted, 60 | it defaults to `"_default"`. 61 | 62 | Setting a key's value to `nil` will remove that key from the corresponding database. 63 | 64 | In case of error, `nil` and a string describing the error will be returned instead. 65 | 66 | [Back to TOC](#table-of-contents) 67 | 68 | #### get_env_info 69 | 70 | **syntax:** *status, err = lmdb.get_env_info()* 71 | 72 | **context:** *any context **except** init_by_lua** 73 | 74 | Get the LMDB database runtime information. `status` table struct as below. 75 | ``` 76 | { 77 | "page_size": 4096, 78 | "max_readers":126, 79 | "num_readers": 1, 80 | "allocated_pages": 2, 81 | "in_use_pages": 0, 82 | "entries": 0, 83 | "map_size": 10485760 # in bytes 84 | } 85 | ``` 86 | 87 | In case of error, `nil` and a string describing the error will be returned instead. 88 | 89 | 90 | [Back to TOC](#table-of-contents) 91 | 92 | ### db\_drop 93 | 94 | **syntax:** *ok, err = lmdb.db_drop(delele?, db?)* 95 | 96 | **context:** *any context **except** init_by_lua** 97 | 98 | Clears the contents of database `db`. If `delete` is `true`, then the database handle is also dropped. 99 | 100 | In case of error, `nil` and a string describing the error will be returned instead. 101 | 102 | [Back to TOC](#table-of-contents) 103 | 104 | ### prefix 105 | 106 | **syntax:** *for key, value in lmdb.prefix(prefix) do* 107 | 108 | **context:** *any context* 109 | 110 | Returns all key and their associated value for keys starting with `prefix`. 111 | For example, if the database contains: 112 | 113 | ``` 114 | key1: value1 115 | key11: value11 116 | key2: value2 117 | ``` 118 | 119 | Then a call of `lmdb.prefix("key")` will yield `key1`, `key11` and `key2` respectively. 120 | 121 | In case of errors while fetching from LMDB, `key` will be `nil` and `value` will be 122 | a string describing the error. The caller must anticipate this happening and check each return 123 | value carefully before consuming. 124 | 125 | **Warning on transaction safety:** Since the number of keys that could potentially 126 | be returned with this method could be very large, this method does not return all 127 | results inside a single transaction as this will be very expensive. Instead, this 128 | method gets keys from LMDB in batches using different read transaction. Therefore, it 129 | is possible that the database content has changed between batches. We may introduce a 130 | mechanism for detecting this case in the future, but for now there is a small opportunity 131 | for this to happen and you should guard your application for concurrent writes if this 132 | is a huge concern. This function makes best effort to detect when database content 133 | definitely changed between iterations, in this case `nil, "DB content changed while iterating"` 134 | will be returned from the iterator. 135 | 136 | [Back to TOC](#table-of-contents) 137 | 138 | ### resty.lmdb.transaction 139 | 140 | #### reset 141 | 142 | **syntax:** *txn:reset()* 143 | 144 | **context:** *any context* 145 | 146 | Resets a transaction object. Removes all existing transactions and results (if any) from the object but 147 | keeps the table's capacity. After this call the transaction can be reused as if it was a new transaction 148 | returned by `transaction.begin()`. 149 | 150 | [Back to TOC](#table-of-contents) 151 | 152 | #### get 153 | 154 | **syntax:** *txn:get(key, db?)* 155 | 156 | **context:** *any context* 157 | 158 | Appends a `get` operation in the transactions table. If `db` is omitted, 159 | it defaults to `"_default"`. The output table contains the following 160 | fields: 161 | 162 | * `value`: Value for `key`, or `nil` if `key` is not found 163 | 164 | [Back to TOC](#table-of-contents) 165 | 166 | #### set 167 | 168 | **syntax:** *txn:set(key, value, db?)* 169 | 170 | **context:** *any context* 171 | 172 | Appends a `set` operation in the transactions table. If `db` is omitted, 173 | it defaults to `"_default"`. The output able contains the following 174 | fields: 175 | 176 | * `result`: Always `true` for successful transaction commits 177 | 178 | [Back to TOC](#table-of-contents) 179 | 180 | #### db\_open 181 | 182 | **syntax:** *txn:db_open(create, db?)* 183 | 184 | **context:** *any context* 185 | 186 | Appends a `db_open` operation in the transactions table. If `db` is omitted, 187 | it defaults to `"_default"`. This operation does not return anything 188 | in case of successful transaction commits. 189 | 190 | [Back to TOC](#table-of-contents) 191 | 192 | #### db\_drop 193 | 194 | **syntax:** *txn:db_drop(delete, db?)* 195 | 196 | **context:** *any context* 197 | 198 | Appends a `db_drop` operation in the transactions table. If `db` is omitted, 199 | it defaults to `"_default"`. This operation does not return anything 200 | in case of successful transaction commits. 201 | 202 | [Back to TOC](#table-of-contents) 203 | 204 | #### commit 205 | 206 | **syntax:** *local res, err = txn:commit()* 207 | 208 | **context:** *any context **except** init_by_lua** 209 | 210 | Commits all operations currently inside the transactions table using a single LMDB 211 | transaction. Since LMDB transaction exhibits ACID (atomicity, consistency, isolation, durability) 212 | properties, this method will either commit all operations at once or fail without causing 213 | side effects. 214 | 215 | In case of successful transaction commit, `true` will be returned. Output value of each operation 216 | can be accessed like this: `txn[3].value`. Please note that Lua table index starts at `1` and the 217 | order corresponds to the order operations were appended into the transaction. So the first operation's 218 | output will be inside `txn[1]` and second operation's result will be inside `txn[2]`. 219 | 220 | In case of any error during the transaction, it will be rolled back and `nil` and 221 | an string describing the reason of the failure will be returned instead. Accessing the output value 222 | from the `txn` table when `commit()` returned an error is undefined. 223 | 224 | [Back to TOC](#table-of-contents) 225 | 226 | ### resty.lmdb.prefix 227 | 228 | #### page 229 | 230 | **syntax:** *res, err_or_more = prefix.page(start, prefix, db?, page_size?)* 231 | 232 | **context:** *any context* 233 | 234 | Return all keys `>= start` and starts with `prefix`. If `db` is omitted, 235 | it defaults to `"_default"`. 236 | 237 | If `page_size` is specified, up to `page_size` results will be returned. The `page_size` 238 | cannot be smaller than 1. 239 | 240 | The return value of this function is a table `res` where `res[1].key` and `res[1].value` 241 | corresponds to the first key and value, `res[2].key` and `res[2].value` corresponds to the 242 | second and etc. If no keys matched the provided criteria, then an empty table will be 243 | returned. 244 | 245 | In case of success, the second return value will be a boolean indicating if more keys are 246 | possibly present. However, even when this value is `true`, it is possible subsequent `page` 247 | might return an empty list. If this value is `false`, then it is guaranteed no more keys 248 | matching the `prefix` is available. 249 | 250 | In case of errors, `nil` and an string describing the reason of the failure will be returned. 251 | 252 | This is a low level function, most of the use case should use the higher level 253 | [lmdb.prefix](#prefix) iterator instead. 254 | 255 | [Back to TOC](#table-of-contents) 256 | 257 | ## Directives 258 | 259 | ### lmdb_environment_path 260 | 261 | **syntax:** *lmdb_environment_path path;* 262 | 263 | **context:** *main* 264 | 265 | Set the directory in which the LMDB database files reside. 266 | 267 | [Back to TOC](#table-of-contents) 268 | 269 | ### lmdb_max_databases 270 | 271 | **syntax:** *lmdb_max_databases number;* 272 | 273 | **context:** *main* 274 | 275 | Set the maximum number of named databases, the default value is `1`. 276 | 277 | [Back to TOC](#table-of-contents) 278 | 279 | ### lmdb_map_size 280 | 281 | **syntax:** *lmdb_map_size number;* 282 | 283 | **context:** *main* 284 | 285 | Set the size of the memory map, the default value is `1048576`(1MB). 286 | 287 | [Back to TOC](#table-of-contents) 288 | 289 | ### lmdb_validation_tag 290 | 291 | **syntax:** *lmdb_validation_tag value;* 292 | 293 | **default:** *none* 294 | 295 | **context:** *main* 296 | 297 | Set a content validation tag into LMDB. 298 | When LMDB starts, it will check the tag value, 299 | if the value is different from the directive value, 300 | the content of LMDB will be cleaned up. 301 | 302 | When this directive is not set, tag validation is disabled. 303 | 304 | [Back to TOC](#table-of-contents) 305 | 306 | ## Copyright and license 307 | 308 | Copyright (c) 2021-2022 Kong, Inc. 309 | 310 | Licensed under the Apache License, Version 2.0 . 312 | Files in the project may not be copied, modified, or distributed except according to those terms. 313 | 314 | [Back to TOC](#table-of-contents) 315 | 316 | -------------------------------------------------------------------------------- /config: -------------------------------------------------------------------------------- 1 | ngx_module_type=CORE 2 | ngx_module_name=ngx_lua_resty_lmdb_module 3 | ngx_module_srcs="$ngx_addon_dir/src/ngx_lua_resty_lmdb_module.c 4 | $ngx_addon_dir/src/ngx_lua_resty_lmdb_transaction.c 5 | $ngx_addon_dir/src/ngx_lua_resty_lmdb_prefix.c 6 | $ngx_addon_dir/src/ngx_lua_resty_lmdb_status.c 7 | " 8 | ngx_module_incs="$ngx_addon_dir/lmdb/libraries/liblmdb $ngx_addon_dir/src" 9 | 10 | . auto/module 11 | 12 | LINK_DEPS="$LINK_DEPS $ngx_addon_dir/lmdb/libraries/liblmdb/liblmdb.a" 13 | CORE_LIBS="$CORE_LIBS $ngx_addon_dir/lmdb/libraries/liblmdb/liblmdb.a" 14 | 15 | ngx_addon_name=$ngx_module_name 16 | -------------------------------------------------------------------------------- /config.make: -------------------------------------------------------------------------------- 1 | # Note: indention matters, use TAB below 2 | cat <>$NGX_MAKEFILE 3 | 4 | $ngx_addon_dir/lmdb/libraries/liblmdb/liblmdb.a: 5 | echo "Building liblmdb"; \\ 6 | \$(MAKE) -C $ngx_addon_dir/lmdb/libraries/liblmdb; \\ 7 | echo "Finished building liblmdb" 8 | 9 | EOF -------------------------------------------------------------------------------- /lib/resty/lmdb.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | 3 | 4 | local transaction = require("resty.lmdb.transaction") 5 | local prefix = require("resty.lmdb.prefix") 6 | local status = require("resty.lmdb.status") 7 | 8 | 9 | local assert = assert 10 | local prefix_page = prefix.page 11 | local get_phase = ngx.get_phase 12 | local ngx_sleep = ngx.sleep 13 | 14 | 15 | local CACHED_TXN = transaction.begin(1) 16 | local CAN_YIELD_PHASES = { 17 | rewrite = true, 18 | server_rewrite = true, 19 | access = true, 20 | content = true, 21 | timer = true, 22 | ssl_client_hello = true, 23 | ssl_certificate = true, 24 | ssl_session_fetch = true, 25 | preread = true, 26 | } 27 | 28 | 29 | function _M.get(key, db) 30 | CACHED_TXN:reset() 31 | CACHED_TXN:get(key, db) 32 | local res, err = CACHED_TXN:commit() 33 | if not res then 34 | return nil, err 35 | end 36 | 37 | return CACHED_TXN[1].result 38 | end 39 | 40 | 41 | function _M.set(key, value, db) 42 | CACHED_TXN:reset() 43 | CACHED_TXN:set(key, value, db) 44 | local res, err = CACHED_TXN:commit() 45 | if not res then 46 | return nil, err 47 | end 48 | 49 | return true 50 | end 51 | 52 | 53 | function _M.db_drop(delete, db) 54 | delete = not not delete 55 | 56 | CACHED_TXN:reset() 57 | CACHED_TXN:db_drop(delete, db) 58 | local res, err = CACHED_TXN:commit() 59 | if not res then 60 | return nil, err 61 | end 62 | 63 | return true 64 | end 65 | 66 | 67 | function _M.prefix(prefix, db) 68 | local res, i, res_n, err_or_more 69 | local last = prefix 70 | local can_yield = CAN_YIELD_PHASES[get_phase()] 71 | 72 | return function() 73 | ::more:: 74 | if not res then 75 | -- need to fetch more data 76 | res, err_or_more = prefix_page(last, prefix, db) 77 | if not res then 78 | return nil, err_or_more 79 | end 80 | 81 | res_n = #res 82 | if res_n == 0 or (i and res_n == 1) then 83 | return nil 84 | end 85 | 86 | if i then 87 | -- not the first call to prefix_page 88 | if res[1].key ~= last then 89 | return nil, "DB content changed while iterating" 90 | end 91 | 92 | -- this is not sufficient to prove DB content did not change, 93 | -- but at least the resume point did not change. 94 | -- skip the first key 95 | i = 2 96 | 97 | else 98 | -- first call to prefix_page 99 | i = 1 100 | end 101 | end 102 | 103 | assert(res_n > 0) 104 | 105 | if i > res_n then 106 | if err_or_more then 107 | last = res[i - 1].key 108 | res = nil 109 | 110 | if can_yield then 111 | ngx_sleep(0) 112 | end 113 | 114 | goto more 115 | end 116 | 117 | -- more = false 118 | 119 | return nil 120 | end 121 | 122 | local key = res[i].key 123 | local value = res[i].value 124 | i = i + 1 125 | 126 | return key, value 127 | end 128 | end 129 | 130 | 131 | _M.get_env_info = status.get_env_info 132 | 133 | 134 | return _M 135 | -------------------------------------------------------------------------------- /lib/resty/lmdb/cdefs.lua: -------------------------------------------------------------------------------- 1 | local ffi = require("ffi") 2 | local base = require("resty.core.base") 3 | 4 | 5 | local DEFAULT_VALUE_BUF_SIZE = 512 * 2048 -- 1MB 6 | base.set_string_buf_size(DEFAULT_VALUE_BUF_SIZE) 7 | 8 | 9 | ffi.cdef([[ 10 | typedef unsigned int MDB_dbi; 11 | 12 | 13 | typedef enum { 14 | NGX_LMDB_OP_GET = 0, 15 | NGX_LMDB_OP_PREFIX, 16 | NGX_LMDB_OP_SET, 17 | NGX_LMDB_OP_DB_OPEN, 18 | NGX_LMDB_OP_DB_DROP 19 | } ngx_lua_resty_lmdb_operation_e; 20 | 21 | 22 | typedef struct { 23 | ngx_lua_resty_lmdb_operation_e opcode; 24 | ngx_str_t key; /* GET, SET */ 25 | ngx_str_t value; /* GET, SET */ 26 | MDB_dbi dbi; /* ALL OPS */ 27 | unsigned int flags; /* SET, DROP */ 28 | } ngx_lua_resty_lmdb_operation_t; 29 | 30 | typedef struct { 31 | size_t map_size; /**< Size of the data memory map */ 32 | unsigned int page_size; /**< Size of a database page. */ 33 | unsigned int max_readers; /**< max reader slots in the environment */ 34 | unsigned int num_readers; /**< max reader slots used in the environment */ 35 | unsigned int allocated_pages; /**< number of pages allocated */ 36 | size_t in_use_pages; /**< number of pages currently in-use */ 37 | unsigned int entries; /**< the number of entries (key/value pairs) in the environment */ 38 | } ngx_lua_resty_lmdb_ffi_status_t; 39 | 40 | int ngx_lua_resty_lmdb_ffi_env_info(ngx_lua_resty_lmdb_ffi_status_t *lst, char **err); 41 | 42 | 43 | int ngx_lua_resty_lmdb_ffi_execute(ngx_lua_resty_lmdb_operation_t *ops, 44 | size_t n, int need_write, unsigned char *buf, size_t buf_len, char **err); 45 | int ngx_lua_resty_lmdb_ffi_prefix(ngx_lua_resty_lmdb_operation_t *ops, 46 | size_t n, unsigned char *buf, size_t buf_len, char **err); 47 | ]]) 48 | -------------------------------------------------------------------------------- /lib/resty/lmdb/prefix.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | 3 | 4 | local ffi = require("ffi") 5 | local table_new = require("table.new") 6 | require("resty.lmdb.cdefs") 7 | local transaction = require("resty.lmdb.transaction") 8 | local base = require("resty.core.base") 9 | 10 | 11 | local C = ffi.C 12 | -- DEFAULT_OPS_SIZE must be >= 2, 13 | -- see the function comment for ngx_lua_resty_lmdb_ffi_prefix 14 | local DEFAULT_OPS_SIZE = 512 15 | local DEFAULT_DB = transaction.DEFAULT_DB 16 | local NGX_ERROR = ngx.ERROR 17 | local NGX_AGAIN = ngx.AGAIN 18 | 19 | 20 | local ffi_string = ffi.string 21 | local ffi_new = ffi.new 22 | local get_dbi = transaction.get_dbi 23 | local err_ptr = base.get_errmsg_ptr() 24 | local get_string_buf = base.get_string_buf 25 | local get_string_buf_size = base.get_string_buf_size 26 | local assert = assert 27 | local math_max = math.max 28 | 29 | function _M.page(start, prefix, db, page_size) 30 | if not page_size then 31 | page_size = DEFAULT_OPS_SIZE 32 | end 33 | 34 | assert(page_size >= 1, "page_size must be at least 1") 35 | 36 | -- the function ngx_lua_resty_lmdb_ffi_prefix requires at least 2 operations 37 | -- special handling for the case when page_size is less than 2 38 | local query_size = math_max(page_size, 2) 39 | 40 | local value_buf_size = get_string_buf_size() 41 | local ops = ffi_new("ngx_lua_resty_lmdb_operation_t[?]", query_size) 42 | 43 | local dbi, err = get_dbi(false, db or DEFAULT_DB) 44 | if err then 45 | return nil, "unable to open DB for access: " .. err 46 | 47 | elseif not dbi then 48 | return nil, "DB " .. db .. " does not exist" 49 | end 50 | 51 | ops[0].dbi = dbi 52 | 53 | ::again:: 54 | ops[0].opcode = C.NGX_LMDB_OP_PREFIX 55 | ops[0].key.data = start 56 | ops[0].key.len = #start 57 | 58 | ops[1].opcode = C.NGX_LMDB_OP_PREFIX 59 | ops[1].key.data = prefix 60 | ops[1].key.len = #prefix 61 | 62 | local buf = get_string_buf(value_buf_size, false) 63 | local ret = C.ngx_lua_resty_lmdb_ffi_prefix(ops, query_size, 64 | buf, value_buf_size, err_ptr) 65 | if ret == NGX_ERROR then 66 | return nil, ffi_string(err_ptr[0]) 67 | end 68 | 69 | if ret == NGX_AGAIN then 70 | value_buf_size = value_buf_size * 2 71 | goto again 72 | end 73 | 74 | if ret == 0 then 75 | -- unlikely case 76 | return {}, false 77 | end 78 | 79 | assert(ret > 0) 80 | 81 | -- special handling for the case when page_size is less than 2 82 | if ret > page_size then 83 | -- we then blindly use ret == page_size to indicate there are more keys 84 | ret = page_size 85 | end 86 | 87 | local res = table_new(ret, 0) 88 | 89 | for i = 1, ret do 90 | local cop = ops[i - 1] 91 | 92 | assert(cop.opcode == C.NGX_LMDB_OP_PREFIX) 93 | 94 | local pair = { 95 | key = ffi_string(cop.key.data, cop.key.len), 96 | value = ffi_string(cop.value.data, cop.value.len), 97 | } 98 | 99 | res[i] = pair 100 | end 101 | 102 | -- if ret == page_size, then it is possible there are more keys 103 | return res, ret == page_size 104 | end 105 | 106 | 107 | return _M 108 | -------------------------------------------------------------------------------- /lib/resty/lmdb/status.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | 3 | 4 | require("resty.lmdb.cdefs") 5 | local ffi = require("ffi") 6 | local base = require("resty.core.base") 7 | 8 | local C = ffi.C 9 | local ffi_string = ffi.string 10 | local ffi_new = ffi.new 11 | local tonumber = tonumber 12 | local NGX_ERROR = ngx.ERROR 13 | 14 | local err_ptr = base.get_errmsg_ptr() 15 | 16 | 17 | function _M.get_env_info() 18 | local env_status = ffi_new("ngx_lua_resty_lmdb_ffi_status_t[1]") 19 | local ret = C.ngx_lua_resty_lmdb_ffi_env_info(env_status, err_ptr) 20 | if ret == NGX_ERROR then 21 | return nil, ffi_string(err_ptr[0]) 22 | end 23 | 24 | assert(env_status[0] ~= nil) 25 | 26 | return { 27 | map_size = tonumber(env_status[0].map_size), 28 | page_size = tonumber(env_status[0].page_size), 29 | max_readers = tonumber(env_status[0].max_readers), 30 | num_readers = tonumber(env_status[0].num_readers), 31 | allocated_pages = tonumber(env_status[0].allocated_pages), 32 | in_use_pages = tonumber(env_status[0].in_use_pages), 33 | entries = tonumber(env_status[0].entries), 34 | } 35 | end 36 | 37 | 38 | return _M 39 | -------------------------------------------------------------------------------- /lib/resty/lmdb/transaction.lua: -------------------------------------------------------------------------------- 1 | local _M = {} 2 | 3 | 4 | require("resty.lmdb.cdefs") 5 | local ffi = require("ffi") 6 | local base = require("resty.core.base") 7 | local table_new = require("table.new") 8 | 9 | 10 | local err_ptr = base.get_errmsg_ptr() 11 | local get_string_buf = base.get_string_buf 12 | local get_string_buf_size = base.get_string_buf_size 13 | local setmetatable = setmetatable 14 | local assert = assert 15 | local math_max = math.max 16 | local type = type 17 | local C = ffi.C 18 | local ffi_string = ffi.string 19 | local ffi_new = ffi.new 20 | local MIN_OPS_N = 16 21 | local NGX_ERROR = ngx.ERROR 22 | local NGX_AGAIN = ngx.AGAIN 23 | local NGX_OK = ngx.OK 24 | local _TXN_MT = {} 25 | _TXN_MT.__index = _TXN_MT 26 | 27 | 28 | local CACHED_DBI = {} 29 | local MDB_CREATE = 0x40000 30 | local DEFAULT_DB = "_default" 31 | _M.DEFAULT_DB = DEFAULT_DB 32 | 33 | 34 | function _M.begin(hint) 35 | hint = hint or 4 36 | 37 | local txn = table_new(hint, 4) 38 | 39 | txn.n = 0 40 | txn.write = false 41 | txn.ops_capacity = 0 42 | txn.ops = nil 43 | 44 | return setmetatable(txn, _TXN_MT) 45 | end 46 | 47 | 48 | local normalize_key 49 | do 50 | local resty_sha256 = assert(require("resty.sha256").new()) 51 | 52 | -- lmdb has 511 bytes limitation for key 53 | local MAX_KEY_SIZE = 511 54 | 55 | local sha256 = function(str) 56 | resty_sha256:reset() 57 | resty_sha256:update(str) 58 | return resty_sha256:final() 59 | end 60 | 61 | normalize_key = function(key) 62 | if key and #key > MAX_KEY_SIZE then 63 | return assert(sha256(key)) 64 | end 65 | 66 | return key 67 | end 68 | end 69 | 70 | 71 | local get_dbi 72 | do 73 | local CACHED_TXN_DBI = _M.begin(1) 74 | function _M.get_dbi(create, db) 75 | local dbi = CACHED_DBI[db] 76 | if dbi then 77 | return dbi 78 | end 79 | 80 | CACHED_TXN_DBI:reset() 81 | CACHED_TXN_DBI:db_open(create, db) 82 | local res, err = CACHED_TXN_DBI:commit() 83 | if not res then 84 | return nil, err 85 | end 86 | 87 | dbi = CACHED_TXN_DBI[1].result 88 | 89 | -- dbi already cached by commit() 90 | 91 | return dbi 92 | end 93 | get_dbi = _M.get_dbi 94 | end 95 | 96 | 97 | function _TXN_MT:reset() 98 | self.n = 0 99 | self.write = false 100 | end 101 | 102 | 103 | function _TXN_MT:_new_op() 104 | local n = self.n + 1 105 | self.n = n 106 | 107 | local op = self[n] 108 | if not op then 109 | op = table_new(0, 4) 110 | self[n] = op 111 | end 112 | 113 | return op 114 | end 115 | 116 | 117 | function _TXN_MT:get(key, db) 118 | local op = self:_new_op() 119 | op.opcode = "GET" 120 | op.key = normalize_key(key) 121 | op.db = db or DEFAULT_DB 122 | end 123 | 124 | 125 | function _TXN_MT:set(key, value, db) 126 | local op = self:_new_op() 127 | op.opcode = "SET" 128 | op.key = normalize_key(key) 129 | op.value = value 130 | op.db = db or DEFAULT_DB 131 | op.flags = 0 132 | 133 | self.write = true 134 | end 135 | 136 | 137 | function _TXN_MT:db_open(create, db) 138 | assert(type(create) == "boolean") 139 | 140 | local op = self:_new_op() 141 | op.opcode = "DB_OPEN" 142 | op.db = db or DEFAULT_DB 143 | op.flags = create and MDB_CREATE or 0 144 | 145 | self.write = true 146 | end 147 | 148 | 149 | function _TXN_MT:db_drop(delete, db) 150 | assert(type(delete) == "boolean") 151 | 152 | local op = self:_new_op() 153 | 154 | op.opcode = "DB_DROP" 155 | op.db = db or DEFAULT_DB 156 | op.flags = delete 157 | 158 | self.write = true 159 | end 160 | 161 | 162 | function _TXN_MT:commit() 163 | local value_buf_size = get_string_buf_size() 164 | local ops 165 | 166 | if self.ops_capacity >= self.n then 167 | ops = self.ops 168 | 169 | else 170 | self.ops_capacity = math_max(self.n, MIN_OPS_N) 171 | ops = ffi_new("ngx_lua_resty_lmdb_operation_t[?]", self.ops_capacity) 172 | self.ops = ops 173 | end 174 | 175 | for i = 1, self.n do 176 | local lop = self[i] 177 | local cop = ops[i - 1] 178 | 179 | if lop.opcode == "GET" then 180 | local dbi, err = get_dbi(false, lop.db) 181 | if err then 182 | return nil, "unable to open DB for GET '" .. lop.key .. "': " .. err 183 | 184 | elseif not dbi then 185 | return nil, "DB " .. lop.db .. " does not exist" 186 | end 187 | 188 | cop.opcode = C.NGX_LMDB_OP_GET 189 | cop.key.data = lop.key 190 | cop.key.len = #lop.key 191 | cop.dbi = dbi 192 | 193 | elseif lop.opcode == "SET" then 194 | local dbi, err = get_dbi(true, lop.db) 195 | if err then 196 | return nil, "unable to open DB for SET '" .. lop.key .. "': " .. err 197 | 198 | elseif not dbi then 199 | return nil, "DB " .. lop.db .. " does not exist" 200 | end 201 | 202 | cop.opcode = C.NGX_LMDB_OP_SET 203 | cop.key.data = lop.key 204 | cop.key.len = #lop.key 205 | local val = cop.value 206 | if lop.value == nil then 207 | val.data = nil 208 | val.len = 0 209 | 210 | else 211 | val.data = lop.value 212 | val.len = #lop.value 213 | end 214 | cop.dbi = dbi 215 | cop.flags = lop.flags 216 | 217 | elseif lop.opcode == "DB_OPEN" then 218 | cop.opcode = C.NGX_LMDB_OP_DB_OPEN 219 | cop.key.data = lop.db 220 | cop.key.len = #lop.db 221 | cop.flags = lop.flags 222 | 223 | elseif lop.opcode == "DB_DROP" then 224 | local dbi, err = get_dbi(false, lop.db) 225 | if err then 226 | return nil, "unable to open DB for DROP: " .. err 227 | 228 | elseif not dbi then 229 | return nil, "DB " .. lop.db .. " does not exist" 230 | end 231 | 232 | cop.opcode = C.NGX_LMDB_OP_DB_DROP 233 | cop.flags = lop.flags 234 | cop.dbi = dbi 235 | 236 | else 237 | assert(false) 238 | end 239 | end 240 | 241 | ::again:: 242 | local buf = get_string_buf(value_buf_size, false) 243 | local ret = C.ngx_lua_resty_lmdb_ffi_execute(ops, self.n, self.write, 244 | buf, value_buf_size, err_ptr) 245 | if ret == NGX_ERROR then 246 | return nil, ffi_string(err_ptr[0]) 247 | end 248 | 249 | if ret == NGX_AGAIN then 250 | value_buf_size = value_buf_size * 2 251 | goto again 252 | end 253 | 254 | assert(ret == NGX_OK) 255 | 256 | for i = 1, self.n do 257 | local cop = ops[i - 1] 258 | local lop = self[i] 259 | 260 | if cop.opcode == C.NGX_LMDB_OP_GET then 261 | if cop.value.data ~= nil then 262 | lop.result = ffi_string(cop.value.data, cop.value.len) 263 | 264 | else 265 | lop.result = nil 266 | end 267 | 268 | elseif cop.opcode == C.NGX_LMDB_OP_SET then 269 | -- Set does not return flags 270 | lop.result = true 271 | 272 | elseif cop.opcode == C.NGX_LMDB_OP_DB_OPEN then 273 | -- cache the DBi 274 | lop.result = cop.dbi 275 | CACHED_DBI[lop.db] = cop.dbi 276 | 277 | elseif cop.opcode == C.NGX_LMDB_OP_DB_DROP then 278 | -- remove cached DBi 279 | lop.result = true 280 | 281 | if lop.flags then 282 | -- delete == true 283 | CACHED_DBI[lop.db] = nil 284 | end 285 | end 286 | end 287 | 288 | return true 289 | end 290 | 291 | 292 | return _M 293 | -------------------------------------------------------------------------------- /src/ngx_lua_resty_lmdb_module.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | #define NGX_LUA_RESTY_LMDB_FILE_MODE 0600 5 | #define NGX_LUA_RESTY_LMDB_DIR_MODE 0700 6 | 7 | /* 126 is the default number of max readers in LMDB */ 8 | #define NGX_LUA_RESTY_LMDB_DEFAULT_READERS 126 9 | #define NGX_LUA_RESTY_LMDB_MAX_READERS_REDUNDANCY 16 10 | 11 | #define NGX_LUA_RESTY_LMDB_VALIDATION_KEY "validation_tag" 12 | 13 | 14 | static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { 15 | ngx_string("/data.mdb"), 16 | ngx_string("/lock.mdb"), 17 | ngx_null_string, 18 | }; 19 | 20 | 21 | static void *ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle); 22 | static char *ngx_lua_resty_lmdb_init_conf(ngx_cycle_t *cycle, void *conf); 23 | static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle); 24 | static ngx_int_t ngx_lua_resty_lmdb_init_worker(ngx_cycle_t *cycle); 25 | static void ngx_lua_resty_lmdb_exit_worker(ngx_cycle_t *cycle); 26 | 27 | 28 | static ngx_command_t ngx_lua_resty_lmdb_commands[] = { 29 | 30 | { ngx_string("lmdb_environment_path"), 31 | NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, 32 | ngx_conf_set_path_slot, 33 | 0, 34 | offsetof(ngx_lua_resty_lmdb_conf_t, env_path), 35 | NULL }, 36 | 37 | { ngx_string("lmdb_max_databases"), 38 | NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, 39 | ngx_conf_set_size_slot, 40 | 0, 41 | offsetof(ngx_lua_resty_lmdb_conf_t, max_databases), 42 | NULL }, 43 | 44 | { ngx_string("lmdb_map_size"), 45 | NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, 46 | ngx_conf_set_size_slot, 47 | 0, 48 | offsetof(ngx_lua_resty_lmdb_conf_t, map_size), 49 | NULL }, 50 | 51 | { ngx_string("lmdb_validation_tag"), 52 | NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, 53 | ngx_conf_set_str_slot, 54 | 0, 55 | offsetof(ngx_lua_resty_lmdb_conf_t, validation_tag), 56 | NULL }, 57 | 58 | ngx_null_command 59 | }; 60 | 61 | 62 | static ngx_core_module_t ngx_lua_resty_lmdb_module_ctx = { 63 | ngx_string("lua_resty_lmdb"), 64 | ngx_lua_resty_lmdb_create_conf, 65 | ngx_lua_resty_lmdb_init_conf 66 | }; 67 | 68 | 69 | ngx_module_t ngx_lua_resty_lmdb_module = { 70 | NGX_MODULE_V1, 71 | &ngx_lua_resty_lmdb_module_ctx, /* module context */ 72 | ngx_lua_resty_lmdb_commands, /* module directives */ 73 | NGX_CORE_MODULE, /* module type */ 74 | NULL, /* init master */ 75 | ngx_lua_resty_lmdb_init, /* init module */ 76 | ngx_lua_resty_lmdb_init_worker, /* init process */ 77 | NULL, /* init thread */ 78 | NULL, /* exit thread */ 79 | ngx_lua_resty_lmdb_exit_worker, /* exit process */ 80 | NULL, /* exit master */ 81 | NGX_MODULE_V1_PADDING 82 | }; 83 | 84 | 85 | static void * 86 | ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle) 87 | { 88 | ngx_lua_resty_lmdb_conf_t *lcf; 89 | 90 | lcf = ngx_pcalloc(cycle->pool, sizeof(ngx_lua_resty_lmdb_conf_t)); 91 | if (lcf == NULL) { 92 | return NULL; 93 | } 94 | 95 | /* 96 | * set by ngx_pcalloc(): 97 | * 98 | * conf->env_path = NULL; 99 | * conf->env = NULL; 100 | */ 101 | 102 | lcf->max_databases = NGX_CONF_UNSET_SIZE; 103 | lcf->map_size = NGX_CONF_UNSET_SIZE; 104 | 105 | return lcf; 106 | } 107 | 108 | 109 | static char * 110 | ngx_lua_resty_lmdb_init_conf(ngx_cycle_t *cycle, void *conf) 111 | { 112 | ngx_lua_resty_lmdb_conf_t *lcf = conf; 113 | 114 | ngx_conf_init_size_value(lcf->max_databases, 1); 115 | 116 | /* same as mdb.c DEFAULT_MAPSIZE */ 117 | ngx_conf_init_size_value(lcf->map_size, 1048576); 118 | 119 | return NGX_CONF_OK; 120 | } 121 | 122 | 123 | static ngx_int_t 124 | ngx_lua_resty_lmdb_create_env(ngx_cycle_t *cycle, 125 | ngx_lua_resty_lmdb_conf_t *lcf, 126 | ngx_flag_t is_master) 127 | { 128 | int rc; 129 | 130 | rc = mdb_env_create(&lcf->env); 131 | if (rc != 0) { 132 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 133 | "unable to create LMDB environment: %s", 134 | mdb_strerror(rc)); 135 | return NGX_ERROR; 136 | } 137 | 138 | rc = mdb_env_set_mapsize(lcf->env, lcf->map_size); 139 | if (rc != 0) { 140 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 141 | "unable to set map size for LMDB: %s", 142 | mdb_strerror(rc)); 143 | goto failed; 144 | } 145 | 146 | rc = mdb_env_set_maxdbs(lcf->env, lcf->max_databases); 147 | if (rc != 0) { 148 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 149 | "unable to set maximum DB count for LMDB: %s", 150 | mdb_strerror(rc)); 151 | goto failed; 152 | } 153 | 154 | return NGX_OK; 155 | 156 | failed: 157 | 158 | mdb_env_close(lcf->env); 159 | lcf->env = NULL; 160 | 161 | return NGX_ERROR; 162 | } 163 | 164 | 165 | static ngx_int_t 166 | ngx_lua_resty_lmdb_remove_files(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_conf_t *lcf) 167 | { 168 | ngx_file_info_t fi; 169 | 170 | u_char name_buf[NGX_MAX_PATH]; 171 | ngx_str_t *names = ngx_lua_resty_lmdb_file_names; 172 | ngx_str_t *name; 173 | ngx_path_t *path = lcf->env_path; 174 | 175 | if (ngx_file_info(path->name.data, &fi) == NGX_FILE_ERROR) { 176 | ngx_log_error(NGX_LOG_CRIT, cycle->log, ngx_errno, 177 | ngx_file_info_n " \"%s\" failed", path->name.data); 178 | return NGX_ERROR; 179 | } 180 | 181 | if (ngx_is_dir(&fi)) { 182 | 183 | /* try to remove all lmdb files */ 184 | for (name = names; name->len; name++) { 185 | ngx_snprintf(name_buf, NGX_MAX_PATH, 186 | "%V%V%Z", &path->name, name); 187 | 188 | ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, 189 | "lmdb file remove: \"%s\"", name_buf); 190 | 191 | if (ngx_delete_file(name_buf) == NGX_FILE_ERROR) { 192 | ngx_log_error(NGX_LOG_WARN, cycle->log, ngx_errno, 193 | ngx_delete_file_n " \"%s\" failed", 194 | name_buf); 195 | } 196 | } 197 | 198 | return NGX_OK; 199 | } 200 | 201 | ngx_lua_resty_lmdb_assert(!ngx_is_dir(&fi)); 202 | 203 | /* try to delete the file */ 204 | if (ngx_delete_file(path->name.data) == NGX_FILE_ERROR) { 205 | ngx_log_error(NGX_LOG_WARN, cycle->log, ngx_errno, 206 | ngx_delete_file_n " \"%V\" failed", &path->name); 207 | } 208 | 209 | /* ensure lmdb directory exists */ 210 | if (ngx_create_dir( 211 | path->name.data, NGX_LUA_RESTY_LMDB_DIR_MODE) == NGX_FILE_ERROR) { 212 | 213 | ngx_log_error(NGX_LOG_WARN, cycle->log, ngx_errno, 214 | ngx_create_dir_n " \"%V\" failed", &path->name); 215 | } 216 | 217 | return NGX_OK; 218 | } 219 | 220 | 221 | static ngx_int_t 222 | ngx_lua_resty_lmdb_open_file(ngx_cycle_t *cycle, 223 | ngx_lua_resty_lmdb_conf_t *lcf, 224 | ngx_flag_t is_master) 225 | { 226 | int rc; 227 | int dead; 228 | size_t readers; 229 | ngx_core_conf_t *ccf; 230 | 231 | if (ngx_lua_resty_lmdb_create_env(cycle, lcf, is_master) != NGX_OK) { 232 | return NGX_ERROR; 233 | } 234 | 235 | /* Set max readers depending on the number of worker processes */ 236 | ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); 237 | readers = (size_t) ccf->worker_processes + NGX_LUA_RESTY_LMDB_MAX_READERS_REDUNDANCY; 238 | 239 | if (readers < NGX_LUA_RESTY_LMDB_DEFAULT_READERS) { 240 | readers = NGX_LUA_RESTY_LMDB_DEFAULT_READERS; 241 | } 242 | 243 | rc = mdb_env_set_maxreaders(lcf->env, readers); 244 | if (rc != 0) { 245 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 246 | "unable to set max readers for LMDB environment: %s", 247 | mdb_strerror(rc)); 248 | mdb_env_close(lcf->env); 249 | lcf->env = NULL; 250 | 251 | return NGX_ERROR; 252 | } 253 | 254 | rc = mdb_env_open(lcf->env, (const char *) lcf->env_path->name.data, 255 | 0, NGX_LUA_RESTY_LMDB_FILE_MODE); 256 | 257 | /* 258 | * may be MDB_VERSION_MISMATCH or MDB_INVALID 259 | * try to remove the invalid LMDB files and open it again 260 | */ 261 | 262 | if (is_master == 1 && 263 | (rc == ENOTDIR || rc == MDB_VERSION_MISMATCH || rc == MDB_INVALID)) { 264 | 265 | mdb_env_close(lcf->env); 266 | lcf->env = NULL; 267 | 268 | ngx_log_error(NGX_LOG_WARN, cycle->log, 0, 269 | "LMDB database is corrupted or incompatible, removing"); 270 | 271 | if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { 272 | return NGX_ERROR; 273 | } 274 | 275 | if (ngx_lua_resty_lmdb_create_env(cycle, lcf, is_master) != NGX_OK) { 276 | return NGX_ERROR; 277 | } 278 | 279 | rc = mdb_env_open(lcf->env, (const char *) lcf->env_path->name.data, 280 | 0, NGX_LUA_RESTY_LMDB_FILE_MODE); 281 | } 282 | 283 | if (rc != 0) { 284 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 285 | "unable to open LMDB environment: %s", mdb_strerror(rc)); 286 | 287 | mdb_env_close(lcf->env); 288 | lcf->env = NULL; 289 | 290 | return NGX_ERROR; 291 | } 292 | 293 | rc = mdb_reader_check(lcf->env, &dead); 294 | if (rc != 0) { 295 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 296 | "unable to check LMDB reader slots: %s", mdb_strerror(rc)); 297 | /* this is not a fatal error */ 298 | 299 | } else if (dead > 0) { 300 | ngx_log_error(NGX_LOG_WARN, cycle->log, 0, 301 | "found and cleared %d stale readers from LMDB", dead); 302 | } 303 | 304 | rc = mdb_txn_begin(lcf->env, NULL, MDB_RDONLY, &lcf->ro_txn); 305 | if (rc != 0) { 306 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 307 | "unable to open LMDB read only transaction: %s", 308 | mdb_strerror(rc)); 309 | 310 | mdb_env_close(lcf->env); 311 | lcf->env = NULL; 312 | 313 | return NGX_ERROR; 314 | } 315 | 316 | mdb_txn_reset(lcf->ro_txn); 317 | 318 | return NGX_OK; 319 | } 320 | 321 | 322 | static ngx_int_t 323 | ngx_lua_resty_lmdb_close_file(ngx_cycle_t *cycle, 324 | ngx_lua_resty_lmdb_conf_t *lcf) 325 | { 326 | mdb_txn_abort(lcf->ro_txn); 327 | mdb_env_close(lcf->env); 328 | 329 | lcf->ro_txn = NULL; 330 | lcf->env = NULL; 331 | 332 | return NGX_OK; 333 | } 334 | 335 | 336 | static ngx_int_t 337 | ngx_lua_resty_lmdb_verify_file_status(ngx_cycle_t *cycle, 338 | ngx_lua_resty_lmdb_conf_t *lcf) 339 | { 340 | ngx_core_conf_t *ccf; 341 | ngx_file_info_t fi; 342 | 343 | u_char name_buf[NGX_MAX_PATH]; 344 | ngx_str_t *names = ngx_lua_resty_lmdb_file_names; 345 | ngx_str_t *name; 346 | 347 | ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); 348 | 349 | if (ccf->user == (ngx_uid_t) NGX_CONF_UNSET_UINT) { 350 | return NGX_OK; 351 | } 352 | 353 | /* check directory */ 354 | 355 | ngx_snprintf(name_buf, NGX_MAX_PATH, 356 | "%V%Z", &lcf->env_path->name); 357 | 358 | if (ngx_file_info(name_buf, &fi) == NGX_FILE_ERROR) { 359 | ngx_log_error(NGX_LOG_CRIT, cycle->log, ngx_errno, 360 | ngx_file_info_n " \"%s\" failed", name_buf); 361 | return NGX_ERROR; 362 | } 363 | 364 | if (fi.st_uid != ccf->user) { 365 | if (chown((const char *) name_buf, ccf->user, -1) == -1) { 366 | ngx_log_error(NGX_LOG_WARN, cycle->log, ngx_errno, 367 | "chown(\"%s\", %d) failed, " 368 | "LMDB files/directory is not owned by the current Nginx user, " 369 | "this may cause permission issues or security risks later", 370 | name_buf, ccf->user); 371 | } 372 | } 373 | 374 | /* check files */ 375 | 376 | for (name = names; name->len; name++) { 377 | ngx_snprintf(name_buf, NGX_MAX_PATH, 378 | "%V%V%Z", &lcf->env_path->name, name); 379 | 380 | if (ngx_file_info(name_buf, &fi) == NGX_FILE_ERROR) { 381 | ngx_log_error(NGX_LOG_CRIT, cycle->log, ngx_errno, 382 | ngx_file_info_n " \"%s\" failed", name_buf); 383 | return NGX_ERROR; 384 | } 385 | 386 | if (fi.st_uid != ccf->user) { 387 | if (chown((const char *) name_buf, ccf->user, -1) == -1) { 388 | ngx_log_error(NGX_LOG_WARN, cycle->log, ngx_errno, 389 | "chown(\"%s\", %d) failed, " 390 | "LMDB files/directory is not owned by the current Nginx user, " 391 | "this may cause permission issues or security risks later", 392 | name_buf, ccf->user); 393 | } 394 | } 395 | } 396 | 397 | return NGX_OK; 398 | } 399 | 400 | 401 | static ngx_int_t 402 | ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, 403 | ngx_lua_resty_lmdb_conf_t *lcf) 404 | { 405 | int rc; 406 | MDB_dbi dbi; 407 | MDB_val key; 408 | MDB_val value; 409 | MDB_txn *txn = NULL; 410 | 411 | /* check tag value in lmdb */ 412 | 413 | if (lcf->validation_tag.data == NULL) { 414 | return NGX_OK; 415 | } 416 | 417 | ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, 418 | "LMDB validation enabled, using validation tag: \"%V\"", 419 | &lcf->validation_tag); 420 | 421 | ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); 422 | 423 | rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); 424 | if (rc != 0) { 425 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 426 | "unable to open LMDB transaction: %s", 427 | mdb_strerror(rc)); 428 | return NGX_ERROR; 429 | } 430 | 431 | ngx_lua_resty_lmdb_assert(txn); 432 | 433 | rc = mdb_dbi_open(txn, NULL, MDB_CREATE, &dbi); 434 | if (rc != 0) { 435 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 436 | "unable to open LMDB database: %s", 437 | mdb_strerror(rc)); 438 | goto failed; 439 | } 440 | 441 | key.mv_size = sizeof(NGX_LUA_RESTY_LMDB_VALIDATION_KEY) - 1; 442 | key.mv_data = NGX_LUA_RESTY_LMDB_VALIDATION_KEY; 443 | 444 | rc = mdb_get(txn, dbi, &key, &value); 445 | if (rc == 0) { 446 | /* key found, compare with validation_tag value */ 447 | if (lcf->validation_tag.len == value.mv_size && 448 | ngx_strncmp(lcf->validation_tag.data, 449 | value.mv_data, value.mv_size) == 0) { 450 | 451 | mdb_txn_abort(txn); 452 | return NGX_OK; 453 | } 454 | 455 | ngx_log_error(NGX_LOG_WARN, cycle->log, 0, 456 | "LMDB validation tag \"%*s\" did not match configured tag \"%V\"", 457 | value.mv_size, value.mv_data, 458 | &lcf->validation_tag); 459 | 460 | } else if (rc == MDB_NOTFOUND) { 461 | ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, 462 | "LMDB validation tag does not exist, assuming empty database"); 463 | 464 | mdb_txn_abort(txn); 465 | return NGX_DECLINED; 466 | 467 | } else { 468 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 469 | "unable to get LMDB validation tag: %s", 470 | mdb_strerror(rc)); 471 | } 472 | 473 | failed: 474 | 475 | mdb_txn_abort(txn); 476 | return NGX_ERROR; 477 | } 478 | 479 | 480 | static ngx_int_t 481 | ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, 482 | ngx_lua_resty_lmdb_conf_t *lcf) 483 | { 484 | int rc; 485 | MDB_dbi dbi; 486 | MDB_val key; 487 | MDB_val value; 488 | MDB_txn *txn = NULL; 489 | 490 | ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); 491 | 492 | rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); 493 | if (rc != 0) { 494 | ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, 495 | "unable to open LMDB transaction: %s", 496 | mdb_strerror(rc)); 497 | return NGX_ERROR; 498 | } 499 | 500 | ngx_lua_resty_lmdb_assert(txn); 501 | 502 | rc = mdb_dbi_open(txn, NULL, MDB_CREATE, &dbi); 503 | if (rc != 0) { 504 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 505 | "unable to open LMDB database: %s", 506 | mdb_strerror(rc)); 507 | goto failed; 508 | } 509 | 510 | /* set tag value to lmdb db */ 511 | 512 | key.mv_size = sizeof(NGX_LUA_RESTY_LMDB_VALIDATION_KEY) - 1; 513 | key.mv_data = NGX_LUA_RESTY_LMDB_VALIDATION_KEY; 514 | 515 | value.mv_size = lcf->validation_tag.len; 516 | value.mv_data = lcf->validation_tag.data; 517 | 518 | rc = mdb_put(txn, dbi, &key, &value, 0); 519 | if (rc != 0) { 520 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 521 | "unable to set LMDB validation tag: %s", 522 | mdb_strerror(rc)); 523 | goto failed; 524 | } 525 | 526 | rc = mdb_txn_commit(txn); 527 | if (rc != 0) { 528 | ngx_log_error(NGX_LOG_ERR, cycle->log, 0, 529 | "unable to commit validation tag into LMDB: %s", 530 | mdb_strerror(rc)); 531 | return NGX_ERROR; 532 | } 533 | 534 | ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, 535 | "set LMDB validation tag: \"%V\"", 536 | &lcf->validation_tag); 537 | 538 | return NGX_OK; 539 | 540 | failed: 541 | 542 | mdb_txn_abort(txn); 543 | return NGX_ERROR; 544 | } 545 | 546 | 547 | static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) 548 | { 549 | ngx_int_t rc; 550 | ngx_lua_resty_lmdb_conf_t *lcf; 551 | 552 | lcf = (ngx_lua_resty_lmdb_conf_t *) ngx_get_conf(cycle->conf_ctx, 553 | ngx_lua_resty_lmdb_module); 554 | 555 | if (lcf == NULL || lcf->env_path == NULL) { 556 | return NGX_OK; 557 | } 558 | 559 | /* ensure lmdb file is ok */ 560 | 561 | if (ngx_lua_resty_lmdb_open_file(cycle, lcf, 1) != NGX_OK) { 562 | return NGX_ERROR; 563 | } 564 | 565 | /* check lmdb validation tag */ 566 | 567 | rc = ngx_lua_resty_lmdb_validate(cycle, lcf); 568 | 569 | if (rc != NGX_OK) { 570 | ngx_lua_resty_lmdb_close_file(cycle, lcf); 571 | 572 | ngx_log_error((rc == NGX_DECLINED ? NGX_LOG_NOTICE : NGX_LOG_WARN), 573 | cycle->log, 0, 574 | "LMDB validation tag mismatch, wiping the database"); 575 | 576 | /* remove lmdb files to clean data */ 577 | if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { 578 | return NGX_ERROR; 579 | } 580 | 581 | /* open lmdb file again */ 582 | if (ngx_lua_resty_lmdb_open_file(cycle, lcf, 1) != NGX_OK) { 583 | return NGX_ERROR; 584 | } 585 | 586 | /* write tag into lmdb */ 587 | if (ngx_lua_resty_lmdb_write_tag(cycle, lcf) != NGX_OK) { 588 | ngx_lua_resty_lmdb_close_file(cycle, lcf); 589 | return NGX_ERROR; 590 | } 591 | } 592 | 593 | if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { 594 | return NGX_ERROR; 595 | } 596 | 597 | /* change to proper permission */ 598 | 599 | if (ngx_lua_resty_lmdb_verify_file_status(cycle, lcf) != NGX_OK) { 600 | return NGX_ERROR; 601 | } 602 | 603 | return NGX_OK; 604 | } 605 | 606 | 607 | static ngx_int_t ngx_lua_resty_lmdb_init_worker(ngx_cycle_t *cycle) 608 | { 609 | ngx_lua_resty_lmdb_conf_t *lcf; 610 | 611 | lcf = (ngx_lua_resty_lmdb_conf_t *) ngx_get_conf(cycle->conf_ctx, 612 | ngx_lua_resty_lmdb_module); 613 | 614 | if (lcf == NULL || lcf->env_path == NULL) { 615 | return NGX_OK; 616 | } 617 | 618 | if (ngx_lua_resty_lmdb_open_file(cycle, lcf, 0) != NGX_OK) { 619 | return NGX_ERROR; 620 | } 621 | 622 | return NGX_OK; 623 | } 624 | 625 | 626 | static void ngx_lua_resty_lmdb_exit_worker(ngx_cycle_t *cycle) 627 | { 628 | ngx_lua_resty_lmdb_conf_t *lcf; 629 | 630 | lcf = (ngx_lua_resty_lmdb_conf_t *) ngx_get_conf(cycle->conf_ctx, 631 | ngx_lua_resty_lmdb_module); 632 | 633 | if (lcf == NULL || lcf->env_path == NULL) { 634 | return; 635 | } 636 | 637 | if (lcf->env != NULL) { 638 | ngx_lua_resty_lmdb_close_file(cycle, lcf); 639 | } 640 | } 641 | 642 | 643 | -------------------------------------------------------------------------------- /src/ngx_lua_resty_lmdb_module.h: -------------------------------------------------------------------------------- 1 | #ifndef _NGX_LUA_RESTY_LMDB_MODULE_H_INCLUDED_ 2 | #define _NGX_LUA_RESTY_LMDB_MODULE_H_INCLUDED_ 3 | 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | struct ngx_lua_resty_lmdb_conf_s { 11 | ngx_path_t *env_path; 12 | size_t max_databases; 13 | size_t map_size; 14 | MDB_env *env; 15 | MDB_txn *ro_txn; 16 | 17 | ngx_str_t validation_tag; 18 | }; 19 | 20 | 21 | typedef struct ngx_lua_resty_lmdb_conf_s ngx_lua_resty_lmdb_conf_t; 22 | 23 | 24 | typedef enum { 25 | NGX_LMDB_OP_GET = 0, 26 | NGX_LMDB_OP_PREFIX, 27 | NGX_LMDB_OP_SET, 28 | NGX_LMDB_OP_DB_OPEN, 29 | NGX_LMDB_OP_DB_DROP 30 | } ngx_lua_resty_lmdb_operation_e; 31 | 32 | 33 | struct ngx_lua_resty_lmdb_operation_s { 34 | ngx_lua_resty_lmdb_operation_e opcode; 35 | ngx_str_t key; /* GET, SET */ 36 | ngx_str_t value; /* GET, SET */ 37 | MDB_dbi dbi; /* ALL OPS */ 38 | unsigned int flags; /* SET, DROP */ 39 | }; 40 | 41 | 42 | typedef struct ngx_lua_resty_lmdb_operation_s ngx_lua_resty_lmdb_operation_t; 43 | 44 | 45 | typedef struct { 46 | size_t map_size; /**< Size of the data memory map */ 47 | unsigned int page_size; /**< Size of a database page. */ 48 | unsigned int max_readers; /**< max reader slots in the environment */ 49 | unsigned int num_readers; /**< max reader slots used in the environment */ 50 | unsigned int allocated_pages; /**< number of pages allocated */ 51 | size_t in_use_pages; /**< number of pages used */ 52 | unsigned int entries; /**< the number of entries (key/value pairs) in the environment */ 53 | } ngx_lua_resty_lmdb_ffi_status_t; 54 | 55 | 56 | extern ngx_module_t ngx_lua_resty_lmdb_module; 57 | 58 | 59 | #ifdef NGX_LUA_USE_ASSERT 60 | #include 61 | # define ngx_lua_resty_lmdb_assert(a) assert(a) 62 | #else 63 | # define ngx_lua_resty_lmdb_assert(a) 64 | #endif 65 | 66 | 67 | #endif /* _NGX_LUA_RESTY_LMDB_MODULE_H_INCLUDED_ */ 68 | -------------------------------------------------------------------------------- /src/ngx_lua_resty_lmdb_prefix.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | /* 5 | * This function is the FFI call used for prefix lookups. 6 | * It is very similar to `ngx_lua_resty_lmdb_ffi_execute` inside 7 | * ngx_lua_resty_lmdb_transaction.c, 8 | * except we can only specify one key as the starting point at a time. 9 | * 10 | * The `ops[0]` will be the resume point and ops[1] will be 11 | * the desired prefix, this function returns all keys 12 | * >= `ops[0].key` with the same prefix as ops[1].key and up to `n` 13 | * will be returned at a time. They are usually not the same except 14 | * for the call to return the first page result. 15 | * 16 | * Returns: 17 | * * >= 0 - number of keys found. If return < `n`, then it is the last 18 | * key in the map (no more keys afterward the last result) 19 | * * `NGX_ERROR` - an error occurred, *err will contain the error string 20 | * * `NGX_AGAIN` - `buf_len` is not enough, try again with larger `buf` 21 | */ 22 | int ngx_lua_resty_lmdb_ffi_prefix(ngx_lua_resty_lmdb_operation_t *ops, 23 | size_t n, u_char *buf, size_t buf_len, const char **err) 24 | { 25 | ngx_lua_resty_lmdb_conf_t *lcf; 26 | size_t i; 27 | MDB_txn *txn; 28 | int rc; 29 | MDB_val key; 30 | MDB_val value; 31 | MDB_cursor *cur; 32 | ngx_lua_resty_lmdb_operation_t prefix; 33 | 34 | ngx_lua_resty_lmdb_assert(n >= 2); 35 | prefix = ops[1]; 36 | ngx_lua_resty_lmdb_assert(prefix.opcode == NGX_LMDB_OP_PREFIX); 37 | 38 | lcf = (ngx_lua_resty_lmdb_conf_t *) ngx_get_conf(ngx_cycle->conf_ctx, 39 | ngx_lua_resty_lmdb_module); 40 | 41 | if (lcf == NULL || lcf->env == NULL) { 42 | *err = "no LMDB environment defined"; 43 | return NGX_ERROR; 44 | } 45 | 46 | txn = lcf->ro_txn; 47 | rc = mdb_txn_renew(txn); 48 | if (rc != 0) { 49 | *err = mdb_strerror(rc); 50 | return NGX_ERROR; 51 | } 52 | 53 | rc = mdb_cursor_open(txn, ops[0].dbi, &cur); 54 | if (rc != 0) { 55 | *err = mdb_strerror(rc); 56 | mdb_txn_reset(txn); 57 | 58 | return NGX_ERROR; 59 | } 60 | 61 | /* we always have at least one ops slot as asserted above */ 62 | 63 | for (i = 0; i < n; i++) { 64 | key.mv_size = ops[i].key.len; 65 | key.mv_data = ops[i].key.data; 66 | 67 | rc = mdb_cursor_get(cur, &key, &value, i == 0 ? MDB_SET_RANGE : MDB_NEXT); 68 | if (rc == 0) { 69 | /* is this still a prefix of saved prefix? */ 70 | 71 | if (key.mv_size < prefix.key.len 72 | || ngx_memcmp(key.mv_data, prefix.key.data, prefix.key.len) != 0) 73 | { 74 | /* caller asked "123" but we got "13" */ 75 | mdb_cursor_close(cur); 76 | mdb_txn_reset(txn); 77 | 78 | return i; 79 | } 80 | 81 | /* key found, copy result into buf */ 82 | if (key.mv_size + value.mv_size > buf_len) { 83 | mdb_cursor_close(cur); 84 | mdb_txn_reset(txn); 85 | 86 | return NGX_AGAIN; 87 | } 88 | 89 | ops[i].key.data = buf; 90 | ops[i].key.len = key.mv_size; 91 | buf = ngx_cpymem(buf, key.mv_data, key.mv_size); 92 | 93 | ops[i].value.data = buf; 94 | ops[i].value.len = value.mv_size; 95 | buf = ngx_cpymem(buf, value.mv_data, value.mv_size); 96 | 97 | ops[i].opcode = NGX_LMDB_OP_PREFIX; 98 | 99 | buf_len -= key.mv_size + value.mv_size; 100 | 101 | } else if (rc == MDB_NOTFOUND) { 102 | mdb_cursor_close(cur); 103 | mdb_txn_reset(txn); 104 | 105 | return i; 106 | 107 | } else { 108 | *err = mdb_strerror(rc); 109 | 110 | mdb_cursor_close(cur); 111 | mdb_txn_reset(txn); 112 | 113 | return NGX_ERROR; 114 | } 115 | } 116 | 117 | mdb_cursor_close(cur); 118 | mdb_txn_reset(txn); 119 | 120 | return i; 121 | } 122 | -------------------------------------------------------------------------------- /src/ngx_lua_resty_lmdb_status.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | int ngx_lua_resty_lmdb_ffi_env_info(ngx_lua_resty_lmdb_ffi_status_t *lst, 5 | const char **err) 6 | { 7 | ngx_lua_resty_lmdb_conf_t *lcf; 8 | MDB_stat mst; 9 | MDB_envinfo mei; 10 | 11 | lcf = (ngx_lua_resty_lmdb_conf_t *) ngx_get_conf(ngx_cycle->conf_ctx, 12 | ngx_lua_resty_lmdb_module); 13 | 14 | if (lcf == NULL || lcf->env == NULL) { 15 | *err = "no LMDB environment defined"; 16 | return NGX_ERROR; 17 | } 18 | 19 | if (mdb_env_stat(lcf->env, &mst)) { 20 | *err = "mdb_env_stat() failed"; 21 | return NGX_ERROR; 22 | } 23 | 24 | if (mdb_env_info(lcf->env, &mei)) { 25 | *err = "mdb_env_info() failed"; 26 | return NGX_ERROR; 27 | } 28 | 29 | lst->map_size = mei.me_mapsize; 30 | lst->page_size = mst.ms_psize; 31 | lst->max_readers = mei.me_maxreaders; 32 | lst->num_readers = mei.me_numreaders; 33 | lst->in_use_pages = mst.ms_branch_pages + mst.ms_leaf_pages 34 | + mst.ms_overflow_pages; 35 | lst->allocated_pages = mei.me_last_pgno + 1; 36 | lst->entries = mst.ms_entries; 37 | 38 | return NGX_OK; 39 | } 40 | -------------------------------------------------------------------------------- /src/ngx_lua_resty_lmdb_transaction.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | /* 5 | * This function is the FFI call used for single key operations. 6 | * 7 | * Params: 8 | * * `ops` - operations array containing instructions to perform 9 | * * `n` - number of `ops` 10 | * * `need_write` - 1 if contains edit operations, 0 if not 11 | * * `buf` - results buf 12 | * * `buf_len` - size of buf 13 | * * `err` - err message output 14 | * 15 | * This function executes operations specified in `ops`. If the operation 16 | * requires passing value back to the caller, then it is written to the 17 | * `buf` buffer, up to `buf_len`. If the `buf` passed in is not large enough, 18 | * then `NGX_AGAIN` is returned and the caller is instructed to retry with a 19 | * larger buffer. 20 | * 21 | * Returns: 22 | * * `NGX_OK` - operations completed successfully 23 | * * `NGX_ERROR` - an error occurred, *err will contain the error string 24 | * * `NGX_AGAIN` - `buf_len` is not enough, try again with larger `buf` 25 | */ 26 | int ngx_lua_resty_lmdb_ffi_execute(ngx_lua_resty_lmdb_operation_t *ops, 27 | size_t n, int need_write, u_char *buf, size_t buf_len, const char **err) 28 | { 29 | ngx_lua_resty_lmdb_conf_t *lcf; 30 | size_t i; 31 | MDB_txn *txn; 32 | int rc; 33 | MDB_val key; 34 | MDB_val value; 35 | 36 | lcf = (ngx_lua_resty_lmdb_conf_t *) ngx_get_conf(ngx_cycle->conf_ctx, 37 | ngx_lua_resty_lmdb_module); 38 | 39 | if (lcf == NULL || lcf->env == NULL) { 40 | *err = "no LMDB environment defined"; 41 | return NGX_ERROR; 42 | } 43 | 44 | if (need_write) { 45 | rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); 46 | 47 | } else { 48 | txn = lcf->ro_txn; 49 | rc = mdb_txn_renew(txn); 50 | } 51 | if (rc != 0) { 52 | *err = mdb_strerror(rc); 53 | return NGX_ERROR; 54 | } 55 | 56 | for (i = 0; i < n; i++) { 57 | switch (ops[i].opcode) { 58 | case NGX_LMDB_OP_GET: 59 | key.mv_size = ops[i].key.len; 60 | key.mv_data = ops[i].key.data; 61 | 62 | rc = mdb_get(txn, ops[i].dbi, &key, &value); 63 | if (rc == 0) { 64 | /* key found, copy result into buf */ 65 | if (value.mv_size > buf_len) { 66 | if (need_write) { 67 | mdb_txn_abort(txn); 68 | 69 | } else { 70 | mdb_txn_reset(txn); 71 | } 72 | 73 | return NGX_AGAIN; 74 | } 75 | 76 | ops[i].value.data = buf; 77 | ops[i].value.len = value.mv_size; 78 | 79 | buf = ngx_cpymem(buf, value.mv_data, value.mv_size); 80 | buf_len -= value.mv_size; 81 | 82 | } else if (rc == MDB_NOTFOUND) { 83 | ngx_str_null(&ops[i].value); 84 | 85 | } else { 86 | *err = mdb_strerror(rc); 87 | goto err; 88 | } 89 | 90 | break; 91 | 92 | case NGX_LMDB_OP_SET: 93 | ngx_lua_resty_lmdb_assert(need_write); 94 | 95 | key.mv_size = ops[i].key.len; 96 | key.mv_data = ops[i].key.data; 97 | value.mv_size = ops[i].value.len; 98 | value.mv_data = ops[i].value.data; 99 | 100 | rc = value.mv_data != NULL 101 | ? mdb_put(txn, ops[i].dbi, &key, &value, ops[i].flags) 102 | : mdb_del(txn, ops[i].dbi, &key, NULL); 103 | if (rc != 0) { 104 | *err = mdb_strerror(rc); 105 | goto err; 106 | 107 | } 108 | 109 | break; 110 | 111 | case NGX_LMDB_OP_DB_DROP: 112 | ngx_lua_resty_lmdb_assert(need_write); 113 | 114 | rc = mdb_drop(txn, ops[i].dbi, ops[i].flags); 115 | if (rc != 0) { 116 | *err = mdb_strerror(rc); 117 | goto err; 118 | 119 | } 120 | 121 | break; 122 | 123 | case NGX_LMDB_OP_DB_OPEN: 124 | /* 125 | * write access is always required no matter 126 | * the database already exists or not because the dbi has to be 127 | * written into the shared map 128 | */ 129 | ngx_lua_resty_lmdb_assert(need_write); 130 | 131 | rc = mdb_dbi_open(txn, (const char *) ops[i].key.data, 132 | ops[i].flags, &ops[i].dbi); 133 | if (rc != 0) { 134 | *err = mdb_strerror(rc); 135 | goto err; 136 | 137 | } 138 | 139 | break; 140 | 141 | default: 142 | *err = "unknown opcode"; 143 | goto err; 144 | } 145 | } 146 | 147 | if (need_write) { 148 | rc = mdb_txn_commit(txn); 149 | if (rc != 0) { 150 | *err = mdb_strerror(rc); 151 | 152 | return NGX_ERROR; 153 | } 154 | 155 | } else { 156 | mdb_txn_reset(txn); 157 | } 158 | 159 | return NGX_OK; 160 | 161 | err: 162 | if (need_write) { 163 | mdb_txn_abort(txn); 164 | 165 | } else { 166 | mdb_txn_reset(txn); 167 | } 168 | 169 | return NGX_ERROR; 170 | } 171 | -------------------------------------------------------------------------------- /t/00-init_mdb.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path /tmp/test.mdb; 14 | lmdb_map_size 5m; 15 | }; 16 | 17 | our $HttpConfig = qq{ 18 | lua_package_path "$pwd/lib/?.lua;;"; 19 | }; 20 | 21 | no_long_string(); 22 | #no_diff(); 23 | 24 | run_tests(); 25 | 26 | __DATA__ 27 | 28 | === TEST 1: simple set() / get() 29 | --- http_config eval: $::HttpConfig 30 | --- main_config eval: $::MainConfig 31 | --- config 32 | location = /t { 33 | content_by_lua_block { 34 | local l = require("resty.lmdb") 35 | 36 | ngx.say(l.set("test", "value")) 37 | ngx.say(l.get("test")) 38 | ngx.say(l.get("test_not_exist")) 39 | } 40 | } 41 | --- request 42 | GET /t 43 | --- response_body 44 | true 45 | value 46 | nil 47 | --- no_error_log 48 | [error] 49 | [warn] 50 | [crit] 51 | -------------------------------------------------------------------------------- /t/01-sanity.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path /tmp/test.mdb; 14 | lmdb_map_size 5m; 15 | }; 16 | 17 | our $HttpConfig = qq{ 18 | lua_package_path "$pwd/lib/?.lua;;"; 19 | }; 20 | 21 | our $HttpConfigWithInit = qq{ 22 | lua_package_path "$pwd/lib/?.lua;;"; 23 | 24 | init_by_lua_block { 25 | local l = require("resty.lmdb") 26 | 27 | local res, err = l.set("test", "value") 28 | 29 | package.loaded.res = res 30 | package.loaded.err = err 31 | } 32 | }; 33 | 34 | no_long_string(); 35 | #no_diff(); 36 | 37 | run_tests(); 38 | 39 | __DATA__ 40 | 41 | === TEST 1: simple set() / get() 42 | --- http_config eval: $::HttpConfig 43 | --- main_config eval: $::MainConfig 44 | --- config 45 | location = /t { 46 | content_by_lua_block { 47 | local l = require("resty.lmdb") 48 | 49 | ngx.say(l.set("test", "value")) 50 | ngx.say(l.get("test")) 51 | ngx.say(l.get("test_not_exist")) 52 | } 53 | } 54 | --- request 55 | GET /t 56 | --- response_body 57 | true 58 | value 59 | nil 60 | --- no_error_log 61 | [error] 62 | [warn] 63 | [crit] 64 | 65 | 66 | 67 | === TEST 2: clear using set() 68 | --- http_config eval: $::HttpConfig 69 | --- main_config eval: $::MainConfig 70 | --- config 71 | location = /t { 72 | content_by_lua_block { 73 | local l = require("resty.lmdb") 74 | 75 | ngx.say(l.set("test", "value")) 76 | ngx.say(l.set("test", nil)) 77 | ngx.say(l.get("test")) 78 | } 79 | } 80 | --- request 81 | GET /t 82 | --- response_body 83 | true 84 | true 85 | nil 86 | --- no_error_log 87 | [error] 88 | [warn] 89 | [crit] 90 | 91 | 92 | 93 | === TEST 3: db_drop() 94 | --- http_config eval: $::HttpConfig 95 | --- main_config eval: $::MainConfig 96 | --- config 97 | location = /t { 98 | content_by_lua_block { 99 | local l = require("resty.lmdb") 100 | 101 | ngx.say(l.set("test", "value")) 102 | ngx.say(l.db_drop()) 103 | ngx.say(l.get("test")) 104 | } 105 | } 106 | --- request 107 | GET /t 108 | --- response_body 109 | true 110 | true 111 | nil 112 | --- no_error_log 113 | [error] 114 | [warn] 115 | [crit] 116 | 117 | 118 | 119 | === TEST 4: db_drop(delete = true) 120 | --- http_config eval: $::HttpConfig 121 | --- main_config eval: $::MainConfig 122 | --- config 123 | location = /t { 124 | content_by_lua_block { 125 | local l = require("resty.lmdb") 126 | 127 | ngx.say(l.set("test", "value")) 128 | ngx.say(l.db_drop(true)) 129 | ngx.say(l.db_drop(true)) 130 | ngx.say(l.get("test")) 131 | } 132 | } 133 | --- request 134 | GET /t 135 | --- response_body 136 | true 137 | true 138 | nilunable to open DB for DROP: MDB_NOTFOUND: No matching key/data pair found 139 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 140 | --- no_error_log 141 | [error] 142 | [warn] 143 | [crit] 144 | 145 | 146 | 147 | === TEST 5: works fine when not enabled 148 | --- http_config eval: $::HttpConfig 149 | --- config 150 | location = /t { 151 | content_by_lua_block { 152 | ngx.say("good") 153 | } 154 | } 155 | --- request 156 | GET /t 157 | --- response_body 158 | good 159 | --- no_error_log 160 | [error] 161 | [warn] 162 | [crit] 163 | 164 | 165 | 166 | === TEST 6: does not crash when called in init_by_lua 167 | --- http_config eval: $::HttpConfigWithInit 168 | --- main_config eval: $::MainConfig 169 | --- config 170 | location /t { 171 | content_by_lua_block { 172 | ngx.say(package.loaded.res) 173 | ngx.say(package.loaded.err) 174 | } 175 | } 176 | --- request 177 | GET /t 178 | --- response_body 179 | nil 180 | unable to open DB for SET 'test': no LMDB environment defined 181 | --- no_error_log 182 | [error] 183 | [warn] 184 | [crit] 185 | -------------------------------------------------------------------------------- /t/02-multiple_dbs.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path /tmp/test.mdb; 14 | lmdb_max_databases 3; 15 | }; 16 | 17 | our $HttpConfig = qq{ 18 | lua_package_path "$pwd/lib/?.lua;;"; 19 | }; 20 | 21 | no_long_string(); 22 | #no_diff(); 23 | 24 | run_tests(); 25 | 26 | __DATA__ 27 | 28 | === TEST 1: simple set() / get() 29 | --- http_config eval: $::HttpConfig 30 | --- main_config eval: $::MainConfig 31 | --- config 32 | location = /t { 33 | content_by_lua_block { 34 | local l = require("resty.lmdb") 35 | 36 | ngx.say(l.set("test", "value", "custom_db")) 37 | ngx.say(l.get("test", "custom_db")) 38 | ngx.say(l.set("test", "value")) -- main DB 39 | ngx.say(l.get("not_exist")) 40 | ngx.say(l.get("test", "not_exist")) 41 | ngx.say(l.get("test", "not_exist1")) 42 | ngx.say(l.get("test", "not_exist2")) 43 | } 44 | } 45 | --- request 46 | GET /t 47 | --- response_body 48 | true 49 | value 50 | true 51 | nil 52 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 53 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 54 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 55 | --- no_error_log 56 | [error] 57 | [warn] 58 | [crit] 59 | 60 | 61 | 62 | === TEST 2: clear using set() 63 | --- http_config eval: $::HttpConfig 64 | --- main_config eval: $::MainConfig 65 | --- config 66 | location = /t { 67 | content_by_lua_block { 68 | local l = require("resty.lmdb") 69 | 70 | ngx.say(l.set("test", "value")) 71 | ngx.say(l.set("test", "value", "custom_db")) 72 | ngx.say(l.set("test", nil, "custom_db")) 73 | ngx.say(l.get("test", "custom_db")) 74 | ngx.say(l.get("test")) 75 | } 76 | } 77 | --- request 78 | GET /t 79 | --- response_body 80 | true 81 | true 82 | true 83 | nil 84 | value 85 | --- no_error_log 86 | [error] 87 | [warn] 88 | [crit] 89 | 90 | 91 | 92 | === TEST 3: db_drop() 93 | --- http_config eval: $::HttpConfig 94 | --- main_config eval: $::MainConfig 95 | --- config 96 | location = /t { 97 | content_by_lua_block { 98 | local l = require("resty.lmdb") 99 | 100 | ngx.say(l.set("test", "value")) 101 | ngx.say(l.set("test", "value", "custom_db")) 102 | ngx.say(l.db_drop(false, "custom_db")) 103 | ngx.say(l.get("test", "custom_db")) 104 | ngx.say(l.get("test")) 105 | } 106 | } 107 | --- request 108 | GET /t 109 | --- response_body 110 | true 111 | true 112 | true 113 | nil 114 | value 115 | --- no_error_log 116 | [error] 117 | [warn] 118 | [crit] 119 | 120 | 121 | 122 | === TEST 4: db_drop(delete = true) 123 | --- http_config eval: $::HttpConfig 124 | --- main_config eval: $::MainConfig 125 | --- config 126 | location = /t { 127 | content_by_lua_block { 128 | local l = require("resty.lmdb") 129 | 130 | ngx.say(l.set("test", "value")) 131 | ngx.say(l.set("test", "value", "custom_db")) 132 | ngx.say(l.db_drop(true, "custom_db")) 133 | ngx.say(l.get("test", "custom_db")) 134 | ngx.say(l.get("test")) 135 | } 136 | } 137 | --- request 138 | GET /t 139 | --- response_body 140 | true 141 | true 142 | true 143 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 144 | value 145 | --- no_error_log 146 | [error] 147 | [warn] 148 | [crit] 149 | -------------------------------------------------------------------------------- /t/03-transactions.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path /tmp/test.mdb; 14 | lmdb_max_databases 3; 15 | }; 16 | 17 | our $HttpConfig = qq{ 18 | lua_package_path "$pwd/lib/?.lua;;"; 19 | }; 20 | 21 | no_long_string(); 22 | #no_diff(); 23 | 24 | run_tests(); 25 | 26 | __DATA__ 27 | 28 | === TEST 1: simple set() / get() 29 | --- http_config eval: $::HttpConfig 30 | --- main_config eval: $::MainConfig 31 | --- config 32 | location = /t { 33 | content_by_lua_block { 34 | local txn = require("resty.lmdb.transaction") 35 | local l = require("resty.lmdb") 36 | 37 | l.set("testbalabala", "aaaa") 38 | ngx.say(l.db_drop(false)) 39 | 40 | local t = txn.begin() 41 | t:get("not_found") 42 | t:set("test", "value") 43 | t:get("test") 44 | t:set("test", nil) 45 | t:get("test") 46 | assert(t.n == 5) 47 | assert(t.write == true) 48 | ngx.say(t:commit()) 49 | assert(t.ops_capacity == 16) 50 | 51 | for i = 1, 5 do 52 | ngx.say(i, ": ", t[i].result) 53 | end 54 | 55 | ngx.say(l.get("test")) 56 | } 57 | } 58 | --- request 59 | GET /t 60 | --- response_body 61 | true 62 | true 63 | 1: nil 64 | 2: true 65 | 3: value 66 | 4: true 67 | 5: nil 68 | nil 69 | --- no_error_log 70 | [error] 71 | [warn] 72 | [crit] 73 | 74 | 75 | 76 | === TEST 2: transaction isolation 77 | --- http_config eval: $::HttpConfig 78 | --- main_config eval: $::MainConfig 79 | --- config 80 | location = /t { 81 | content_by_lua_block { 82 | local txn = require("resty.lmdb.transaction") 83 | local l = require("resty.lmdb") 84 | 85 | l.set("testbalabala", "aaaa") 86 | ngx.say(l.db_drop(false)) 87 | 88 | local t = txn.begin() 89 | t:set("test", "value") 90 | 91 | ngx.say(l.get("test")) 92 | ngx.say(t:commit()) 93 | ngx.say(l.get("test")) 94 | } 95 | } 96 | --- request 97 | GET /t 98 | --- response_body 99 | true 100 | nil 101 | true 102 | value 103 | --- no_error_log 104 | [error] 105 | [warn] 106 | [crit] 107 | 108 | 109 | 110 | === TEST 3: transaction failure aborts the entire transaction 111 | --- http_config eval: $::HttpConfig 112 | --- main_config eval: $::MainConfig 113 | --- config 114 | location = /t { 115 | content_by_lua_block { 116 | local txn = require("resty.lmdb.transaction") 117 | local l = require("resty.lmdb") 118 | 119 | l.set("testbalabala", "aaaa") 120 | ngx.say(l.db_drop(false)) 121 | 122 | local t = txn.begin() 123 | t:set("test", "value") 124 | t:get("test", "not_exist") -- this will fail 125 | 126 | ngx.say(t:commit()) 127 | ngx.say(l.get("test")) 128 | } 129 | } 130 | --- request 131 | GET /t 132 | --- response_body 133 | true 134 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 135 | nil 136 | --- no_error_log 137 | [error] 138 | [warn] 139 | [crit] 140 | 141 | 142 | 143 | === TEST 4: full update with drop_db 144 | --- http_config eval: $::HttpConfig 145 | --- main_config eval: $::MainConfig 146 | --- config 147 | location = /t { 148 | content_by_lua_block { 149 | local txn = require("resty.lmdb.transaction") 150 | local l = require("resty.lmdb") 151 | 152 | ngx.say(l.set("test", "value")) 153 | ngx.say(l.set("test1", "value1")) 154 | 155 | local t = txn.begin() 156 | t:db_drop(false) 157 | t:set("test", "new") 158 | t:set("test1", "new1") 159 | 160 | ngx.say(t:commit()) 161 | 162 | ngx.say(l.get("test")) 163 | ngx.say(l.get("test1")) 164 | } 165 | } 166 | --- request 167 | GET /t 168 | --- response_body 169 | true 170 | true 171 | true 172 | new 173 | new1 174 | --- no_error_log 175 | [error] 176 | [warn] 177 | [crit] 178 | -------------------------------------------------------------------------------- /t/04-not_enabled.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $HttpConfig = qq{ 13 | lua_package_path "$pwd/lib/?.lua;;"; 14 | }; 15 | 16 | no_long_string(); 17 | #no_diff(); 18 | 19 | run_tests(); 20 | 21 | __DATA__ 22 | 23 | === TEST 1: no LMDB environment defined 24 | --- http_config eval: $::HttpConfig 25 | --- config 26 | location = /t { 27 | content_by_lua_block { 28 | local l = require("resty.lmdb") 29 | 30 | ngx.say(l.db_drop(false)) 31 | ngx.say(l.get("test")) 32 | } 33 | } 34 | --- request 35 | GET /t 36 | --- response_body 37 | nilunable to open DB for DROP: no LMDB environment defined 38 | nilunable to open DB for GET 'test': no LMDB environment defined 39 | --- no_error_log 40 | [error] 41 | [warn] 42 | [crit] 43 | -------------------------------------------------------------------------------- /t/07-status.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path html/test.mdb; 14 | lmdb_max_databases 3; 15 | lmdb_map_size 5m; 16 | }; 17 | 18 | our $MainConfig2 = qq{ 19 | lmdb_environment_path html/test.mdb; 20 | lmdb_max_databases 3; 21 | lmdb_map_size 10m; 22 | }; 23 | 24 | our $HttpConfig = qq{ 25 | lua_package_path "$pwd/lib/?.lua;;"; 26 | }; 27 | 28 | no_long_string(); 29 | #no_diff(); 30 | 31 | run_tests(); 32 | 33 | __DATA__ 34 | 35 | === TEST 1: simple get env_info with 5M LMDB Map 36 | --- http_config eval: $::HttpConfig 37 | --- main_config eval: $::MainConfig 38 | --- config 39 | location = /t { 40 | content_by_lua_block { 41 | local l = require("resty.lmdb") 42 | local info = l.get_env_info() 43 | ngx.say(info["map_size"]) 44 | ngx.say(info["page_size"]) 45 | ngx.say(info["max_readers"]) 46 | ngx.say(info["num_readers"]) 47 | ngx.say(info["allocated_pages"]) 48 | ngx.say(info["in_use_pages"]) 49 | ngx.say(info["entries"]) 50 | } 51 | } 52 | --- request 53 | GET /t 54 | --- response_body_like chomp 55 | \d+ 56 | \d+ 57 | \d+ 58 | \d+ 59 | \d+ 60 | \d+ 61 | \d+ 62 | 63 | --- no_error_log 64 | [error] 65 | [warn] 66 | [crit] 67 | 68 | 69 | === TEST 2: simple get env_info with 10M LMDB Map 70 | --- http_config eval: $::HttpConfig 71 | --- main_config eval: $::MainConfig2 72 | --- config 73 | location = /t { 74 | content_by_lua_block { 75 | local l = require("resty.lmdb") 76 | local info = l.get_env_info() 77 | ngx.say(info["map_size"]) 78 | ngx.say(info["page_size"]) 79 | } 80 | } 81 | --- request 82 | GET /t 83 | --- response_body 84 | 10485760 85 | 4096 86 | --- no_error_log 87 | [error] 88 | [warn] 89 | [crit] 90 | 91 | 92 | 93 | === TEST 3: simple get env_info after set() / get() 94 | --- http_config eval: $::HttpConfig 95 | --- main_config eval: $::MainConfig 96 | --- config 97 | location = /t { 98 | content_by_lua_block { 99 | local l = require("resty.lmdb") 100 | local info = l.get_env_info() 101 | 102 | ngx.say(info["map_size"]) 103 | ngx.say(info["page_size"]) 104 | 105 | local old_in_use_pages = info["in_use_pages"] 106 | local a = string.rep("Abcdef", 5000) 107 | ngx.say(l.set("test", a)) 108 | ngx.say(l.get("test_not_exist")) 109 | 110 | local info = l.get_env_info() 111 | ngx.say(info["map_size"]) 112 | ngx.say(info["page_size"]) 113 | 114 | local in_use_pages = info["in_use_pages"] 115 | ngx.say(in_use_pages >= old_in_use_pages) 116 | } 117 | } 118 | --- request 119 | GET /t 120 | --- response_body 121 | 5242880 122 | 4096 123 | true 124 | nil 125 | 5242880 126 | 4096 127 | true 128 | --- no_error_log 129 | [error] 130 | [warn] 131 | [crit] 132 | 133 | 134 | 135 | === TEST 4: simple get env_info after transaction set() / get() 136 | --- http_config eval: $::HttpConfig 137 | --- main_config eval: $::MainConfig 138 | --- config 139 | location = /t { 140 | content_by_lua_block { 141 | local txn = require("resty.lmdb.transaction") 142 | local l = require("resty.lmdb") 143 | 144 | l.set("testbalabala", "aaaa") 145 | ngx.say(l.db_drop(false)) 146 | 147 | local info = l.get_env_info() 148 | ngx.say(info["map_size"]) 149 | ngx.say(info["page_size"]) 150 | local old_in_use_pages = info["in_use_pages"] 151 | local old_allocated_pages = info["allocated_pages"] 152 | local t = txn.begin() 153 | t:get("not_found") 154 | t:set("test", "value") 155 | t:get("test") 156 | t:set("test", nil) 157 | t:get("test") 158 | assert(t.n == 5) 159 | assert(t.write == true) 160 | ngx.say(t:commit()) 161 | assert(t.ops_capacity == 16) 162 | 163 | for i = 1, 5 do 164 | ngx.say(i, ": ", t[i].result) 165 | end 166 | 167 | ngx.say(l.get("test")) 168 | 169 | local info = l.get_env_info() 170 | ngx.say(info["map_size"]) 171 | ngx.say(info["page_size"]) 172 | local in_use_pages = info["in_use_pages"] 173 | local allocated_pages = info["allocated_pages"] 174 | ngx.say(in_use_pages >= old_in_use_pages) 175 | ngx.say(allocated_pages >= old_allocated_pages) 176 | } 177 | } 178 | --- request 179 | GET /t 180 | --- response_body 181 | true 182 | 5242880 183 | 4096 184 | true 185 | 1: nil 186 | 2: true 187 | 3: value 188 | 4: true 189 | 5: nil 190 | nil 191 | 5242880 192 | 4096 193 | true 194 | true 195 | 196 | --- no_error_log 197 | [error] 198 | [warn] 199 | [crit] 200 | -------------------------------------------------------------------------------- /t/08-max-key-size.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path /tmp/test.mdb; 14 | lmdb_map_size 5m; 15 | }; 16 | 17 | our $HttpConfig = qq{ 18 | lua_package_path "$pwd/lib/?.lua;;"; 19 | }; 20 | 21 | no_long_string(); 22 | #no_diff(); 23 | 24 | run_tests(); 25 | 26 | __DATA__ 27 | 28 | === TEST 1: key size is 512 set() / get() 29 | --- http_config eval: $::HttpConfig 30 | --- main_config eval: $::MainConfig 31 | --- config 32 | location = /t { 33 | content_by_lua_block { 34 | local l = require("resty.lmdb") 35 | 36 | local key = string.rep("a", 512) 37 | ngx.say(l.set(key, "value")) 38 | ngx.say(l.get(key)) 39 | } 40 | } 41 | --- request 42 | GET /t 43 | --- response_body 44 | true 45 | value 46 | --- no_error_log 47 | [error] 48 | [warn] 49 | [crit] 50 | 51 | 52 | === TEST 2: key size is 511 set() / get() 53 | --- http_config eval: $::HttpConfig 54 | --- main_config eval: $::MainConfig 55 | --- config 56 | location = /t { 57 | content_by_lua_block { 58 | local l = require("resty.lmdb") 59 | 60 | local key = string.rep("a", 511) 61 | ngx.say(l.set(key, "value")) 62 | ngx.say(l.get(key)) 63 | } 64 | } 65 | --- request 66 | GET /t 67 | --- response_body 68 | true 69 | value 70 | --- no_error_log 71 | [error] 72 | [warn] 73 | [crit] 74 | 75 | 76 | === TEST 3: key size is 1024 set() / get() 77 | --- http_config eval: $::HttpConfig 78 | --- main_config eval: $::MainConfig 79 | --- config 80 | location = /t { 81 | content_by_lua_block { 82 | local l = require("resty.lmdb") 83 | 84 | local key = string.rep("a", 1024) 85 | ngx.say(l.set(key, "value")) 86 | ngx.say(l.get(key)) 87 | } 88 | } 89 | --- request 90 | GET /t 91 | --- response_body 92 | true 93 | value 94 | --- no_error_log 95 | [error] 96 | [warn] 97 | [crit] 98 | 99 | 100 | === TEST 4: key size is 448 set() / get() 101 | --- http_config eval: $::HttpConfig 102 | --- main_config eval: $::MainConfig 103 | --- config 104 | location = /t { 105 | content_by_lua_block { 106 | local l = require("resty.lmdb") 107 | 108 | local key = string.rep("a", 448) 109 | ngx.say(l.set(key, "value")) 110 | ngx.say(l.get(key)) 111 | } 112 | } 113 | --- request 114 | GET /t 115 | --- response_body 116 | true 117 | value 118 | --- no_error_log 119 | [error] 120 | [warn] 121 | [crit] 122 | 123 | 124 | === TEST 5: different keys are ok 125 | --- http_config eval: $::HttpConfig 126 | --- main_config eval: $::MainConfig 127 | --- config 128 | location = /t { 129 | content_by_lua_block { 130 | local l = require("resty.lmdb") 131 | 132 | local key1 = string.rep("a", 512) 133 | ngx.say(l.set(key1, "value1")) 134 | ngx.say(l.get(key1)) 135 | 136 | local key2 = string.rep("b", 512) 137 | ngx.say(l.set(key2, "value2")) 138 | ngx.say(l.get(key2)) 139 | } 140 | } 141 | --- request 142 | GET /t 143 | --- response_body 144 | true 145 | value1 146 | true 147 | value2 148 | --- no_error_log 149 | [error] 150 | [warn] 151 | [crit] 152 | -------------------------------------------------------------------------------- /t/09-check-env-1.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | # should remove it and create the proper lmdb files 13 | system("rm -rf /tmp/test.mdb && echo 123 > /tmp/test.mdb"); 14 | 15 | our $MainConfig = qq{ 16 | lmdb_environment_path /tmp/test.mdb; 17 | lmdb_map_size 5m; 18 | }; 19 | 20 | our $HttpConfig = qq{ 21 | lua_package_path "$pwd/lib/?.lua;;"; 22 | }; 23 | 24 | no_long_string(); 25 | #no_diff(); 26 | 27 | run_tests(); 28 | 29 | __DATA__ 30 | 31 | === TEST 1: simple set() / get() 32 | --- http_config eval: $::HttpConfig 33 | --- main_config eval: $::MainConfig 34 | --- config 35 | location = /t { 36 | content_by_lua_block { 37 | local l = require("resty.lmdb") 38 | 39 | ngx.say(l.set("test", "value")) 40 | ngx.say(l.get("test")) 41 | ngx.say(l.get("test_not_exist")) 42 | } 43 | } 44 | --- request 45 | GET /t 46 | --- response_body 47 | true 48 | value 49 | nil 50 | --- no_error_log 51 | [error] 52 | [crit] 53 | [alert] 54 | -------------------------------------------------------------------------------- /t/09-check-env-2.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | # should remove them and create the proper lmdb files 13 | system("rm -rf /tmp/test.mdb && mkdir /tmp/test.mdb"); 14 | system("echo 123 >/tmp/test.mdb/data.mdb"); 15 | system("echo 123 >/tmp/test.mdb/lock.mdb"); 16 | 17 | our $MainConfig = qq{ 18 | lmdb_environment_path /tmp/test.mdb; 19 | lmdb_map_size 5m; 20 | }; 21 | 22 | our $HttpConfig = qq{ 23 | lua_package_path "$pwd/lib/?.lua;;"; 24 | }; 25 | 26 | no_long_string(); 27 | #no_diff(); 28 | 29 | run_tests(); 30 | 31 | __DATA__ 32 | 33 | === TEST 1: simple set() / get() 34 | --- http_config eval: $::HttpConfig 35 | --- main_config eval: $::MainConfig 36 | --- config 37 | location = /t { 38 | content_by_lua_block { 39 | local l = require("resty.lmdb") 40 | 41 | ngx.say(l.set("test", "value")) 42 | ngx.say(l.get("test")) 43 | ngx.say(l.get("test_not_exist")) 44 | } 45 | } 46 | --- request 47 | GET /t 48 | --- response_body 49 | true 50 | value 51 | nil 52 | --- no_error_log 53 | [error] 54 | [crit] 55 | [alert] 56 | -------------------------------------------------------------------------------- /t/10-prefix.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path /tmp/test.mdb; 14 | lmdb_map_size 5m; 15 | }; 16 | 17 | our $HttpConfig = qq{ 18 | lua_package_path "$pwd/lib/?.lua;;"; 19 | }; 20 | 21 | no_long_string(); 22 | #no_diff(); 23 | 24 | run_tests(); 25 | 26 | __DATA__ 27 | 28 | === TEST 1: prefix() operation 29 | --- http_config eval: $::HttpConfig 30 | --- main_config eval: $::MainConfig 31 | --- config 32 | location = /t { 33 | content_by_lua_block { 34 | local l = require("resty.lmdb") 35 | 36 | ngx.say(l.db_drop(true)) 37 | ngx.say(l.set("test", "value")) 38 | ngx.say(l.set("test1", "value1")) 39 | ngx.say(l.set("test2", "value2")) 40 | ngx.say(l.set("test3", "value3")) 41 | ngx.say(l.set("u", "value4")) 42 | ngx.say(l.set("u1", "value5")) 43 | 44 | for k, v in l.prefix("tes") do 45 | ngx.say("key: ", k, " value: ", v) 46 | end 47 | } 48 | } 49 | --- request 50 | GET /t 51 | --- response_body 52 | true 53 | true 54 | true 55 | true 56 | true 57 | true 58 | true 59 | key: test value: value 60 | key: test1 value: value1 61 | key: test2 value: value2 62 | key: test3 value: value3 63 | --- no_error_log 64 | [error] 65 | [warn] 66 | [crit] 67 | 68 | 69 | 70 | === TEST 2: prefix() operation not found 71 | --- http_config eval: $::HttpConfig 72 | --- main_config eval: $::MainConfig 73 | --- config 74 | location = /t { 75 | content_by_lua_block { 76 | local l = require("resty.lmdb") 77 | 78 | ngx.say(l.db_drop(true)) 79 | ngx.say(l.set("test", "value")) 80 | 81 | for k, v in l.prefix("test1") do 82 | ngx.say("key: ", k, " value: ", v) 83 | end 84 | } 85 | } 86 | --- request 87 | GET /t 88 | --- response_body 89 | true 90 | true 91 | --- no_error_log 92 | [error] 93 | [warn] 94 | [crit] 95 | 96 | 97 | 98 | === TEST 3: prefix() operation only 1 result 99 | --- http_config eval: $::HttpConfig 100 | --- main_config eval: $::MainConfig 101 | --- config 102 | location = /t { 103 | content_by_lua_block { 104 | local l = require("resty.lmdb") 105 | 106 | ngx.say(l.db_drop(true)) 107 | ngx.say(l.set("test", "value")) 108 | 109 | for k, v in l.prefix("test") do 110 | ngx.say("key: ", k, " value: ", v) 111 | end 112 | } 113 | } 114 | --- request 115 | GET /t 116 | --- response_body 117 | true 118 | true 119 | key: test value: value 120 | --- no_error_log 121 | [error] 122 | [warn] 123 | [crit] 124 | 125 | 126 | 127 | === TEST 4: prefix() operation 511-513 keys (edge cases) 128 | --- http_config eval: $::HttpConfig 129 | --- main_config eval: $::MainConfig 130 | --- config 131 | location = /t { 132 | content_by_lua_block { 133 | local l = require("resty.lmdb") 134 | 135 | for i = 511, 513 do 136 | ngx.say(l.db_drop(true)) 137 | 138 | for j = 1, i do 139 | assert(l.set("test:" .. j, "value:" .. j)) 140 | end 141 | 142 | local j = 0 143 | local found = {} 144 | for k, v in l.prefix("test:") do 145 | j = j + 1 146 | found[k] = v 147 | end 148 | 149 | ngx.say("j = ", j) 150 | 151 | for j = 1, i do 152 | assert(found["test:" .. j] == "value:" .. j) 153 | end 154 | 155 | ngx.say("done") 156 | end 157 | } 158 | } 159 | --- request 160 | GET /t 161 | --- response_body 162 | true 163 | j = 511 164 | done 165 | true 166 | j = 512 167 | done 168 | true 169 | j = 513 170 | done 171 | --- no_error_log 172 | [error] 173 | [warn] 174 | [crit] 175 | 176 | 177 | 178 | === TEST 5: large values 179 | --- http_config eval: $::HttpConfig 180 | --- main_config eval: $::MainConfig 181 | --- config 182 | location = /t { 183 | content_by_lua_block { 184 | local l = require("resty.lmdb") 185 | 186 | ngx.say(l.db_drop(true)) 187 | for i = 1, 1024 do 188 | assert(l.set("test:" .. i, "value:" .. string.rep("x", i))) 189 | end 190 | 191 | local j = 0 192 | local found = {} 193 | for k, v in l.prefix("test:") do 194 | j = j + 1 195 | found[k] = v 196 | end 197 | 198 | ngx.say("j = ", j) 199 | 200 | for i = 1, 1024 do 201 | assert(found["test:" .. i] == "value:" .. string.rep("x", i)) 202 | end 203 | 204 | ngx.say("done") 205 | } 206 | } 207 | --- request 208 | GET /t 209 | --- response_body 210 | true 211 | j = 1024 212 | done 213 | --- no_error_log 214 | [error] 215 | [warn] 216 | [crit] 217 | 218 | 219 | 220 | === TEST 6: prefix.page() operation 221 | --- http_config eval: $::HttpConfig 222 | --- main_config eval: $::MainConfig 223 | --- config 224 | location = /t { 225 | content_by_lua_block { 226 | local l = require("resty.lmdb") 227 | 228 | ngx.say(l.db_drop(true)) 229 | ngx.say(l.set("test", "value")) 230 | ngx.say(l.set("test1", "value1")) 231 | ngx.say(l.set("test2", "value2")) 232 | ngx.say(l.set("test3", "value3")) 233 | ngx.say(l.set("u", "value4")) 234 | ngx.say(l.set("u1", "value5")) 235 | 236 | local p = require("resty.lmdb.prefix") 237 | 238 | local res, err = p.page("test", "test") 239 | if not res then 240 | ngx.say("page errored: ", err) 241 | end 242 | 243 | for _, pair in ipairs(res) do 244 | ngx.say("key: ", pair.key, " value: ", pair.value) 245 | end 246 | } 247 | } 248 | --- request 249 | GET /t 250 | --- response_body 251 | true 252 | true 253 | true 254 | true 255 | true 256 | true 257 | true 258 | key: test value: value 259 | key: test1 value: value1 260 | key: test2 value: value2 261 | key: test3 value: value3 262 | --- no_error_log 263 | [error] 264 | [warn] 265 | [crit] 266 | 267 | 268 | 269 | === TEST 7: prefix.page() operation with custom page size 270 | --- http_config eval: $::HttpConfig 271 | --- main_config eval: $::MainConfig 272 | --- config 273 | location = /t { 274 | content_by_lua_block { 275 | local l = require("resty.lmdb") 276 | 277 | ngx.say(l.db_drop(true)) 278 | ngx.say(l.set("test", "value")) 279 | ngx.say(l.set("test1", "value1")) 280 | ngx.say(l.set("test2", "value2")) 281 | ngx.say(l.set("test3", "value3")) 282 | ngx.say(l.set("u", "value4")) 283 | ngx.say(l.set("u1", "value5")) 284 | 285 | local p = require("resty.lmdb.prefix") 286 | 287 | local res, err = p.page("test", "test", nil, 2) 288 | if not res then 289 | ngx.say("page errored: ", err) 290 | end 291 | 292 | ngx.say("FIRST PAGE") 293 | for _, pair in ipairs(res) do 294 | ngx.say("key: ", pair.key, " value: ", pair.value) 295 | end 296 | 297 | res, err = p.page("test1\x00", "test", nil, 2) 298 | if not res then 299 | ngx.say("page errored: ", err) 300 | end 301 | 302 | ngx.say("SECOND PAGE") 303 | for _, pair in ipairs(res) do 304 | ngx.say("key: ", pair.key, " value: ", pair.value) 305 | end 306 | } 307 | } 308 | --- request 309 | GET /t 310 | --- response_body 311 | true 312 | true 313 | true 314 | true 315 | true 316 | true 317 | true 318 | FIRST PAGE 319 | key: test value: value 320 | key: test1 value: value1 321 | SECOND PAGE 322 | key: test2 value: value2 323 | key: test3 value: value3 324 | --- no_error_log 325 | [error] 326 | [warn] 327 | [crit] 328 | 329 | 330 | 331 | === TEST 8: prefix.page() operation with page size 1 332 | --- http_config eval: $::HttpConfig 333 | --- main_config eval: $::MainConfig 334 | --- config 335 | location = /t { 336 | content_by_lua_block { 337 | local l = require("resty.lmdb") 338 | 339 | l.db_drop(true) 340 | l.set("test", "value") 341 | l.set("test1", "value1") 342 | l.set("test2", "value2") 343 | l.set("test3", "value3") 344 | l.set("u", "value4") 345 | l.set("u1", "value5") 346 | 347 | local p = require("resty.lmdb.prefix") 348 | 349 | ngx.say("PAGE SIZE 1") 350 | local res, more = assert(p.page("test", "test", nil, 1)) 351 | 352 | assert(#res == 1) 353 | ngx.say(res[1].key, " ", res[1].value) 354 | assert(more) 355 | 356 | ngx.say("PAGE SIZE 2") 357 | res, more = p.page("test", "test", nil, 2) 358 | assert(more) 359 | 360 | assert(#res == 2) 361 | for _, pair in ipairs(res) do 362 | ngx.say(pair.key, " ", pair.value) 363 | end 364 | 365 | ngx.say("PAGE SIZE 0") 366 | local ok, err = pcall(p.page, "test", "test", nil, 0) 367 | if not ok then 368 | ngx.say(err) 369 | end 370 | } 371 | } 372 | --- request 373 | GET /t 374 | --- response_body 375 | PAGE SIZE 1 376 | test value 377 | PAGE SIZE 2 378 | test value 379 | test1 value1 380 | PAGE SIZE 0 381 | .../lua-resty-lmdb/lua-resty-lmdb/lib/resty/lmdb/prefix.lua:34: page_size must be at least 1 382 | --- no_error_log 383 | [error] 384 | [warn] 385 | [crit] 386 | 387 | 388 | 389 | === TEST 9: prefix.page() operation with large page size [KAG-5874] 390 | --- http_config eval: $::HttpConfig 391 | --- main_config eval: $::MainConfig 392 | --- config 393 | location = /t { 394 | content_by_lua_block { 395 | local l = require("resty.lmdb") 396 | 397 | ngx.say(l.db_drop(true)) 398 | 399 | ngx.say(l.set("test", "value")) 400 | 401 | local inserted = { test = "value" } 402 | 403 | for i = 1, 2048 do 404 | -- string.rep with 120 makes sure each page goes just over the 1MB 405 | -- default buffer size and triggers a realloc 406 | assert(l.set(string.rep("test", 120) .. i, string.rep("value", 120))) 407 | inserted[string.rep("test", 120) .. i] = string.rep("value", 120) 408 | end 409 | 410 | ngx.say(l.set("u", "value4")) 411 | ngx.say(l.set("u1", "value5")) 412 | 413 | local p = require("resty.lmdb.prefix") 414 | 415 | local res, err = p.page("test", "test", nil, 1000) 416 | if not res then 417 | ngx.say("page errored: ", err) 418 | end 419 | 420 | ngx.say("FIRST PAGE") 421 | for _, pair in ipairs(res) do 422 | inserted[pair.key] = nil 423 | end 424 | 425 | res, err = p.page(res[#res].key .. "\x00", "test", nil, 1000) 426 | if not res then 427 | ngx.say("page errored: ", err) 428 | end 429 | ngx.say("SECOND PAGE") 430 | for _, pair in ipairs(res) do 431 | inserted[pair.key] = nil 432 | end 433 | 434 | res, err = p.page(res[#res].key .. "\x00", "test", nil, 1000) 435 | if not res then 436 | ngx.say("page errored: ", err) 437 | end 438 | ngx.say("THIRD PAGE") 439 | for _, pair in ipairs(res) do 440 | inserted[pair.key] = nil 441 | end 442 | 443 | ngx.say(next(inserted)) 444 | } 445 | } 446 | --- request 447 | GET /t 448 | --- response_body 449 | true 450 | true 451 | true 452 | true 453 | FIRST PAGE 454 | SECOND PAGE 455 | THIRD PAGE 456 | nil 457 | --- no_error_log 458 | [error] 459 | [warn] 460 | [crit] 461 | -------------------------------------------------------------------------------- /t/11-validation-tag-plain.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | # run this test after 00-init_mdb.t 4 | 5 | use Test::Nginx::Socket::Lua; 6 | use Cwd qw(cwd); 7 | 8 | repeat_each(1); 9 | 10 | plan tests => repeat_each() * blocks() * 7 + 2; 11 | 12 | my $pwd = cwd(); 13 | 14 | # remove db for testing 15 | system("rm -rf /tmp/test10-plain.mdb"); 16 | 17 | our $MainConfig1 = qq{ 18 | lmdb_environment_path /tmp/test10-plain.mdb; 19 | lmdb_map_size 5m; 20 | }; 21 | 22 | our $MainConfig2 = qq{ 23 | lmdb_environment_path /tmp/test10-plain.mdb; 24 | lmdb_map_size 5m; 25 | lmdb_validation_tag 3.3; 26 | }; 27 | 28 | our $MainConfig3 = qq{ 29 | lmdb_environment_path /tmp/test10-plain.mdb; 30 | lmdb_map_size 5m; 31 | lmdb_validation_tag 3.4; 32 | }; 33 | 34 | our $HttpConfig = qq{ 35 | lua_package_path "$pwd/lib/?.lua;;"; 36 | }; 37 | 38 | no_long_string(); 39 | #no_diff(); 40 | 41 | no_shuffle(); 42 | run_tests(); 43 | 44 | __DATA__ 45 | 46 | === TEST 1: no validation_tag 47 | --- http_config eval: $::HttpConfig 48 | --- main_config eval: $::MainConfig1 49 | --- config 50 | location = /t { 51 | content_by_lua_block { 52 | local l = require("resty.lmdb") 53 | 54 | ngx.say(l.set("test", "value")) 55 | ngx.say(l.get("test")) 56 | ngx.say(l.get("test_not_exist")) 57 | 58 | ngx.say(l.get("validation_tag")) 59 | } 60 | } 61 | --- request 62 | GET /t 63 | --- response_body 64 | true 65 | value 66 | nil 67 | nil 68 | --- no_error_log 69 | [warn] 70 | [error] 71 | [crit] 72 | 73 | 74 | === TEST 2: start and set validation_tag 75 | --- http_config eval: $::HttpConfig 76 | --- main_config eval: $::MainConfig2 77 | --- config 78 | location = /t { 79 | content_by_lua_block { 80 | local l = require("resty.lmdb") 81 | 82 | ngx.say(l.get("test")) 83 | 84 | ngx.say(l.set("test", "value")) 85 | ngx.say(l.get("test")) 86 | ngx.say(l.get("test_not_exist")) 87 | } 88 | } 89 | --- request 90 | GET /t 91 | --- response_body 92 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 93 | true 94 | value 95 | nil 96 | --- error_log 97 | LMDB validation enabled, using validation tag: "3.3" 98 | LMDB validation tag does not exist 99 | LMDB validation tag mismatch, wiping the database 100 | set LMDB validation tag: "3.3" 101 | --- no_error_log 102 | [warn] 103 | [error] 104 | [crit] 105 | 106 | 107 | === TEST 3: change validation_tag 108 | --- http_config eval: $::HttpConfig 109 | --- main_config eval: $::MainConfig3 110 | --- config 111 | location = /t { 112 | content_by_lua_block { 113 | local l = require("resty.lmdb") 114 | 115 | ngx.say(l.get("test")) 116 | 117 | ngx.say(l.set("test", "value")) 118 | ngx.say(l.get("test")) 119 | ngx.say(l.get("test_not_exist")) 120 | } 121 | } 122 | --- request 123 | GET /t 124 | --- response_body 125 | nilunable to open DB for GET 'test': MDB_NOTFOUND: No matching key/data pair found 126 | true 127 | value 128 | nil 129 | --- error_log 130 | LMDB validation enabled, using validation tag: "3.4" 131 | LMDB validation tag "3.3" did not match configured tag "3.4" 132 | LMDB validation tag mismatch, wiping the database 133 | set LMDB validation tag: "3.4" 134 | --- no_error_log 135 | [emerg] 136 | [error] 137 | [crit] 138 | -------------------------------------------------------------------------------- /t/12-max-readers.t: -------------------------------------------------------------------------------- 1 | # vim:set ft= ts=4 sw=4 et: 2 | 3 | use Test::Nginx::Socket::Lua; 4 | use Cwd qw(cwd); 5 | 6 | repeat_each(2); 7 | 8 | plan tests => repeat_each() * blocks() * 5; 9 | 10 | my $pwd = cwd(); 11 | 12 | our $MainConfig = qq{ 13 | lmdb_environment_path /tmp/test.mdb; 14 | lmdb_map_size 5m; 15 | }; 16 | 17 | our $HttpConfig = qq{ 18 | lua_package_path "$pwd/lib/?.lua;;"; 19 | 20 | init_worker_by_lua_block { 21 | local lmdb_set = require("resty.lmdb").set 22 | lmdb_set("worker" .. ngx.worker.id(), tostring(ngx.worker.id())) 23 | } 24 | }; 25 | 26 | no_long_string(); 27 | #no_diff(); 28 | 29 | master_process_enabled('on'); 30 | workers(200); 31 | 32 | run_tests(); 33 | 34 | __DATA__ 35 | 36 | === TEST 1: simple set() / get() 37 | --- http_config eval: $::HttpConfig 38 | --- main_config eval: $::MainConfig 39 | --- config 40 | location = /t { 41 | content_by_lua_block { 42 | local l = require("resty.lmdb") 43 | 44 | ngx.say(l.set("test", "value")) 45 | ngx.say(l.get("test")) 46 | ngx.say(l.get("test_not_exist")) 47 | } 48 | } 49 | --- request 50 | GET /t 51 | --- response_body 52 | true 53 | value 54 | nil 55 | --- no_error_log 56 | [error] 57 | [warn] 58 | [crit] 59 | -------------------------------------------------------------------------------- /valgrind.suppress: -------------------------------------------------------------------------------- 1 | { 2 | 3 | Memcheck:Cond 4 | fun:str_fastcmp 5 | fun:lj_str_new 6 | fun:lua_pushlstring 7 | fun:emptybuffer 8 | fun:luaL_pushresult 9 | fun:luaL_gsub 10 | fun:ngx_http_lua_set_path.isra.7.constprop.21 11 | fun:ngx_http_lua_new_state 12 | fun:ngx_http_lua_init_vm 13 | fun:ngx_http_lua_init 14 | fun:ngx_http_block 15 | fun:ngx_conf_handler 16 | fun:ngx_conf_parse 17 | fun:ngx_init_cycle 18 | fun:main 19 | } 20 | { 21 | 22 | Memcheck:Param 23 | write(buf) 24 | fun:__write_nocancel 25 | fun:ngx_log_error_core 26 | fun:ngx_resolver_read_response 27 | } 28 | { 29 | 30 | Memcheck:Cond 31 | fun:ngx_sprintf_num 32 | fun:ngx_vslprintf 33 | fun:ngx_log_error_core 34 | fun:ngx_resolver_read_response 35 | fun:ngx_epoll_process_events 36 | fun:ngx_process_events_and_timers 37 | fun:ngx_single_process_cycle 38 | fun:main 39 | } 40 | { 41 | 42 | Memcheck:Addr1 43 | fun:ngx_vslprintf 44 | fun:ngx_snprintf 45 | fun:ngx_sock_ntop 46 | fun:ngx_event_accept 47 | } 48 | { 49 | 50 | Memcheck:Param 51 | write(buf) 52 | fun:__write_nocancel 53 | fun:ngx_log_error_core 54 | fun:ngx_resolver_read_response 55 | fun:ngx_event_process_posted 56 | fun:ngx_process_events_and_timers 57 | fun:ngx_single_process_cycle 58 | fun:main 59 | } 60 | { 61 | 62 | Memcheck:Cond 63 | fun:ngx_sprintf_num 64 | fun:ngx_vslprintf 65 | fun:ngx_log_error_core 66 | fun:ngx_resolver_read_response 67 | fun:ngx_event_process_posted 68 | fun:ngx_process_events_and_timers 69 | fun:ngx_single_process_cycle 70 | fun:main 71 | } 72 | { 73 | 74 | Memcheck:Leak 75 | fun:malloc 76 | fun:ngx_alloc 77 | obj:* 78 | } 79 | { 80 | 81 | exp-sgcheck:SorG 82 | fun:ngx_http_lua_ndk_set_var_get 83 | } 84 | { 85 | 86 | exp-sgcheck:SorG 87 | fun:ngx_http_variables_init_vars 88 | fun:ngx_http_block 89 | } 90 | { 91 | 92 | exp-sgcheck:SorG 93 | fun:ngx_conf_parse 94 | } 95 | { 96 | 97 | exp-sgcheck:SorG 98 | fun:ngx_vslprintf 99 | fun:ngx_log_error_core 100 | } 101 | { 102 | 103 | Memcheck:Leak 104 | fun:malloc 105 | fun:ngx_alloc 106 | fun:ngx_calloc 107 | fun:ngx_event_process_init 108 | } 109 | { 110 | 111 | Memcheck:Param 112 | epoll_ctl(event) 113 | fun:epoll_ctl 114 | } 115 | { 116 | 117 | Memcheck:Leak 118 | fun:malloc 119 | fun:ngx_alloc 120 | fun:ngx_event_process_init 121 | } 122 | { 123 | 124 | Memcheck:Cond 125 | fun:ngx_conf_flush_files 126 | fun:ngx_single_process_cycle 127 | } 128 | { 129 | 130 | Memcheck:Cond 131 | fun:memcpy 132 | fun:ngx_vslprintf 133 | fun:ngx_log_error_core 134 | fun:ngx_http_charset_header_filter 135 | } 136 | { 137 | 138 | Memcheck:Param 139 | socketcall.setsockopt(optval) 140 | fun:setsockopt 141 | fun:drizzle_state_connect 142 | } 143 | { 144 | 145 | Memcheck:Leak 146 | fun:malloc 147 | fun:ngx_alloc 148 | fun:ngx_pool_cleanup_add 149 | } 150 | { 151 | 152 | Memcheck:Cond 153 | fun:ngx_conf_flush_files 154 | fun:ngx_single_process_cycle 155 | fun:main 156 | } 157 | { 158 | 159 | Memcheck:Leak 160 | fun:malloc 161 | fun:ngx_alloc 162 | fun:ngx_palloc_large 163 | fun:ngx_palloc 164 | fun:ngx_array_push 165 | fun:ngx_http_get_variable_index 166 | fun:ngx_http_memc_add_variable 167 | fun:ngx_http_memc_init 168 | fun:ngx_http_block 169 | fun:ngx_conf_parse 170 | fun:ngx_init_cycle 171 | fun:main 172 | } 173 | { 174 | 175 | Memcheck:Leak 176 | fun:malloc 177 | fun:ngx_alloc 178 | fun:ngx_event_process_init 179 | fun:ngx_single_process_cycle 180 | fun:main 181 | } 182 | { 183 | 184 | Memcheck:Leak 185 | fun:malloc 186 | fun:ngx_alloc 187 | fun:ngx_crc32_table_init 188 | fun:main 189 | } 190 | { 191 | 192 | Memcheck:Leak 193 | fun:malloc 194 | fun:ngx_alloc 195 | fun:ngx_event_process_init 196 | fun:ngx_worker_process_init 197 | fun:ngx_worker_process_cycle 198 | fun:ngx_spawn_process 199 | fun:ngx_start_worker_processes 200 | fun:ngx_master_process_cycle 201 | fun:main 202 | } 203 | { 204 | 205 | Memcheck:Leak 206 | fun:malloc 207 | fun:ngx_alloc 208 | fun:ngx_palloc_large 209 | fun:ngx_palloc 210 | fun:ngx_pcalloc 211 | fun:ngx_hash_init 212 | fun:ngx_http_variables_init_vars 213 | fun:ngx_http_block 214 | fun:ngx_conf_parse 215 | fun:ngx_init_cycle 216 | fun:main 217 | } 218 | { 219 | 220 | Memcheck:Leak 221 | fun:malloc 222 | fun:ngx_alloc 223 | fun:ngx_palloc_large 224 | fun:ngx_palloc 225 | fun:ngx_pcalloc 226 | fun:ngx_http_upstream_drizzle_create_srv_conf 227 | fun:ngx_http_upstream 228 | fun:ngx_conf_parse 229 | fun:ngx_http_block 230 | fun:ngx_conf_parse 231 | fun:ngx_init_cycle 232 | fun:main 233 | } 234 | { 235 | 236 | Memcheck:Leak 237 | fun:malloc 238 | fun:ngx_alloc 239 | fun:ngx_palloc_large 240 | fun:ngx_palloc 241 | fun:ngx_pcalloc 242 | fun:ngx_hash_keys_array_init 243 | fun:ngx_http_variables_add_core_vars 244 | fun:ngx_http_core_preconfiguration 245 | fun:ngx_http_block 246 | fun:ngx_conf_parse 247 | fun:ngx_init_cycle 248 | fun:main 249 | } 250 | { 251 | 252 | Memcheck:Leak 253 | fun:malloc 254 | fun:ngx_alloc 255 | fun:ngx_palloc_large 256 | fun:ngx_palloc 257 | fun:ngx_array_push 258 | fun:ngx_hash_add_key 259 | fun:ngx_http_add_variable 260 | fun:ngx_http_echo_add_variables 261 | fun:ngx_http_echo_handler_init 262 | fun:ngx_http_block 263 | fun:ngx_conf_parse 264 | fun:ngx_init_cycle 265 | } 266 | { 267 | 268 | Memcheck:Leak 269 | fun:malloc 270 | fun:ngx_alloc 271 | fun:ngx_palloc_large 272 | fun:ngx_palloc 273 | fun:ngx_pcalloc 274 | fun:ngx_http_upstream_drizzle_create_srv_conf 275 | fun:ngx_http_core_server 276 | fun:ngx_conf_parse 277 | fun:ngx_http_block 278 | fun:ngx_conf_parse 279 | fun:ngx_init_cycle 280 | fun:main 281 | } 282 | { 283 | 284 | Memcheck:Leak 285 | fun:malloc 286 | fun:ngx_alloc 287 | fun:ngx_palloc_large 288 | fun:ngx_palloc 289 | fun:ngx_pcalloc 290 | fun:ngx_http_upstream_drizzle_create_srv_conf 291 | fun:ngx_http_block 292 | fun:ngx_conf_parse 293 | fun:ngx_init_cycle 294 | fun:main 295 | } 296 | { 297 | 298 | Memcheck:Leak 299 | fun:malloc 300 | fun:ngx_alloc 301 | fun:ngx_palloc_large 302 | fun:ngx_palloc 303 | fun:ngx_array_push 304 | fun:ngx_hash_add_key 305 | fun:ngx_http_variables_add_core_vars 306 | fun:ngx_http_core_preconfiguration 307 | fun:ngx_http_block 308 | fun:ngx_conf_parse 309 | fun:ngx_init_cycle 310 | fun:main 311 | } 312 | { 313 | 314 | Memcheck:Leak 315 | fun:malloc 316 | fun:ngx_alloc 317 | fun:ngx_palloc_large 318 | fun:ngx_palloc 319 | fun:ngx_pcalloc 320 | fun:ngx_init_cycle 321 | fun:main 322 | } 323 | { 324 | 325 | Memcheck:Leak 326 | fun:malloc 327 | fun:ngx_alloc 328 | fun:ngx_palloc_large 329 | fun:ngx_palloc 330 | fun:ngx_hash_init 331 | fun:ngx_http_upstream_init_main_conf 332 | fun:ngx_http_block 333 | fun:ngx_conf_parse 334 | fun:ngx_init_cycle 335 | fun:main 336 | } 337 | { 338 | 339 | Memcheck:Leak 340 | fun:malloc 341 | fun:ngx_alloc 342 | fun:ngx_palloc_large 343 | fun:ngx_palloc 344 | fun:ngx_pcalloc 345 | fun:ngx_http_drizzle_keepalive_init 346 | fun:ngx_http_upstream_drizzle_init 347 | fun:ngx_http_upstream_init_main_conf 348 | fun:ngx_http_block 349 | fun:ngx_conf_parse 350 | fun:ngx_init_cycle 351 | fun:main 352 | } 353 | { 354 | 355 | Memcheck:Leak 356 | fun:malloc 357 | fun:ngx_alloc 358 | fun:ngx_palloc_large 359 | fun:ngx_palloc 360 | fun:ngx_hash_init 361 | fun:ngx_http_variables_init_vars 362 | fun:ngx_http_block 363 | fun:ngx_conf_parse 364 | fun:ngx_init_cycle 365 | fun:main 366 | } 367 | { 368 | 369 | Memcheck:Cond 370 | fun:index 371 | fun:expand_dynamic_string_token 372 | fun:_dl_map_object 373 | fun:map_doit 374 | fun:_dl_catch_error 375 | fun:do_preload 376 | fun:dl_main 377 | fun:_dl_sysdep_start 378 | fun:_dl_start 379 | } 380 | { 381 | 382 | Memcheck:Leak 383 | match-leak-kinds: definite 384 | fun:malloc 385 | fun:ngx_alloc 386 | fun:ngx_set_environment 387 | fun:ngx_single_process_cycle 388 | } 389 | { 390 | 391 | Memcheck:Leak 392 | match-leak-kinds: definite 393 | fun:malloc 394 | fun:ngx_alloc 395 | fun:ngx_set_environment 396 | fun:ngx_worker_process_init 397 | fun:ngx_worker_process_cycle 398 | } 399 | { 400 | 401 | Memcheck:Leak 402 | match-leak-kinds: definite 403 | fun:malloc 404 | fun:ngx_alloc 405 | fun:ngx_event_process_init 406 | fun:ngx_worker_process_init 407 | } 408 | 409 | 410 | { 411 | 412 | Memcheck:Addr1 413 | fun:ngx_http_lua_ngx_echo 414 | fun:ngx_http_lua_ngx_say 415 | fun:lj_BC_FUNCC 416 | fun:lua_resume 417 | fun:ngx_http_lua_run_thread 418 | fun:ngx_http_lua_content_by_chunk 419 | fun:ngx_http_lua_content_handler_inline 420 | fun:ngx_http_lua_content_handler 421 | fun:ngx_http_core_content_phase 422 | fun:ngx_http_core_run_phases 423 | fun:ngx_http_handler 424 | fun:ngx_http_process_request 425 | fun:ngx_http_process_request_headers 426 | fun:ngx_http_process_request_line 427 | fun:ngx_http_wait_request_handler 428 | fun:ngx_epoll_process_events 429 | fun:ngx_process_events_and_timers 430 | fun:ngx_single_process_cycle 431 | fun:main 432 | } 433 | { 434 | 435 | Memcheck:Param 436 | writev(vector[...]) 437 | fun:writev 438 | fun:ngx_writev 439 | fun:ngx_linux_sendfile_chain 440 | fun:ngx_http_write_filter 441 | fun:ngx_http_chunked_body_filter 442 | fun:ngx_http_gzip_body_filter 443 | fun:ngx_http_postpone_filter 444 | fun:ngx_http_ssi_body_filter 445 | fun:ngx_http_charset_body_filter 446 | fun:ngx_http_trailers_filter 447 | fun:ngx_http_lua_capture_body_filter 448 | fun:ngx_output_chain 449 | fun:ngx_http_copy_filter 450 | fun:ngx_http_range_body_filter 451 | fun:ngx_http_output_filter 452 | fun:ngx_http_send_special 453 | fun:ngx_http_lua_send_special 454 | fun:ngx_http_lua_send_chain_link 455 | fun:ngx_http_lua_run_thread 456 | fun:ngx_http_lua_content_by_chunk 457 | fun:ngx_http_lua_content_handler_inline 458 | fun:ngx_http_lua_content_handler 459 | fun:ngx_http_core_content_phase 460 | fun:ngx_http_core_run_phases 461 | fun:ngx_http_handler 462 | fun:ngx_http_process_request 463 | fun:ngx_http_process_request_headers 464 | fun:ngx_http_process_request_line 465 | fun:ngx_http_wait_request_handler 466 | fun:ngx_epoll_process_events 467 | fun:ngx_process_events_and_timers 468 | fun:ngx_single_process_cycle 469 | fun:main 470 | } 471 | --------------------------------------------------------------------------------