├── .github └── workflows │ └── erlang.yml ├── .gitignore ├── LICENSE.md ├── README.md ├── c_src ├── .gitignore ├── Makefile ├── Makefile.win ├── sqlite3.c ├── sqlite3.h └── sqlite_nif.c ├── priv └── .gitignore ├── rebar.config ├── rebar.lock ├── src ├── sqlite.app.src └── sqlite.erl └── test ├── sqlite_SUITE.erl └── sqlite_bench_SUITE.erl /.github/workflows/erlang.yml: -------------------------------------------------------------------------------- 1 | name: Build, Test, Document, Dialyze 2 | 3 | on: 4 | pull_request: 5 | types: [ opened, reopened, synchronize ] 6 | push: 7 | branches: 8 | - 'master' 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | name: Erlang/OTP ${{matrix.otp}} / rebar3 ${{matrix.rebar3}} 14 | strategy: 15 | matrix: 16 | otp: ['24.3.4', '25.2.1', '26.1.2'] 17 | rebar3: ['3.20.0'] 18 | steps: 19 | - uses: actions/checkout@v3 20 | - uses: erlef/setup-beam@v1 21 | with: 22 | otp-version: ${{matrix.otp}} 23 | rebar3-version: ${{matrix.rebar3}} 24 | - name: Run tests 25 | run: rebar3 ct 26 | - name: ExDoc Documentation 27 | run: if [ $(rebar3 version | awk '{print $5}') -gt 23 ]; then rebar3 ex_doc; fi; 28 | - shell: bash 29 | name: Dialyzer 30 | run: rebar3 dialyzer 31 | 32 | # Windows CI disabled: nmake is missing from setup-beam@v1 33 | # windows: 34 | # name: Test on Windows 35 | # runs-on: windows-2022 36 | # steps: 37 | # - uses: actions/checkout@v3 38 | # - uses: erlef/setup-beam@v1 39 | # with: 40 | # otp-version: '24' 41 | # rebar3-version: '3.16.1' 42 | # - run: rebar3 ct 43 | 44 | macos: 45 | name: Test on MacOS 46 | runs-on: macos-latest 47 | 48 | steps: 49 | - uses: actions/checkout@v3 50 | - name: Install Erlang 51 | run: brew install erlang rebar3 52 | - name: Run tests 53 | run: rebar3 ct 54 | 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | erl_crash.dump 3 | rebar3.crashdump 4 | .idea 5 | *.iml 6 | .vscode 7 | doc 8 | perf.data -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Maxim Fedorov 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 19 | THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 20 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 22 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 23 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 26 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 27 | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 | POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqlite 2 | 3 | [![Build Status](https://github.com/max-au/sqlite/actions/workflows/erlang.yml/badge.svg?branch=master)](https://github.com/max-au/sqlite/actions) [![Hex.pm](https://img.shields.io/hexpm/v/sqlite.svg)](https://hex.pm/packages/sqlite) [![Hex Docs](https://img.shields.io/badge/hex-docs-blue.svg)](https://hexdocs.pm/sqlite) 4 | 5 | sqlite3 NIF bindings for Erlang. 6 | 7 | See `sqlite:query/3` for the data type conversion details. Current version 8 | assumes UTF8 encoding for all TEXT columns. 9 | 10 | All functions that may affect scheduling run in the dirty I/O scheduler. 11 | Keep an eye on the dirty scheduler utilisation, and bump the number when 12 | necessary (using `+SDio` argument). 13 | 14 | ## Build 15 | Use `rebar3 compile` to build the application using included SQLite3 16 | amalgamation. Specify `USE_SYSTEM_SQLITE` environment variable to 17 | build against sqlite version provided by the system (tested with 18 | sqlite 3.37.2 and newer). 19 | 20 | sqlite3 amalgamation build flags are accepted through `CFLAGS`, e.g. 21 | `CFLAGS="-DSQLITE_USE_URI" rebar3 compile`. 22 | 23 | For a debug build, specify `DEBUG=1` environment variable. Both 24 | NIF bindings and sqlite amalgamation files are affected by this flag. Remember 25 | to run `rebar3 clean` when changing build flavour. 26 | 27 | Requires OTP 24 and above (older versions do not support extended error 28 | specification per EEP-54). Find a list of supported Operating Systems in 29 | the corresponding section. 30 | 31 | ## Quick start 32 | The easiest way to try it out is to run `rebar3 shell` within the cloned 33 | project folder. 34 | 35 | ``` 36 | 1> Conn = sqlite:open("mem", #{flags => [memory]}). %% open an in-memory database 37 | #Ref<0.1238832725.87949353.240438> 38 | 2> sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)"). 39 | [] 40 | 3> Prepared = sqlite:prepare(Conn, "INSERT INTO kv (val) VALUES (?1)", #{persistent => true}). 41 | #Ref<0.1238832725.87949353.240439> 42 | 4> [sqlite:execute(Prepared, [Seq]) || Seq <- lists:seq(1, 100)]. 43 | [[],[],[],[],[],[]|...] 44 | 5> sqlite:query(Conn, "SELECT * FROM kv"). 45 | [{1,1}, {2,2}, <...> 46 | ``` 47 | 48 | ## Configuration 49 | Due to significant performance difference observed for in-memory databases, 50 | there is an option to fall back to sqlite built-in memory allocation 51 | functions. Set `enif_malloc` configuration variable to `false` (the default 52 | is `true`) to let sqlite use built-in allocators instead of the recommended 53 | `enif_alloc`. As a consequence, memory allocated by sqlite3 will no longer 54 | be visible via usual Erlang functions. For example `erlang:memory/1` won't be 55 | able to report sqlite3 allocations). Use this option only when you need that 56 | last bit of performance. 57 | 58 | ## Profiling with Linux perf 59 | 60 | Ensure ERTS with frame pointers enabled (`frmptr` flavour) is available. 61 | 62 | ```bash 63 | export ERL_FLAGS="+JPperf true -emu_type frmptr" 64 | export CFLAGS="-fno-omit-frame-pointer"" 65 | 66 | # run the benchmark test case 67 | rebar3 ct --suite sqlite_QUITE --case benchmark_prepared 68 | 69 | # take 10 second of perf profiling 70 | perf record -g -p `pidof rebar3` -F 500 -- sleep 10 71 | 72 | # convert to a flame graph 73 | perf script | ./stackcollapse-perf.pl > out.perf-folded 74 | ./flamegraph.pl out.perf-folded > perf.svg 75 | ``` 76 | 77 | ## Operating system support 78 | Initial version has been tested on Linux (Ubuntu), Mac OS, FreeBSD 13 and 79 | Windows 10. 80 | 81 | ### Mac OS 82 | If you see Apple Silicon (ARM64) build failing with this diagnostic: 83 | ``` 84 | Undefined symbols for architecture arm64: 85 | "_enif_alloc", referenced from: 86 | ``` 87 | Clear the `LDFLAGS` variable with `unset LDFLAGS` and try again. 88 | 89 | ### Windows 90 | Windows build, with `rebar3` as an escript, assuming Ericsson official installation 91 | into `C:\Program Files\Erlang OTP`. If you have it elsewhere, set `ERTS_INCLUDE_DIR` 92 | environment variable appropriately, to the `usr/include` subfolder of your installation. 93 | 94 | Build requires `x64 Native Tools Command Prompt`. Tested with VS 2022. Ensure 95 | to see the slogan: `[vcvarsall.bat] Environment initialized for: 'x64'` 96 | 97 | ``` 98 | # set up PATH to escript/erl 99 | set PATH=%PATH%;C:\Program Files\Erlang OTP\bin 100 | # download or build rebar3 (assuming it is now in the current directory) 101 | # use rebar3 to build sqlite and run Common Test 102 | escript.exe rebar3 ct 103 | ``` 104 | 105 | ## Missing Features for 1.0 106 | * performance: use "trylock" and then reschedule instead of dirty NIF from the beginning 107 | * performance: transparently handle BUSY and LOCKED instead of busy wait in dirty scheduler 108 | * asynchronous APIs (using message exchange, e.g. for `query` and `execute`) 109 | * diagnostic routines to enumerate prepared statements/statements running 110 | 111 | ## Features beyond 1.0 112 | These features will not make it into 1.0, but are useful and may be implemented 113 | in the following releases: 114 | * non-experimental sqlite hooks support (commit, preupdate, rollback, update, wal) 115 | * sqlite snapshot, vfs, blob and serialization support 116 | * improved performance for concurrent access to the same connection/statement, 117 | running on a normal scheduler and yielding 118 | * support for atom() and map() type conversion 119 | * named parameters with '@' and Tcl syntax -------------------------------------------------------------------------------- /c_src/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | -------------------------------------------------------------------------------- /c_src/Makefile: -------------------------------------------------------------------------------- 1 | # General variables 2 | C_SRC_DIR := $(shell pwd) 3 | PROJECT_DIR := $(abspath $(C_SRC_DIR)/..) 4 | 5 | # Figure out Unix-specific details: library extension 6 | TARGET := $(PROJECT_DIR)/priv/sqlite.so 7 | UNAME_SYS := $(shell uname -s) 8 | 9 | ifeq ($(UNAME_SYS), Darwin) 10 | CC ?= cc 11 | LDFLAGS ?= -undefined dynamic_lookup 12 | else ifeq ($(UNAME_SYS), FreeBSD) 13 | CC ?= cc 14 | else ifeq ($(UNAME_SYS), Linux) 15 | CC ?= gcc 16 | endif 17 | 18 | # Erlang NIF includes 19 | ERTS_INCLUDE_DIR = $(shell erl -noshell -eval "io:format(\"~ts/erts-~ts/include/\", [code:root_dir(), erlang:system_info(version)])." -s init stop) 20 | 21 | CFLAGS += -std=c99 -finline-functions -Wall -Wmissing-prototypes -fPIC -I $(ERTS_INCLUDE_DIR) 22 | 23 | # Set up DEBUG flags if DEBUG environment variable is set 24 | # The "variable trick" below allows using "DEBUG=1" 25 | ifdef DEBUG 26 | CFLAGS += -O0 -g3 -fno-omit-frame-pointer -DSQLITE_DEBUG 27 | else 28 | CFLAGS += -O3 29 | endif 30 | 31 | # Always build a shared binary (so, dll) 32 | LDFLAGS += -shared 33 | 34 | # Sources and object files in C_SRC 35 | #SRC := $(shell find $(C_SRC_DIR) -type f \( -name "*.c" \)) 36 | ifdef USE_SYSTEM_SQLITE 37 | SRC := ${C_SRC_DIR}/sqlite_nif.c 38 | CFLAGS += -DUSE_SYSTEM_SQLITE 39 | LDLIBS += -lsqlite3 40 | else 41 | # SQLite amalgamation is a part of this project, but it's possible 42 | # that someone wants to use the system-provided one 43 | SRC := ${C_SRC_DIR}/sqlite_nif.c ${C_SRC_DIR}/sqlite3.c 44 | endif 45 | 46 | OBJ = $(addsuffix .o, $(basename $(SRC))) 47 | 48 | # Disable implicit rules, they aren't helpful for this Makefile, polluting debug output 49 | .SUFFIXES: 50 | 51 | $(TARGET): $(OBJ) 52 | $(CC) $(OBJ) $(LDFLAGS) $(LDLIBS) -o $(TARGET) 53 | 54 | %.o: %.c 55 | $(CC) $(CFLAGS) -o $@ -c $< 56 | 57 | .PHONY: clean 58 | 59 | clean: 60 | @rm -f $(OBJ) $(TARGET) 61 | -------------------------------------------------------------------------------- /c_src/Makefile.win: -------------------------------------------------------------------------------- 1 | LDFLAGS = /DLL 2 | 3 | !if "$(DEBUG)" == "1" 4 | CFLAGS= $(CFLAGS) /MDd 5 | !else 6 | CFLAGS= $(CFLAGS) /MD /O2 7 | !endif 8 | 9 | C_SRC = c_src 10 | TARGET = priv\sqlite.dll 11 | 12 | !ifndef ERTS_INCLUDE_DIR 13 | ERTS_INCLUDE_DIR = C:\Program Files\Erlang OTP\usr\include 14 | !endif 15 | 16 | CFLAGS = $(CFLAGS) /I"$(ERTS_INCLUDE_DIR)" 17 | 18 | # Do not use system-provided sqlite on Windows 19 | OBJ = $(C_SRC)\sqlite3.obj $(C_SRC)\sqlite_nif.obj 20 | 21 | all: $(TARGET) 22 | 23 | $(TARGET): $(OBJ) 24 | link $(LDFLAGS) $** /OUT:$@ 25 | 26 | .c.obj: 27 | cl /c $(CFLAGS) /Fo$*.obj $*.c 28 | 29 | 30 | clean: 31 | @del /f /q $(C_SRC)\*.obj 32 | @del /f /q priv\sqlite.* 33 | -------------------------------------------------------------------------------- /c_src/sqlite_nif.c: -------------------------------------------------------------------------------- 1 | /* 2 | * TODO: cleanup code from "memory assertions" - handle OOM gracefully 3 | */ 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #ifdef USE_SYSTEM_SQLITE 10 | #include 11 | #else 12 | #include "./sqlite3.h" 13 | #endif 14 | 15 | #if SQLITE_VERSION_NUMBER < 3038000 16 | static int sqlite3_error_offset(sqlite3 *db) {return -1;} 17 | #endif 18 | 19 | 20 | /* Not imported from erl_nif.h */ 21 | #define THE_NON_VALUE 0 22 | 23 | /* Assertion: import from ERTS */ 24 | #ifdef DEBUG 25 | void erl_assert_error(const char* expr, const char* func, const char* file, int line); 26 | # define ASSERT(e) ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0))) 27 | #else 28 | # define ASSERT(e) ((void) (e)) 29 | #endif 30 | 31 | 32 | /* Atom definitions (atoms cannot change dynamically) */ 33 | static ERL_NIF_TERM am_ok; 34 | static ERL_NIF_TERM am_error; 35 | static ERL_NIF_TERM am_true; 36 | static ERL_NIF_TERM am_false; 37 | static ERL_NIF_TERM am_general; 38 | static ERL_NIF_TERM am_reason; 39 | static ERL_NIF_TERM am_badarg; 40 | static ERL_NIF_TERM am_undefined; 41 | static ERL_NIF_TERM am_position; 42 | static ERL_NIF_TERM am_out_of_memory; /* out of memory */ 43 | static ERL_NIF_TERM am_sqlite_error; /* sqlite-specific error code */ 44 | static ERL_NIF_TERM am_blob; 45 | static ERL_NIF_TERM am_done; 46 | static ERL_NIF_TERM am_busy; 47 | 48 | /* database open modes */ 49 | static ERL_NIF_TERM am_mode; 50 | static ERL_NIF_TERM am_flags; 51 | static ERL_NIF_TERM am_uri; 52 | static ERL_NIF_TERM am_read_only; 53 | static ERL_NIF_TERM am_read_write; 54 | static ERL_NIF_TERM am_read_write_create; 55 | static ERL_NIF_TERM am_memory; 56 | static ERL_NIF_TERM am_shared; 57 | static ERL_NIF_TERM am_busy_timeout; 58 | 59 | static ERL_NIF_TERM am_persistent; 60 | static ERL_NIF_TERM am_no_vtab; 61 | 62 | /* atoms for notifications */ 63 | static ERL_NIF_TERM am_insert; 64 | static ERL_NIF_TERM am_update; 65 | static ERL_NIF_TERM am_delete; 66 | 67 | /* connection status atoms */ 68 | static ERL_NIF_TERM am_lookaside_memory; 69 | static ERL_NIF_TERM am_pager_cache_memory; 70 | static ERL_NIF_TERM am_schema; 71 | static ERL_NIF_TERM am_statement; 72 | static ERL_NIF_TERM am_deferred_fks; 73 | static ERL_NIF_TERM am_used; 74 | static ERL_NIF_TERM am_max; 75 | static ERL_NIF_TERM am_hit; 76 | static ERL_NIF_TERM am_miss_size; 77 | static ERL_NIF_TERM am_miss_full; 78 | static ERL_NIF_TERM am_shared; 79 | static ERL_NIF_TERM am_miss; 80 | static ERL_NIF_TERM am_write; 81 | static ERL_NIF_TERM am_spill; 82 | 83 | /* statement info atoms */ 84 | static ERL_NIF_TERM am_fullscan_step; 85 | static ERL_NIF_TERM am_sort; 86 | static ERL_NIF_TERM am_autoindex; 87 | static ERL_NIF_TERM am_vm_step; 88 | static ERL_NIF_TERM am_reprepare; 89 | static ERL_NIF_TERM am_run; 90 | static ERL_NIF_TERM am_filter_miss; 91 | static ERL_NIF_TERM am_filter_hit; 92 | static ERL_NIF_TERM am_memory_used; 93 | 94 | /* system info atoms */ 95 | static ERL_NIF_TERM am_page_cache; 96 | static ERL_NIF_TERM am_malloc; 97 | static ERL_NIF_TERM am_version; 98 | 99 | /* forward definitions (linked list) */ 100 | typedef struct connection_t connection_t; 101 | typedef struct statement_t statement_t; 102 | typedef struct delayed_close_t delayed_close_t; 103 | 104 | /* prepared statement: stored as a part of the connection */ 105 | struct statement_t { 106 | sqlite3_stmt* statement; /* sqlite3 handle */ 107 | statement_t* next; /* next statement in the overall list */ 108 | statement_t* released; /* next released statement that can be reused */ 109 | }; 110 | 111 | /* This structure has pointers from the backup resource, source and 112 | destination connections. In order to change any member of the structure, 113 | both source and destination mutexes must be help. The order is important: 114 | to avoid a deadlock, source must be taken first. 115 | */ 116 | typedef struct { 117 | sqlite3_backup* backup; 118 | connection_t* source; 119 | connection_t* destination; 120 | } backup_t; 121 | 122 | struct connection_t { 123 | ErlNifMutex* mutex; /* used to protect the entire structure from concurrent access */ 124 | sqlite3* connection; /* sqlite3 handle */ 125 | ErlNifPid tracer; /* process to send ROWID updates */ 126 | ERL_NIF_TERM ref; /* sent to the tracer process */ 127 | 128 | backup_t* source; /* references backup from the source */ 129 | backup_t* destination; /* references backup object from the destination, prohibiting all access*/ 130 | 131 | ErlNifMutex* stmt_mutex; /* mutex protecting statement lists */ 132 | statement_t* statements; /* list of all statements for this connection */ 133 | statement_t* released; /* head of the 'released statements sublist' */ 134 | }; 135 | 136 | /* resources: connection and connection-to-deallocate */ 137 | static ErlNifResourceType *s_connection_resource; 138 | /* delayed deallocation process */ 139 | static ErlNifPid s_delayed_close_pid; 140 | 141 | struct delayed_close_t { 142 | sqlite3* connection; 143 | statement_t* statements; /* list of all statements for this connection */ 144 | backup_t* source; 145 | backup_t* destination; 146 | delayed_close_t* next; 147 | }; 148 | 149 | /* queue of connections to delete */ 150 | static delayed_close_t* s_delayed_close_queue; 151 | /* mutex protecting the queue (could be done lockless, but there is no sense in it) */ 152 | static ErlNifMutex* s_delayed_close_mutex; 153 | 154 | /* forward declaration, to emergency use in the destructor */ 155 | static ERL_NIF_TERM sqlite_dirty_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); 156 | 157 | /* Connection destructor. 158 | It may be executed in the context of the normal scheduler, while doing GC, 159 | so it must not perform any long-running activities. 160 | Closing sqlite database may take any amount of time, hence it must be 161 | running by a dirty I/O scheduler. 162 | To achieve that, the on_load function of a NIF starts a new process that 163 | keeps running only for the sake of executing a (hidden from the official API) 164 | dirty I/O NIF that actually does the cleanup. Here, in the destructor, 165 | no clean-up happens, only the message gets sent to the process. If the process 166 | does not exist, resource leak happens. 167 | */ 168 | static void sqlite_connection_destroy(ErlNifEnv *env, void *arg) { 169 | connection_t *conn = (connection_t*)arg; 170 | 171 | /* there is no need in locking the connection structure, it's guaranteed 172 | to be the very last instance */ 173 | if (conn->connection) { 174 | /* can get here only if no explicit close/1 call was successful */ 175 | delayed_close_t* node = enif_alloc(sizeof(delayed_close_t)); 176 | node->connection = conn->connection; 177 | node->statements = conn->statements; 178 | node->source = conn->source; 179 | node->destination = conn->destination; 180 | enif_mutex_lock(s_delayed_close_mutex); 181 | node->next = s_delayed_close_queue; 182 | s_delayed_close_queue = node; 183 | enif_mutex_unlock(s_delayed_close_mutex); 184 | /* notify garbage collection process */ 185 | if (!enif_send(env, &s_delayed_close_pid, NULL, am_undefined)) { 186 | enif_fprintf(stderr, "sqlite connection deallocated during GC"); 187 | /* last resort, cleanup resources locally */ 188 | sqlite_dirty_close_nif(env, 0, NULL); 189 | } 190 | } else { 191 | /* No need to schedule a dirty NIF, the connection was already closed. 192 | At this point there are no outstanding statements, and no referenced 193 | statements either - so simply free the entire statements list. */ 194 | statement_t* stmt = conn->statements; 195 | statement_t* freed; 196 | while (stmt) { 197 | ASSERT(stmt->statement == NULL); 198 | freed = stmt; 199 | stmt = stmt->next; 200 | enif_free(freed); 201 | } 202 | 203 | /* backup has also been released */ 204 | ASSERT(conn->source == NULL); 205 | ASSERT(conn->destination == NULL); 206 | } 207 | 208 | enif_mutex_destroy(conn->stmt_mutex); 209 | enif_mutex_destroy(conn->mutex); 210 | } 211 | 212 | /* Prepared statement resource. Has a weak link to the connection it belongs to. 213 | If the connection is closed (either implicitly via GC, or explicitly 214 | via close/1 call), all prepared statements are no longer valid. 215 | */ 216 | static ErlNifResourceType *s_statement_resource; 217 | 218 | typedef struct { 219 | connection_t* connection; 220 | statement_t* reference; 221 | } statement_resource_t; 222 | 223 | /* 224 | Prepared statement management is tricky. Any operation with the statement 225 | (prepare, finalize, or step) requires exclusive connection object ownership. 226 | Therefore all these operations must happen within a dirty NIF, and not 227 | as a part of a normal scheduler. When GC collects a statement, instead of 228 | finalising it in place, just mark it 'released'. GC callback cannot take 229 | the connection mutex! As otherwise it'd be stuck behind dirty NIFs, leading 230 | to a deadlock. 231 | 232 | Using SERIALIZED sqlite mode does not help: it simply wraps all operations 233 | (including sqlite3_bind_xxx) with connection mutex. In theory it allows 234 | multiple NIFs to interleave argument binding, but in practice it's not 235 | recommended doing that anyway. 236 | */ 237 | 238 | static void sqlite_statement_destroy(ErlNifEnv *env, void *arg) { 239 | statement_resource_t *stmt = (statement_resource_t*)arg; 240 | /* the following condition is needed for prepare_nif to reduce lock duration */ 241 | if (stmt->connection) { 242 | enif_mutex_lock(stmt->connection->stmt_mutex); 243 | stmt->reference->released = stmt->connection->released; 244 | stmt->connection->released = stmt->reference; 245 | enif_mutex_unlock(stmt->connection->stmt_mutex); 246 | enif_release_resource(stmt->connection); 247 | } 248 | } 249 | 250 | /* Backup resource. Jumps through similar hoops as the prepared statement to 251 | avoid blocking in the GC callback, but now with 2 connections. 252 | */ 253 | static ErlNifResourceType *s_backup_resource; 254 | 255 | typedef struct backup_finish_t backup_finish_t; 256 | 257 | struct backup_finish_t { 258 | backup_t* unfinished; 259 | backup_finish_t* next; 260 | }; 261 | 262 | /* queue of backups to finish */ 263 | static backup_finish_t* s_backup_finish_queue; 264 | /* mutex protecting delayed backup finish queue */ 265 | static ErlNifMutex* s_backup_finish_mutex; 266 | 267 | typedef struct { 268 | backup_t* reference; 269 | } backup_resource_t; 270 | 271 | /* forward declaration for the dirty NIF */ 272 | static ERL_NIF_TERM sqlite_dirty_backup_finish_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); 273 | 274 | /* must not take connection mutex */ 275 | static void sqlite_backup_destroy(ErlNifEnv *env, void *arg) { 276 | backup_resource_t *backup_res = (backup_resource_t*)arg; 277 | if (backup_res->reference->backup) { 278 | backup_finish_t* fini = (backup_finish_t*)enif_alloc(sizeof(backup_finish_t)); 279 | /* backup process crashed without calling finish */ 280 | enif_mutex_lock(s_backup_finish_mutex); 281 | fini->unfinished = backup_res->reference; 282 | fini->next = s_backup_finish_queue; 283 | s_backup_finish_queue = fini; 284 | enif_mutex_unlock(s_backup_finish_mutex); 285 | /* notify garbage collection process */ 286 | if (!enif_send(env, &s_delayed_close_pid, NULL, am_general)) { 287 | enif_fprintf(stderr, "backup resource deallocating in a normal scheduler"); 288 | /* last resort, cleanup resources locally */ 289 | sqlite_dirty_backup_finish_nif(env, 0, NULL); 290 | } 291 | } else { 292 | enif_release_resource(backup_res->reference->source); 293 | enif_release_resource(backup_res->reference->destination); 294 | enif_free(backup_res->reference); 295 | } 296 | } 297 | 298 | /* -------------------------------------------------------------------------------- */ 299 | /* Helper functions (creating binaries and errors) */ 300 | static ERL_NIF_TERM make_binary(ErlNifEnv *env, const char* str, unsigned int len) { 301 | ErlNifBinary bin; 302 | if (!enif_alloc_binary(len, &bin)) 303 | return THE_NON_VALUE; 304 | memcpy(bin.data, str, len); 305 | ERL_NIF_TERM bin_term = enif_make_binary(env, &bin); 306 | enif_release_binary(&bin); 307 | return bin_term; 308 | } 309 | 310 | /* make a NULL-terminated C string from Erlang binary */ 311 | static const char* binary_to_string(char* to, ErlNifBinary bin, int len) { 312 | int max = bin.size < len - 1 ? bin.size : len - 1; 313 | memcpy(to, bin.data, max); 314 | to[max] = 0; 315 | return to; 316 | } 317 | 318 | /* Creates a tuple to simulate EEP-54 exceptions in the Erlang code). 319 | - reason: the "old" exception reason (e.g. "badarg") 320 | - general: the "general" string in the "cause" of the extended error info 321 | - [reason_text]: optional pre-formatted reason string 322 | - [arg_pos]: optional argument number (when the exception is caused by badarg) 323 | - [arg_error]: optional, value of the arg_pos key in the "cause" map 324 | - [position]: optional, column (for SQL statements) or bound parameter index 325 | */ 326 | static ERL_NIF_TERM make_extended_error_ex(ErlNifEnv *env, ERL_NIF_TERM reason, const char* general, 327 | int arg_pos, const char* arg_error, const char* reason_text, int position) { 328 | /* create the new 'cause' map */ 329 | ERL_NIF_TERM error_info_map = enif_make_new_map(env); 330 | 331 | if (general) 332 | enif_make_map_put(env, error_info_map, am_general, 333 | make_binary(env, general, strlen(general)), &error_info_map); 334 | 335 | if (reason_text) 336 | enif_make_map_put(env, error_info_map, am_reason, 337 | make_binary(env, reason_text, strlen(reason_text)), &error_info_map); 338 | 339 | if (arg_pos > 0 && arg_error) 340 | enif_make_map_put(env, error_info_map, enif_make_int(env, arg_pos), 341 | make_binary(env, arg_error, strlen(arg_error)), &error_info_map); 342 | 343 | if (position >= 0) 344 | enif_make_map_put(env, error_info_map, am_position, enif_make_int(env, position), &error_info_map); 345 | 346 | return enif_make_tuple3(env, am_error, reason, error_info_map); 347 | } 348 | 349 | /* Shortcut for badarg */ 350 | static ERL_NIF_TERM make_badarg(ErlNifEnv *env, const char* general, int arg_pos, const char* arg_error) { 351 | return make_extended_error_ex(env, am_badarg, general, arg_pos, arg_error, NULL, -1); 352 | } 353 | 354 | /* Reason is always {sqlite_error, Code}, general is always generic 355 | error message. When argument index is known, it contains the detailed 356 | error description. When argument index is now know, pre-formatted 357 | reason contains the detailed message. */ 358 | static ERL_NIF_TERM make_sql_error(ErlNifEnv *env, sqlite3* conn, int sqlite_error, int arg_pos) { 359 | ERL_NIF_TERM reason = enif_make_tuple2(env, am_sqlite_error, enif_make_int(env, sqlite_error)); 360 | const char *errmsg = sqlite3_errmsg(conn); 361 | const char *errstr = sqlite3_errstr(sqlite_error); 362 | if (arg_pos > 0) 363 | return make_extended_error_ex(env, reason, errstr, arg_pos, errmsg, NULL, sqlite3_error_offset(conn)); 364 | else 365 | return make_extended_error_ex(env, reason, errstr, -1, NULL, errmsg, -1); 366 | } 367 | 368 | /* -------------------------------------------------------------------------------- */ 369 | /* Connection (opens the database) */ 370 | static ERL_NIF_TERM sqlite_open_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 371 | { 372 | char *filename; 373 | ErlNifBinary fn_bin; 374 | ERL_NIF_TERM mode, flag; 375 | int v2flags = 0; 376 | ErlNifSInt64 busy_timeout = -1; 377 | 378 | /* type checks */ 379 | if (!enif_is_map(env, argv[1])) 380 | return make_badarg(env, NULL, 2, "not a map"); 381 | 382 | /* flags specification */ 383 | if (enif_get_map_value(env, argv[1], am_flags, &mode)) { 384 | while (!enif_is_empty_list(env, mode)) { 385 | if (!enif_get_list_cell(env, mode, &flag, &mode)) 386 | return make_badarg(env, NULL, 2, "invalid flags"); 387 | if (flag == am_memory) 388 | v2flags |= SQLITE_OPEN_MEMORY; 389 | else if (flag == am_uri) 390 | v2flags |= SQLITE_OPEN_URI; 391 | else if (flag == am_shared) 392 | v2flags |= SQLITE_OPEN_SHAREDCACHE; 393 | else 394 | return make_badarg(env, NULL, 2, "invalid flag"); 395 | } 396 | } 397 | 398 | /* opening mode */ 399 | if (enif_get_map_value(env, argv[1], am_mode, &mode)) { 400 | if (mode == am_read_only) 401 | v2flags |= SQLITE_OPEN_READONLY; 402 | else if (mode == am_read_write) 403 | v2flags |= SQLITE_OPEN_READWRITE; 404 | else if (mode == am_read_write_create) 405 | v2flags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; 406 | else 407 | return make_badarg(env, NULL, 2, "unsupported mode"); 408 | } else /* default mode */ 409 | v2flags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; 410 | 411 | v2flags |= SQLITE_OPEN_EXRESCODE; /* always want extended errors */ 412 | 413 | if (enif_get_map_value(env, argv[1], am_busy_timeout, &mode)) { 414 | if (!enif_get_int64(env, mode, &busy_timeout) || busy_timeout < 0) 415 | return make_badarg(env, NULL, 2, "invalid busy timeout"); 416 | } 417 | 418 | /* accept filename as iolist */ 419 | if (enif_inspect_iolist_as_binary(env, argv[0], &fn_bin)) { 420 | /* Erlang binaries are not NULL-terminated, so make a copy */ 421 | filename = enif_alloc(fn_bin.size + 1); 422 | if (!filename) 423 | return enif_raise_exception(env, am_out_of_memory); 424 | memcpy(filename, (const char*)fn_bin.data, fn_bin.size); 425 | filename[fn_bin.size] = 0; 426 | } else 427 | return make_badarg(env, NULL, 1, "invalid file name"); 428 | 429 | /* Open the database. */ 430 | sqlite3* sqlite; 431 | int ret = sqlite3_open_v2(filename, &sqlite, v2flags, NULL); 432 | enif_free(filename); 433 | if (ret != SQLITE_OK) { 434 | sqlite3_close(sqlite); /* non-v2 used because no statement could be there, and no NULL check needed */ 435 | return make_sql_error(env, sqlite, ret, -1); 436 | } 437 | 438 | /* set busy timeout */ 439 | if (busy_timeout != -1) 440 | sqlite3_busy_timeout(sqlite, busy_timeout); 441 | 442 | /* Initialize the resource */ 443 | connection_t* conn = enif_alloc_resource(s_connection_resource, sizeof(connection_t)); 444 | if (!conn) { 445 | sqlite3_close(sqlite); 446 | return enif_raise_exception(env, am_out_of_memory); 447 | } 448 | 449 | enif_set_pid_undefined(&conn->tracer); 450 | conn->connection = sqlite; 451 | conn->statements = NULL; 452 | conn->released = NULL; 453 | conn->source = NULL; 454 | conn->destination = NULL; 455 | conn->mutex = enif_mutex_create("sqlite3_connection"); 456 | ASSERT(conn->mutex); 457 | conn->stmt_mutex = enif_mutex_create("sqlite3_statement"); 458 | ASSERT(conn->stmt_mutex); 459 | 460 | ERL_NIF_TERM connection = enif_make_resource(env, conn); 461 | enif_release_resource(conn); 462 | 463 | return connection; 464 | } 465 | 466 | static ERL_NIF_TERM sqlite_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 467 | { 468 | connection_t *conn; 469 | sqlite3* sqlite_db; 470 | statement_t* open; 471 | 472 | if (!enif_get_resource(env, argv[0], s_connection_resource, (void **) &conn)) 473 | return make_badarg(env, NULL, 1, "not a connection reference"); 474 | 475 | enif_mutex_lock(conn->mutex); 476 | if (conn->connection) { 477 | sqlite_db = conn->connection; 478 | conn->connection = NULL; 479 | } else { 480 | enif_mutex_unlock(conn->mutex); 481 | return make_badarg(env, NULL, 1, "connection already closed"); 482 | } 483 | 484 | /* DO NOT take stmt_mutex, or it will make the destructor wait for connection mutex */ 485 | /* iterate over all statements, removing sqlite3_stmt */ 486 | open = conn->statements; 487 | while (open) { 488 | sqlite3_finalize(open->statement); 489 | open->statement = NULL; /* to avoid double-finalising */ 490 | open = open->next; 491 | } 492 | 493 | /* source connection can always take the destination mutex */ 494 | if (conn->source) { 495 | enif_mutex_lock(conn->source->destination->mutex); 496 | conn->source->destination->destination = NULL; 497 | sqlite3_backup_finish(conn->source->backup); 498 | conn->source->backup = NULL; 499 | enif_mutex_unlock(conn->source->destination->mutex); 500 | conn->source = NULL; 501 | } 502 | 503 | /* destination connection can try, but not necessarily succeed */ 504 | if (conn->destination) { 505 | if (enif_mutex_trylock(conn->destination->source->mutex) == 0) { 506 | conn->destination->source->source = NULL; 507 | sqlite3_backup_finish(conn->destination->backup); 508 | conn->destination->backup = NULL; 509 | enif_mutex_unlock(conn->destination->source->mutex); 510 | conn->destination = NULL; 511 | } else { 512 | /* potential deadlock: must take the mutexes starting 513 | from the backup source */ 514 | connection_t* backup_src = conn->destination->source; 515 | enif_mutex_unlock(conn->mutex); 516 | enif_mutex_lock(backup_src->mutex); 517 | enif_mutex_lock(conn->mutex); 518 | /* here backup object may already be cleaned up, so need to double-check */ 519 | if (backup_src->source) { 520 | sqlite3_backup_finish(conn->destination->backup); 521 | conn->destination->backup = NULL; 522 | backup_src->source = NULL; 523 | conn->destination = NULL; 524 | } 525 | enif_mutex_unlock(conn->mutex); 526 | enif_mutex_unlock(backup_src->mutex); 527 | } 528 | } 529 | 530 | /* close the connection, there must not be any statements left */ 531 | int ret = sqlite3_close(sqlite_db); 532 | ASSERT(ret == SQLITE_OK); 533 | 534 | enif_mutex_unlock(conn->mutex); 535 | 536 | return am_ok; 537 | } 538 | 539 | /* This function should only be called from a "cleanup" process that owns orphan connections 540 | * that weren't closed explicitly. Ideally it should not ever be called. */ 541 | static ERL_NIF_TERM sqlite_dirty_close_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 542 | { 543 | delayed_close_t* next; 544 | 545 | /* iterate over pending connections */ 546 | enif_mutex_lock(s_delayed_close_mutex); 547 | ASSERT(s_delayed_close_queue); 548 | next = s_delayed_close_queue; 549 | s_delayed_close_queue = next->next; 550 | enif_mutex_unlock(s_delayed_close_mutex); 551 | 552 | /* it's guaranteed that no statements are used here, so just finalize all of them */ 553 | statement_t* stmt = next->statements; 554 | statement_t* freed; 555 | while (stmt) { 556 | if (stmt->statement) 557 | sqlite3_finalize(stmt->statement); 558 | freed = stmt; 559 | stmt = stmt->next; 560 | enif_free(freed); 561 | } 562 | 563 | /* no backup can reference this connection */ 564 | ASSERT(next->source == NULL); 565 | ASSERT(next->destination == NULL); 566 | 567 | int ret = sqlite3_close(next->connection); 568 | ASSERT(ret == SQLITE_OK); 569 | enif_free(next); 570 | return am_ok; 571 | } 572 | 573 | static ERL_NIF_TERM bind_parameter(ErlNifEnv *env, sqlite3* conn, sqlite3_stmt* stmt, 574 | ERL_NIF_TERM param, int arg_index, int column, void(* lifetime)(void*)) { 575 | ErlNifSInt64 int_val; 576 | double double_val; 577 | ErlNifBinary bin; 578 | const ERL_NIF_TERM* tuple; 579 | int ret, arity; 580 | 581 | /* supported types: undefined (atom), integer, blob, text, float */ 582 | if (param == am_undefined) { 583 | /* NULL */ 584 | ret = sqlite3_bind_null(stmt, column); 585 | } else if (enif_inspect_iolist_as_binary(env, param, &bin)) { 586 | /* TEXT */ 587 | ret = sqlite3_bind_text64(stmt, column, (char*)bin.data, bin.size, lifetime, SQLITE_UTF8); 588 | } else if (enif_is_number(env, param)) { 589 | /* INTEGER or FLOAT */ 590 | if (enif_get_int64(env, param, &int_val)) 591 | ret = sqlite3_bind_int64(stmt, column, int_val); 592 | else if (enif_get_double(env, param, &double_val)) 593 | ret = sqlite3_bind_double(stmt, column, double_val); 594 | else 595 | return make_extended_error_ex(env, am_badarg, "failed to bind parameter", arg_index, "bignum not supported", 596 | NULL, column); 597 | } else if (enif_get_tuple(env, param, &arity, &tuple) && (tuple[0] == am_blob) && (enif_inspect_binary(env, tuple[1], &bin))) { 598 | ret = sqlite3_bind_blob64(stmt, column, (char*)bin.data, bin.size, lifetime); 599 | } else 600 | return make_extended_error_ex(env, am_badarg, "failed to bind parameter", arg_index, "unsupported type", 601 | NULL, column); 602 | 603 | if (ret != SQLITE_OK) { 604 | const char* errstr = sqlite3_errstr(ret); 605 | ERL_NIF_TERM reason = enif_make_tuple2(env, am_sqlite_error, enif_make_int(env, ret)); 606 | return make_extended_error_ex(env, reason, errstr, arg_index, sqlite3_errmsg(conn), 607 | NULL, column); 608 | } 609 | return THE_NON_VALUE; 610 | } 611 | 612 | /* bind & execute helpers */ 613 | static ERL_NIF_TERM bind(ErlNifEnv *env, sqlite3* conn, sqlite3_stmt* stmt, ERL_NIF_TERM params, 614 | int arg_index, void(* lifetime)(void*)) { 615 | ERL_NIF_TERM param, bind_result; 616 | int column = 1; 617 | int expected = sqlite3_bind_parameter_count(stmt); 618 | 619 | if (enif_is_map(env, params)) 620 | { 621 | ERL_NIF_TERM key; 622 | ErlNifMapIterator iter; 623 | ErlNifBinary bin; 624 | char name[258]; /* atom name is limited to 255 characters anyway */ 625 | name[0] = ':'; 626 | 627 | enif_map_iterator_create(env, params, &iter, ERL_NIF_MAP_ITERATOR_FIRST); 628 | 629 | while (enif_map_iterator_get_pair(env, &iter, &key, ¶m)) { 630 | if (!enif_get_int(env, key, &column)) { 631 | if (enif_is_atom(env, key)) { 632 | enif_get_atom(env, key, name + 1, sizeof(name) - 2, ERL_NIF_LATIN1); 633 | } else if (enif_inspect_iolist_as_binary(env, key, &bin)) { 634 | int len = bin.size < sizeof(name) - 2 ? bin.size : sizeof(name) - 2; 635 | strncpy(name + 1, (const char*)bin.data, len); 636 | name[len + 1] = 0; 637 | } else { 638 | bind_result = make_extended_error_ex(env, am_badarg, NULL, arg_index, 639 | "invalid parameter", NULL, -1); 640 | break; 641 | } 642 | 643 | column = sqlite3_bind_parameter_index(stmt, name); 644 | if (column == 0) { 645 | bind_result = make_extended_error_ex(env, am_badarg, name, arg_index, 646 | "parameter not found", NULL, -1); 647 | break; 648 | } 649 | } 650 | 651 | bind_result = bind_parameter(env, conn, stmt, param, arg_index, column, lifetime); 652 | if (bind_result != THE_NON_VALUE) 653 | break; 654 | 655 | enif_map_iterator_next(env, &iter); 656 | } 657 | enif_map_iterator_destroy(env, &iter); 658 | return bind_result; 659 | } else { 660 | /* bind parameters passed */ 661 | while (!enif_is_empty_list(env, params)) { 662 | if (!enif_get_list_cell(env, params, ¶m, ¶ms)) 663 | return make_extended_error_ex(env, am_badarg, NULL, arg_index, "invalid parameters list", NULL, column); 664 | 665 | bind_result = bind_parameter(env, conn, stmt, param, arg_index, column, lifetime); 666 | if (bind_result != THE_NON_VALUE) 667 | return bind_result; 668 | 669 | column++; 670 | } 671 | 672 | if (column <= expected) 673 | return make_badarg(env, NULL, 2, "not enough parameters"); 674 | } 675 | 676 | return THE_NON_VALUE; 677 | } 678 | 679 | static ERL_NIF_TERM fetch_row(ErlNifEnv *env, sqlite3_stmt* stmt, int column_count, ERL_NIF_TERM values[]) { 680 | int column_type; 681 | unsigned int size; 682 | ErlNifSInt64 int_val; 683 | double double_val; 684 | for (int i=0; i 512) 724 | row = (ERL_NIF_TERM*)enif_alloc(column_count * sizeof(ERL_NIF_TERM)); 725 | else 726 | row = &values_buffer[0]; 727 | 728 | int ret = sqlite3_step(stmt); 729 | switch (ret) { 730 | case SQLITE_DONE: 731 | break; 732 | case SQLITE_ROW: 733 | do { 734 | next_row = fetch_row(env, stmt, column_count, row); 735 | result = enif_make_list_cell(env, next_row, result); 736 | ret = sqlite3_step(stmt); 737 | } while (ret == SQLITE_ROW); 738 | /* check if it was error that bailed out */ 739 | if (ret != SQLITE_DONE) 740 | result = make_sql_error(env, conn, ret, -1); 741 | break; 742 | default: 743 | result = make_sql_error(env, conn, ret, -1); 744 | break; 745 | } 746 | 747 | if (column_count > COLUMNS_ON_STACK) 748 | enif_free(row); 749 | 750 | return result; 751 | } 752 | 753 | /* Query preparation */ 754 | static ERL_NIF_TERM sqlite_prepare_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 755 | { 756 | connection_t* conn; 757 | if (!enif_get_resource(env, argv[0], s_connection_resource, (void **) &conn)) 758 | return make_badarg(env, NULL, 1, "not a connection reference"); 759 | 760 | ErlNifBinary query_bin; 761 | if (!enif_inspect_iolist_as_binary(env, argv[1], &query_bin)) 762 | return make_badarg(env, NULL, 2, "not an iolist"); 763 | 764 | /* type checks */ 765 | if (!enif_is_map(env, argv[2])) 766 | return make_badarg(env, NULL, 3, "not a map"); 767 | 768 | ERL_NIF_TERM flag; 769 | int prep_flags = 0; 770 | if (enif_get_map_value(env, argv[2], am_persistent, &flag)) { 771 | if (flag == am_true) 772 | prep_flags |= SQLITE_PREPARE_PERSISTENT; 773 | else if (flag != am_false) 774 | return make_badarg(env, NULL, 3, "invalid persistent value"); 775 | } 776 | 777 | if (enif_get_map_value(env, argv[2], am_no_vtab, &flag)) { 778 | if (flag == am_true) 779 | prep_flags |= SQLITE_PREPARE_NO_VTAB; 780 | else if (flag != am_false) 781 | return make_badarg(env, NULL, 3, "invalid no_vtab value"); 782 | } 783 | 784 | /* Initialize the resource */ 785 | statement_resource_t* stmt_res = enif_alloc_resource(s_statement_resource, sizeof(statement_resource_t)); 786 | ASSERT(stmt_res); 787 | stmt_res->connection = NULL; 788 | 789 | sqlite3_stmt* stmt; 790 | 791 | enif_mutex_lock(conn->mutex); 792 | if (!conn->connection) { 793 | enif_mutex_unlock(conn->mutex); 794 | enif_release_resource(stmt_res); 795 | return make_badarg(env, NULL, 1, "connection closed"); 796 | } 797 | 798 | if (conn->destination) { 799 | enif_mutex_unlock(conn->mutex); 800 | enif_release_resource(stmt_res); 801 | return make_badarg(env, NULL, 1, "connection busy (backup destination)"); 802 | } 803 | 804 | const unsigned char* tail; 805 | int ret = sqlite3_prepare_v3(conn->connection, (char*)query_bin.data, query_bin.size, prep_flags, &stmt, (const char**)&tail); 806 | if (ret != SQLITE_OK) { 807 | enif_mutex_unlock(conn->mutex); 808 | enif_release_resource(stmt_res); 809 | return make_sql_error(env, conn->connection, ret, 2); 810 | } 811 | 812 | if ((tail - query_bin.data) < query_bin.size) { 813 | enif_mutex_unlock(conn->mutex); 814 | enif_release_resource(stmt_res); 815 | return make_extended_error_ex(env, am_badarg, NULL, 2, "multiple statements are not supported", NULL, 816 | tail - query_bin.data); 817 | } 818 | 819 | statement_t* statement; 820 | enif_mutex_lock(conn->stmt_mutex); 821 | if (conn->released) { 822 | /* reuse a statement that was allocated before and now unused*/ 823 | statement = conn->released; 824 | if (statement->statement) /* could have been released by explicit finish/1 call */ 825 | sqlite3_finalize(statement->statement); 826 | conn->released = statement->released; 827 | } else { 828 | /* create a new statement reference */ 829 | statement = enif_alloc(sizeof(statement_t)); 830 | ASSERT(statement); 831 | statement->next = conn->statements; 832 | conn->statements = statement; 833 | } 834 | 835 | statement->released = NULL; 836 | statement->statement = stmt; 837 | 838 | enif_mutex_unlock(conn->stmt_mutex); 839 | 840 | enif_mutex_unlock(conn->mutex); 841 | 842 | enif_keep_resource(conn); 843 | 844 | stmt_res->reference = statement; 845 | stmt_res->connection = conn; 846 | 847 | ERL_NIF_TERM stmt_res_term = enif_make_resource(env, stmt_res); 848 | enif_release_resource(stmt_res); 849 | 850 | return stmt_res_term; 851 | } 852 | 853 | /* Binds parameters of the prepared statement */ 854 | static ERL_NIF_TERM sqlite_bind_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 855 | { 856 | statement_resource_t* stmt; 857 | 858 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt)) 859 | return make_badarg(env, NULL, 1, "not a prepared statement"); 860 | 861 | enif_mutex_lock(stmt->connection->mutex); 862 | if (!stmt->connection->connection) { 863 | enif_mutex_unlock(stmt->connection->mutex); 864 | return make_badarg(env, NULL, 1, "connection closed"); 865 | } 866 | 867 | if (stmt->connection->destination) { 868 | enif_mutex_unlock(stmt->connection->mutex); 869 | return make_badarg(env, NULL, 1, "connection busy (backup destination)"); 870 | } 871 | 872 | ERL_NIF_TERM result = bind(env, stmt->connection->connection, stmt->reference->statement, 873 | argv[1], 2, SQLITE_TRANSIENT); 874 | enif_mutex_unlock(stmt->connection->mutex); 875 | return result == THE_NON_VALUE ? am_ok : result; 876 | } 877 | 878 | /* Single query step */ 879 | static ERL_NIF_TERM sqlite_step_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 880 | { 881 | statement_resource_t* stmt; 882 | ErlNifSInt64 steps; 883 | 884 | if (!enif_get_int64(env, argv[1], &steps) || steps < 1) 885 | return make_badarg(env, NULL, 2, "invalid step count"); 886 | 887 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt)) 888 | return make_badarg(env, NULL, 1, "not a prepared statement"); 889 | 890 | enif_mutex_lock(stmt->connection->mutex); 891 | if (!stmt->connection->connection) { 892 | enif_mutex_unlock(stmt->connection->mutex); 893 | return make_badarg(env, NULL, 1, "connection closed"); 894 | } 895 | 896 | if (stmt->connection->destination) { 897 | enif_mutex_unlock(stmt->connection->mutex); 898 | return make_badarg(env, NULL, 1, "connection busy (backup destination)"); 899 | } 900 | 901 | int ret; 902 | ERL_NIF_TERM result; 903 | sqlite3_stmt* sqlite_stmt = stmt->reference->statement; 904 | int column_count = sqlite3_column_count(sqlite_stmt); 905 | if (steps == 1) { 906 | /* single step return */ 907 | ret = sqlite3_step(sqlite_stmt); 908 | switch (ret) { 909 | case SQLITE_DONE: 910 | result = am_done; 911 | break; 912 | case SQLITE_BUSY: 913 | result = am_busy; 914 | break; 915 | case SQLITE_ROW: { 916 | ERL_NIF_TERM values_buffer[COLUMNS_ON_STACK]; 917 | ERL_NIF_TERM* row = &values_buffer[0]; 918 | if (column_count > 512) 919 | row = (ERL_NIF_TERM*)enif_alloc(column_count * sizeof(ERL_NIF_TERM)); 920 | else 921 | row = &values_buffer[0]; 922 | result = fetch_row(env, sqlite_stmt, column_count, row); 923 | if (column_count > COLUMNS_ON_STACK) 924 | enif_free(row); 925 | break;} 926 | default: 927 | result = make_sql_error(env, stmt->connection->connection, ret, -1); 928 | break; 929 | } 930 | } else { 931 | /* multi-step return - either [Row] list, or a tuple of {done|busy, [Row]}*/ 932 | ERL_NIF_TERM values_buffer[COLUMNS_ON_STACK]; 933 | ERL_NIF_TERM* row = &values_buffer[0]; 934 | if (column_count > 512) 935 | row = (ERL_NIF_TERM*)enif_alloc(column_count * sizeof(ERL_NIF_TERM)); 936 | else 937 | row = &values_buffer[0]; 938 | 939 | result = enif_make_list(env, 0); 940 | while (steps > 0) { 941 | ERL_NIF_TERM next_row; 942 | steps--; 943 | ret = sqlite3_step(sqlite_stmt); 944 | switch (ret) { 945 | case SQLITE_DONE: 946 | result = enif_make_tuple2(env, am_done, result); 947 | steps = 0; 948 | break; 949 | case SQLITE_BUSY: 950 | result = enif_make_tuple2(env, am_busy, result); 951 | steps = 0; 952 | break; 953 | case SQLITE_ROW: 954 | next_row = fetch_row(env, sqlite_stmt, column_count, row); 955 | result = enif_make_list_cell(env, next_row, result); 956 | break; 957 | default: 958 | result = make_sql_error(env, stmt->connection->connection, ret, -1); 959 | steps = 0; 960 | break; 961 | } 962 | } 963 | if (column_count > COLUMNS_ON_STACK) 964 | enif_free(row); 965 | } 966 | 967 | enif_mutex_unlock(stmt->connection->mutex); 968 | return result; 969 | } 970 | 971 | static ERL_NIF_TERM sqlite_reset_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 972 | { 973 | statement_resource_t* stmt; 974 | 975 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt)) 976 | return make_badarg(env, NULL, 1, "not a prepared statement"); 977 | 978 | enif_mutex_lock(stmt->connection->mutex); 979 | if (!stmt->connection->connection) { 980 | enif_mutex_unlock(stmt->connection->mutex); 981 | return make_badarg(env, NULL, 1, "connection closed"); 982 | } 983 | 984 | int ret = sqlite3_reset(stmt->reference->statement); 985 | enif_mutex_unlock(stmt->connection->mutex); 986 | 987 | if (ret != SQLITE_OK) 988 | return make_sql_error(env, stmt->connection->connection, ret, -1); 989 | return am_ok; 990 | } 991 | 992 | /* Single query step */ 993 | static ERL_NIF_TERM sqlite_clear_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 994 | { 995 | statement_resource_t* stmt; 996 | 997 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt)) 998 | return make_badarg(env, NULL, 1, "not a prepared statement"); 999 | 1000 | enif_mutex_lock(stmt->connection->mutex); 1001 | if (!stmt->connection->connection) { 1002 | enif_mutex_unlock(stmt->connection->mutex); 1003 | return make_badarg(env, NULL, 1, "connection closed"); 1004 | } 1005 | 1006 | int ret = sqlite3_clear_bindings(stmt->reference->statement); 1007 | enif_mutex_unlock(stmt->connection->mutex); 1008 | 1009 | if (ret != SQLITE_OK) 1010 | return make_sql_error(env, stmt->connection->connection, ret, -1); 1011 | return am_ok; 1012 | } 1013 | 1014 | /* Query execution */ 1015 | static ERL_NIF_TERM sqlite_execute_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1016 | { 1017 | statement_resource_t* stmt; 1018 | 1019 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt)) 1020 | return make_badarg(env, NULL, 1, "not a prepared statement"); 1021 | 1022 | enif_mutex_lock(stmt->connection->mutex); 1023 | if (!stmt->connection->connection) { 1024 | enif_mutex_unlock(stmt->connection->mutex); 1025 | return make_badarg(env, NULL, 1, "connection closed"); 1026 | } 1027 | 1028 | if (stmt->connection->destination) { 1029 | enif_mutex_unlock(stmt->connection->mutex); 1030 | return make_badarg(env, NULL, 1, "connection busy (backup destination)"); 1031 | } 1032 | 1033 | /* clear the state from the previous execute */ 1034 | sqlite3_reset(stmt->reference->statement); 1035 | 1036 | ERL_NIF_TERM result = bind(env, stmt->connection->connection, 1037 | stmt->reference->statement, argv[1], 2, SQLITE_TRANSIENT); 1038 | if (result != THE_NON_VALUE) { 1039 | enif_mutex_unlock(stmt->connection->mutex); 1040 | return result; 1041 | } 1042 | result = execute(env, stmt->connection->connection, stmt->reference->statement); 1043 | enif_mutex_unlock(stmt->connection->mutex); 1044 | return result; 1045 | } 1046 | 1047 | static ERL_NIF_TERM sqlite_describe_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1048 | { 1049 | statement_resource_t* stmt_res; 1050 | int count; 1051 | const char* name; 1052 | const char* type; 1053 | ERL_NIF_TERM columns, name_bin, type_bin; 1054 | 1055 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt_res)) 1056 | return make_badarg(env, NULL, 1, "not a prepared statement"); 1057 | 1058 | enif_mutex_lock(stmt_res->connection->mutex); 1059 | if (!stmt_res->connection->connection) { 1060 | enif_mutex_unlock(stmt_res->connection->mutex); 1061 | return make_badarg(env, NULL, 1, "connection closed"); 1062 | } 1063 | 1064 | count = sqlite3_column_count(stmt_res->reference->statement); 1065 | 1066 | columns = enif_make_list(env, 0); 1067 | 1068 | for(int i=count-1; i>=0; i--) { 1069 | name = sqlite3_column_name(stmt_res->reference->statement, i); 1070 | type = sqlite3_column_decltype(stmt_res->reference->statement, i); 1071 | if (name && !type) 1072 | return enif_raise_exception(env, am_out_of_memory); 1073 | name_bin = make_binary(env, name, strlen(name)); 1074 | type_bin = make_binary(env, type, strlen(type)); 1075 | columns = enif_make_list_cell(env, enif_make_tuple2(env, name_bin, type_bin), columns); 1076 | } 1077 | 1078 | enif_mutex_unlock(stmt_res->connection->mutex); 1079 | 1080 | return columns; 1081 | } 1082 | 1083 | static ERL_NIF_TERM sqlite_finish_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1084 | { 1085 | statement_resource_t* stmt_res; 1086 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt_res)) 1087 | return make_badarg(env, NULL, 1, "not a prepared statement"); 1088 | 1089 | enif_mutex_lock(stmt_res->connection->mutex); 1090 | if (!stmt_res->connection->connection) { 1091 | enif_mutex_unlock(stmt_res->connection->mutex); 1092 | return make_badarg(env, NULL, 1, "connection closed"); 1093 | } 1094 | 1095 | if (stmt_res->reference->statement) { 1096 | sqlite3_finalize(stmt_res->reference->statement); 1097 | stmt_res->reference->statement = NULL; 1098 | } 1099 | 1100 | /* keep all other allocated bits until GC collects them */ 1101 | 1102 | enif_mutex_unlock(stmt_res->connection->mutex); 1103 | return am_ok; 1104 | } 1105 | 1106 | /* Combined preparation and execution */ 1107 | static ERL_NIF_TERM sqlite_query_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1108 | { 1109 | connection_t* conn; 1110 | if (!enif_get_resource(env, argv[0], s_connection_resource, (void **) &conn)) 1111 | return make_badarg(env, NULL, 1, "not a connection reference"); 1112 | 1113 | ErlNifBinary query_bin; 1114 | if (!enif_inspect_iolist_as_binary(env, argv[1], &query_bin)) 1115 | return make_badarg(env, NULL, 2, "not an iolist"); 1116 | 1117 | enif_mutex_lock(conn->mutex); 1118 | if (!conn->connection) { 1119 | enif_mutex_unlock(conn->mutex); 1120 | return make_badarg(env, NULL, 1, "connection closed"); 1121 | } 1122 | 1123 | if (conn->destination) { 1124 | enif_mutex_unlock(conn->mutex); 1125 | return make_badarg(env, NULL, 1, "connection busy (backup destination)"); 1126 | } 1127 | 1128 | sqlite3_stmt* stmt; 1129 | const unsigned char* tail; 1130 | int ret = sqlite3_prepare_v2(conn->connection, (char*)query_bin.data, query_bin.size, &stmt, (const char**)&tail); 1131 | if (ret != SQLITE_OK) { 1132 | enif_mutex_unlock(conn->mutex); 1133 | return make_sql_error(env, conn->connection, ret, 2); 1134 | } 1135 | 1136 | if ((tail - query_bin.data) < query_bin.size) { 1137 | enif_mutex_unlock(conn->mutex); 1138 | return make_extended_error_ex(env, am_badarg, NULL, 2, "multiple statements are not supported", NULL, 1139 | tail - query_bin.data); 1140 | } 1141 | 1142 | ERL_NIF_TERM result = bind(env, conn->connection, stmt, argv[2], 3, SQLITE_STATIC); 1143 | if (result != THE_NON_VALUE) { 1144 | sqlite3_finalize(stmt); 1145 | enif_mutex_unlock(conn->mutex); 1146 | return result; 1147 | } 1148 | result = execute(env, conn->connection, stmt); 1149 | 1150 | /* cleanup */ 1151 | sqlite3_finalize(stmt); 1152 | enif_mutex_unlock(conn->mutex); 1153 | 1154 | return result; 1155 | } 1156 | 1157 | static void update_callback(void *arg, int op_type, char const *db, char const *table, sqlite3_int64 rowid) 1158 | { 1159 | connection_t *conn = (connection_t *)arg; 1160 | if (!conn || enif_is_pid_undefined(&conn->tracer)) 1161 | return; 1162 | 1163 | ERL_NIF_TERM op; 1164 | 1165 | switch (op_type) { 1166 | case SQLITE_INSERT: 1167 | op = am_insert; 1168 | break; 1169 | case SQLITE_DELETE: 1170 | op = am_delete; 1171 | break; 1172 | case SQLITE_UPDATE: 1173 | op = am_update; 1174 | break; 1175 | default: 1176 | op = am_undefined; 1177 | break; 1178 | } 1179 | 1180 | /* Need a NIF env to create terms */ 1181 | ErlNifEnv* env = enif_alloc_env(); 1182 | 1183 | ERL_NIF_TERM database = make_binary(env, db, strlen(db)); 1184 | ERL_NIF_TERM tab = make_binary(env, table, strlen(table)); 1185 | ERL_NIF_TERM row = enif_make_int64(env, rowid); 1186 | ERL_NIF_TERM ref = enif_make_copy(env, conn->ref); 1187 | 1188 | ERL_NIF_TERM msg = enif_make_tuple5(env, ref, op, database, tab, row); 1189 | 1190 | if (!enif_send(NULL, &conn->tracer, env, msg)) 1191 | sqlite3_update_hook(conn->connection, NULL, NULL); /* remove failed monitor */ 1192 | 1193 | enif_free_env(env); 1194 | } 1195 | 1196 | /* Connection monitoring */ 1197 | static ERL_NIF_TERM sqlite_monitor_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { 1198 | connection_t* conn; 1199 | if(!enif_get_resource(env, argv[0], s_connection_resource, (void **) &conn)) 1200 | return make_badarg(env, NULL, 1, "not a connection reference"); 1201 | 1202 | if (argv[1] == am_undefined) 1203 | sqlite3_update_hook(conn->connection, NULL, conn); 1204 | else 1205 | if (!enif_get_local_pid(env, argv[1], &conn->tracer)) 1206 | return make_badarg(env, NULL, 2, "not a pid"); 1207 | else { 1208 | conn->ref = argv[0]; 1209 | sqlite3_update_hook(conn->connection, update_callback, conn); 1210 | return conn->ref; 1211 | } 1212 | 1213 | return am_ok; 1214 | } 1215 | 1216 | static ERL_NIF_TERM sqlite_interrupt_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { 1217 | connection_t* conn; 1218 | if(!enif_get_resource(env, argv[0], s_connection_resource, (void **) &conn)) 1219 | return make_badarg(env, NULL, 1, "not a connection reference"); 1220 | 1221 | if (conn->connection) 1222 | sqlite3_interrupt(conn->connection); 1223 | else 1224 | return make_badarg(env, NULL, 1, "connection closed"); 1225 | return am_ok; 1226 | } 1227 | 1228 | 1229 | /* Various informational NIFs */ 1230 | static ERL_NIF_TERM sqlite_system_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1231 | { 1232 | ERL_NIF_TERM info = enif_make_new_map(env); 1233 | const char* vsn; 1234 | sqlite3_int64 cur, hw, page_cur, page_hw, page, undef; 1235 | 1236 | if (sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &cur, &hw, 0) == SQLITE_OK) { 1237 | ERL_NIF_TERM tuple = enif_make_tuple2(env, enif_make_int64(env, cur), enif_make_int64(env, hw)); 1238 | enif_make_map_put(env, info, am_memory_used, tuple, &info); 1239 | } 1240 | 1241 | if ((sqlite3_status64(SQLITE_STATUS_PAGECACHE_USED, &cur, &hw, 0) == SQLITE_OK) && 1242 | (sqlite3_status64(SQLITE_STATUS_PAGECACHE_OVERFLOW, &page_cur, &page_hw, 0) == SQLITE_OK) && 1243 | (sqlite3_status64(SQLITE_STATUS_PAGECACHE_SIZE, &undef, &page, 0) == SQLITE_OK)) { 1244 | ERL_NIF_TERM tuple = enif_make_tuple5(env, enif_make_int64(env, page), enif_make_int64(env, cur), 1245 | enif_make_int64(env, hw), enif_make_int64(env, page_cur), enif_make_int64(env, page_hw)); 1246 | enif_make_map_put(env, info, am_page_cache, tuple, &info); 1247 | } 1248 | 1249 | if ((sqlite3_status64(SQLITE_STATUS_MALLOC_SIZE, &page, &undef, 0) == SQLITE_OK) && 1250 | (sqlite3_status64(SQLITE_STATUS_MALLOC_COUNT, &cur, &hw, 0) == SQLITE_OK)) { 1251 | ERL_NIF_TERM tuple = enif_make_tuple3(env, enif_make_int64(env, undef), 1252 | enif_make_int64(env, cur), enif_make_int64(env, hw)); 1253 | enif_make_map_put(env, info, am_malloc, tuple, &info); 1254 | } 1255 | 1256 | /* sqlite version */ 1257 | vsn = sqlite3_libversion(); 1258 | enif_make_map_put(env, info, am_version, make_binary(env, vsn, strlen(vsn)), &info); 1259 | 1260 | return info; 1261 | } 1262 | 1263 | static ERL_NIF_TERM sqlite_get_last_insert_rowid_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1264 | { 1265 | connection_t* conn; 1266 | if(!enif_get_resource(env, argv[0], s_connection_resource, (void **) &conn)) 1267 | return make_badarg(env, NULL, 1, "not a connection reference"); 1268 | return enif_make_int64(env, sqlite3_last_insert_rowid(conn->connection)); 1269 | } 1270 | 1271 | #define db_cur_stat(stat, atom, place) do {\ 1272 | if (sqlite3_db_status(conn->connection, stat, &cur, &hw, 0) == SQLITE_OK) \ 1273 | enif_make_map_put(env, place, atom, enif_make_int64(env, cur), &place); \ 1274 | }while (0) 1275 | 1276 | #define db_hw_stat(stat, atom, place) do {\ 1277 | if (sqlite3_db_status(conn->connection, stat, &cur, &hw, 0) == SQLITE_OK) \ 1278 | enif_make_map_put(env, place, atom, enif_make_int64(env, hw), &place); \ 1279 | }while (0) 1280 | 1281 | static ERL_NIF_TERM sqlite_status_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1282 | { 1283 | connection_t *conn; 1284 | 1285 | if (!enif_get_resource(env, argv[0], s_connection_resource, (void **) &conn)) 1286 | return make_badarg(env, NULL, 1, "not a connection reference"); 1287 | 1288 | enif_mutex_lock(conn->mutex); 1289 | if (conn->connection == NULL) { 1290 | enif_mutex_unlock(conn->mutex); 1291 | return make_badarg(env, NULL, 1, "connection closed"); 1292 | } 1293 | 1294 | ERL_NIF_TERM status = enif_make_new_map(env); 1295 | ERL_NIF_TERM lookaside = enif_make_new_map(env); 1296 | ERL_NIF_TERM cache = enif_make_new_map(env); 1297 | int cur, hw; 1298 | 1299 | /* lookaside */ 1300 | if (sqlite3_db_status(conn->connection, SQLITE_DBSTATUS_LOOKASIDE_USED, &cur, &hw, 0) == SQLITE_OK) { 1301 | enif_make_map_put(env, lookaside, am_used, enif_make_int64(env, cur), &lookaside); 1302 | enif_make_map_put(env, lookaside, am_max, enif_make_int64(env, hw), &lookaside); 1303 | } 1304 | db_hw_stat(SQLITE_DBSTATUS_LOOKASIDE_HIT, am_hit, lookaside); 1305 | db_hw_stat(SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, am_miss_size, lookaside); 1306 | db_hw_stat(SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, am_miss_full, lookaside); 1307 | 1308 | /* cache */ 1309 | if (sqlite3_db_status(conn->connection, SQLITE_DBSTATUS_CACHE_SPILL, &cur, &hw, 0) == SQLITE_OK) { 1310 | ERL_NIF_TERM tuple = enif_make_tuple2(env, enif_make_int64(env, cur), enif_make_int64(env, hw)); 1311 | enif_make_map_put(env, cache, am_spill, tuple, &cache); 1312 | } 1313 | db_cur_stat(SQLITE_DBSTATUS_CACHE_USED, am_used, cache); 1314 | db_cur_stat(SQLITE_DBSTATUS_CACHE_USED_SHARED, am_shared, cache); 1315 | db_cur_stat(SQLITE_DBSTATUS_CACHE_HIT, am_hit, cache); 1316 | db_cur_stat(SQLITE_DBSTATUS_CACHE_MISS, am_miss, cache); 1317 | db_cur_stat(SQLITE_DBSTATUS_CACHE_WRITE, am_write, cache); 1318 | 1319 | /* misc: schema, statement, ... */ 1320 | db_cur_stat(SQLITE_DBSTATUS_SCHEMA_USED, am_schema, status); 1321 | db_cur_stat(SQLITE_DBSTATUS_STMT_USED, am_statement, status); 1322 | db_cur_stat(SQLITE_DBSTATUS_DEFERRED_FKS, am_deferred_fks, status); 1323 | 1324 | /* now build the result */ 1325 | enif_make_map_put(env, status, am_lookaside_memory, lookaside, &status); 1326 | enif_make_map_put(env, status, am_pager_cache_memory, cache, &status); 1327 | 1328 | enif_mutex_unlock(conn->mutex); 1329 | 1330 | return status; 1331 | } 1332 | 1333 | 1334 | #define statement_stat(op, atom) do {\ 1335 | enif_make_map_put(env, status, atom, enif_make_int64(env, sqlite3_stmt_status(stmt, op, 0)), &status);\ 1336 | }while (0) 1337 | 1338 | /* Statement status */ 1339 | static ERL_NIF_TERM sqlite_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1340 | { 1341 | statement_resource_t* stmt_res; 1342 | 1343 | if (!enif_get_resource(env, argv[0], s_statement_resource, (void **) &stmt_res)) 1344 | return make_badarg(env, NULL, 1, "not a prepared statement"); 1345 | 1346 | /* close/1 may wipe the statement out, so take a lock */ 1347 | enif_mutex_lock(stmt_res->connection->mutex); 1348 | 1349 | sqlite3_stmt* stmt = stmt_res->reference->statement; 1350 | 1351 | if (!stmt) { 1352 | enif_mutex_unlock(stmt_res->connection->mutex); 1353 | return make_badarg(env, NULL, 1, "connection closed"); 1354 | } 1355 | 1356 | ERL_NIF_TERM status = enif_make_new_map(env); 1357 | statement_stat(SQLITE_STMTSTATUS_FULLSCAN_STEP, am_fullscan_step); 1358 | statement_stat(SQLITE_STMTSTATUS_SORT, am_sort); 1359 | statement_stat(SQLITE_STMTSTATUS_AUTOINDEX, am_autoindex); 1360 | statement_stat(SQLITE_STMTSTATUS_VM_STEP, am_vm_step); 1361 | statement_stat(SQLITE_STMTSTATUS_REPREPARE, am_reprepare); 1362 | statement_stat(SQLITE_STMTSTATUS_RUN, am_run); 1363 | statement_stat(SQLITE_STMTSTATUS_FILTER_MISS, am_filter_miss); 1364 | statement_stat(SQLITE_STMTSTATUS_FILTER_HIT, am_filter_hit); 1365 | statement_stat(SQLITE_STMTSTATUS_MEMUSED, am_memory_used); 1366 | 1367 | enif_mutex_unlock(stmt_res->connection->mutex); 1368 | return status; 1369 | } 1370 | 1371 | /* Online backup API */ 1372 | static ERL_NIF_TERM sqlite_backup_init_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1373 | { 1374 | connection_t* src; 1375 | if (!enif_get_resource(env, argv[0], s_connection_resource, (void **) &src)) 1376 | return make_badarg(env, NULL, 1, "not a connection reference"); 1377 | 1378 | connection_t* dst; 1379 | if (!enif_get_resource(env, argv[2], s_connection_resource, (void **) &dst)) 1380 | return make_badarg(env, NULL, 3, "not a connection reference"); 1381 | 1382 | ErlNifBinary from_bin, to_bin; 1383 | if (!enif_inspect_iolist_as_binary(env, argv[1], &from_bin)) 1384 | return make_badarg(env, NULL, 2, "invalid source database"); 1385 | if (!enif_inspect_iolist_as_binary(env, argv[3], &to_bin)) 1386 | return make_badarg(env, NULL, 4, "invalid destination database"); 1387 | 1388 | /* make NULL-terminated string out of Erlang binary */ 1389 | char src_db[256], dst_db[256]; 1390 | binary_to_string(src_db, from_bin, sizeof(src_db)); 1391 | binary_to_string(dst_db, to_bin, sizeof(dst_db)); 1392 | 1393 | /* now start taking locks */ 1394 | enif_mutex_lock(src->mutex); 1395 | if (src->connection == NULL) { 1396 | enif_mutex_unlock(src->mutex); 1397 | return make_badarg(env, NULL, 1, "connection closed"); 1398 | } 1399 | 1400 | if (enif_mutex_trylock(dst->mutex)) { 1401 | enif_mutex_unlock(src->mutex); 1402 | return make_badarg(env, NULL, 3, "failed to lock destination database"); 1403 | } 1404 | 1405 | if (dst->connection == NULL) { 1406 | enif_mutex_unlock(dst->mutex); 1407 | enif_mutex_unlock(src->mutex); 1408 | return make_badarg(env, NULL, 3, "connection closed"); 1409 | } 1410 | 1411 | if (dst->destination) { 1412 | enif_mutex_unlock(dst->mutex); 1413 | enif_mutex_unlock(src->mutex); 1414 | return make_badarg(env, NULL, 3, "connection busy (backup destination)"); 1415 | } 1416 | 1417 | sqlite3_backup* bckp = sqlite3_backup_init(dst->connection, dst_db, src->connection, src_db); 1418 | if (!bckp) { 1419 | enif_mutex_unlock(dst->mutex); 1420 | enif_mutex_unlock(src->mutex); 1421 | return make_sql_error(env, dst->connection, sqlite3_errcode(dst->connection), -1); 1422 | } 1423 | 1424 | /* Initialize the resource */ 1425 | backup_resource_t* backup_res = enif_alloc_resource(s_backup_resource, sizeof(backup_resource_t)); 1426 | if (!backup_res) { 1427 | sqlite3_backup_finish(bckp); 1428 | enif_mutex_unlock(dst->mutex); 1429 | enif_mutex_unlock(src->mutex); 1430 | return enif_raise_exception(env, am_out_of_memory); 1431 | } 1432 | 1433 | backup_t* backup = (backup_t*)enif_alloc(sizeof(backup_t)); 1434 | ASSERT(backup); 1435 | backup_res->reference = backup; 1436 | backup->backup = bckp; 1437 | backup->source = src; 1438 | backup->destination = dst; 1439 | 1440 | src->source = backup; 1441 | dst->destination = backup; 1442 | 1443 | enif_mutex_unlock(dst->mutex); 1444 | enif_mutex_unlock(src->mutex); 1445 | 1446 | enif_keep_resource(src); 1447 | enif_keep_resource(dst); 1448 | 1449 | ERL_NIF_TERM backup_res_term = enif_make_resource(env, backup_res); 1450 | enif_release_resource(backup_res); 1451 | return backup_res_term; 1452 | } 1453 | 1454 | static ERL_NIF_TERM sqlite_backup_finish_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1455 | { 1456 | backup_resource_t* backup_res; 1457 | if (!enif_get_resource(env, argv[0], s_backup_resource, (void **) &backup_res)) 1458 | return make_badarg(env, NULL, 1, "not a backup reference"); 1459 | 1460 | connection_t* src = backup_res->reference->source; 1461 | connection_t* dst = backup_res->reference->destination; 1462 | 1463 | enif_mutex_lock(src->mutex); 1464 | enif_mutex_lock(dst->mutex); 1465 | if (backup_res->reference->backup) { 1466 | sqlite3_backup_finish(backup_res->reference->backup); 1467 | backup_res->reference->backup = NULL; 1468 | src->source = NULL; 1469 | dst->destination = NULL; 1470 | } 1471 | enif_mutex_unlock(dst->mutex); 1472 | enif_mutex_unlock(src->mutex); 1473 | 1474 | return am_ok; 1475 | } 1476 | 1477 | static ERL_NIF_TERM sqlite_backup_step_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1478 | { 1479 | backup_resource_t* backup_res; 1480 | if (!enif_get_resource(env, argv[0], s_backup_resource, (void **) &backup_res)) 1481 | return make_badarg(env, NULL, 1, "not a backup reference"); 1482 | 1483 | int step; 1484 | if (!enif_get_int(env, argv[1], &step) || step == 0) 1485 | return make_badarg(env, NULL, 2, "invalid step size"); 1486 | 1487 | connection_t* src = backup_res->reference->source; 1488 | connection_t* dst = backup_res->reference->destination; 1489 | enif_mutex_lock(src->mutex); 1490 | enif_mutex_lock(dst->mutex); 1491 | 1492 | if (!backup_res->reference->backup) { 1493 | enif_mutex_unlock(dst->mutex); 1494 | enif_mutex_unlock(src->mutex); 1495 | return make_badarg(env, NULL, 1, "backup aborted (connection closed)"); 1496 | } 1497 | 1498 | int ret = sqlite3_backup_step(backup_res->reference->backup, step); 1499 | 1500 | /* FIXME: add handling for SQLITE_BUSY and SQLITE_LOCKED */ 1501 | if (ret != SQLITE_OK && ret != SQLITE_DONE) { 1502 | enif_mutex_unlock(dst->mutex); 1503 | enif_mutex_unlock(src->mutex); 1504 | return make_sql_error(env, NULL, ret, -1); 1505 | } 1506 | 1507 | int remaining = sqlite3_backup_remaining(backup_res->reference->backup); 1508 | int total = sqlite3_backup_pagecount(backup_res->reference->backup); 1509 | 1510 | enif_mutex_unlock(dst->mutex); 1511 | enif_mutex_unlock(src->mutex); 1512 | return ret == SQLITE_DONE ? am_ok : 1513 | enif_make_tuple2(env, enif_make_int(env, remaining), enif_make_int(env, total)); 1514 | } 1515 | 1516 | /* Called when backup reference was not closed with backup_finish, but just lost instead. */ 1517 | static ERL_NIF_TERM sqlite_dirty_backup_finish_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) 1518 | { 1519 | backup_finish_t* next; 1520 | 1521 | /* iterate over pending connections */ 1522 | enif_mutex_lock(s_backup_finish_mutex); 1523 | ASSERT(s_backup_finish_queue); 1524 | next = s_backup_finish_queue; 1525 | s_backup_finish_queue = next->next; 1526 | enif_mutex_unlock(s_backup_finish_mutex); 1527 | 1528 | backup_t* backup = next->unfinished; 1529 | 1530 | /* take src & dst mutexes */ 1531 | enif_mutex_lock(backup->source->mutex); 1532 | enif_mutex_lock(backup->destination->mutex); 1533 | sqlite3_backup_finish(backup->backup); 1534 | backup->source->source = NULL; 1535 | backup->destination->destination = NULL; 1536 | enif_mutex_unlock(backup->destination->mutex); 1537 | enif_mutex_unlock(backup->source->mutex); 1538 | 1539 | enif_release_resource(backup->source); 1540 | enif_release_resource(backup->destination); 1541 | 1542 | enif_free(next); 1543 | return am_ok; 1544 | } 1545 | 1546 | /* NIF Exports */ 1547 | static ErlNifFunc nif_funcs[] = { 1548 | {"sqlite_open_nif", 2, sqlite_open_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1549 | {"sqlite_close_nif", 1, sqlite_close_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1550 | {"sqlite_status_nif", 1, sqlite_status_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1551 | {"sqlite_query_nif", 3, sqlite_query_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1552 | {"sqlite_prepare_nif", 3, sqlite_prepare_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1553 | {"sqlite_bind_nif", 2, sqlite_bind_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1554 | {"sqlite_step_nif", 2, sqlite_step_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1555 | {"sqlite_reset_nif", 1, sqlite_reset_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1556 | {"sqlite_clear_nif", 1, sqlite_clear_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1557 | {"sqlite_info_nif", 1, sqlite_info_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1558 | {"sqlite_finish_nif", 1, sqlite_finish_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1559 | {"sqlite_describe_nif", 1, sqlite_describe_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1560 | {"sqlite_execute_nif", 2, sqlite_execute_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1561 | {"sqlite_monitor_nif", 2, sqlite_monitor_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1562 | {"sqlite_interrupt_nif", 1, sqlite_interrupt_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1563 | {"sqlite_system_info_nif", 0, sqlite_system_info_nif}, 1564 | {"sqlite_get_last_insert_rowid_nif", 1, sqlite_get_last_insert_rowid_nif}, 1565 | {"sqlite_backup_init_nif", 4, sqlite_backup_init_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1566 | {"sqlite_backup_step_nif", 2, sqlite_backup_step_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1567 | {"sqlite_backup_finish_nif", 1, sqlite_backup_finish_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1568 | 1569 | {"sqlite_dirty_close_nif", 0, sqlite_dirty_close_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, 1570 | {"sqlite_dirty_backup_finish_nif", 0, sqlite_dirty_backup_finish_nif, ERL_NIF_DIRTY_JOB_IO_BOUND} 1571 | }; 1572 | 1573 | 1574 | /* memory management */ 1575 | static int sqlmem_init(void* unused) {return 0;} 1576 | static void sqlmem_shutdown(void* unused) {} 1577 | 1578 | static void* sqlmem_alloc(int size) 1579 | { 1580 | ErlNifSInt64* alloc = (ErlNifSInt64*)enif_alloc(size + sizeof(ErlNifSInt64)); 1581 | *alloc = size; 1582 | return alloc + 1; 1583 | } 1584 | 1585 | static void sqlmem_free(void* ptr) 1586 | { 1587 | ErlNifSInt64* alloc = (ErlNifSInt64*)ptr; 1588 | enif_free(alloc - 1); 1589 | } 1590 | 1591 | static void* sqlmem_realloc(void* ptr, int size) 1592 | { 1593 | ErlNifSInt64* alloc = (ErlNifSInt64*)ptr; 1594 | alloc = enif_realloc(alloc - 1, size + sizeof(ErlNifSInt64)); 1595 | *alloc = size; 1596 | return alloc + 1; 1597 | } 1598 | 1599 | static int sqlmem_roundup(int orig) 1600 | { 1601 | int bit = sizeof(ErlNifSInt64) - 1; 1602 | return (orig + bit) & (~bit); 1603 | } 1604 | 1605 | static int sqlmem_size(void* ptr) 1606 | { 1607 | ErlNifSInt64* alloc = (ErlNifSInt64*)ptr; 1608 | return alloc[-1]; 1609 | } 1610 | 1611 | /* Load/unload/reload */ 1612 | static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM load_info) 1613 | { 1614 | am_ok = enif_make_atom(env, "ok"); 1615 | am_error = enif_make_atom(env, "error"); 1616 | am_true = enif_make_atom(env, "true"); 1617 | am_false = enif_make_atom(env, "false"); 1618 | 1619 | int arity; 1620 | const ERL_NIF_TERM* config; 1621 | 1622 | if (!enif_get_tuple(env, load_info, &arity, &config) || 1623 | !enif_get_local_pid(env, config[0], &s_delayed_close_pid) || 1624 | !enif_is_process_alive(env, &s_delayed_close_pid) || 1625 | (config[1] != am_true && config[1] != am_false)) 1626 | return -6; 1627 | 1628 | if (config[1] == am_true) { 1629 | /* configure sqlite mutex & memory allocation */ 1630 | sqlite3_mem_methods mem; 1631 | 1632 | mem.xMalloc = sqlmem_alloc; 1633 | mem.xFree = sqlmem_free; 1634 | mem.xRealloc = sqlmem_realloc; 1635 | mem.xRoundup = sqlmem_roundup; 1636 | mem.xSize = sqlmem_size; 1637 | mem.xInit = sqlmem_init; 1638 | mem.xShutdown = sqlmem_shutdown; 1639 | 1640 | if (sqlite3_config(SQLITE_CONFIG_MALLOC, &mem) != SQLITE_OK) 1641 | return -1; 1642 | } 1643 | 1644 | /* use multi-threaded SQLite mode to enable explicit NIF mutex name */ 1645 | if (sqlite3_config(SQLITE_CONFIG_MULTITHREAD, NULL) != SQLITE_OK) 1646 | return -2; 1647 | 1648 | if (sqlite3_initialize() != SQLITE_OK) 1649 | return -3; 1650 | 1651 | am_general = enif_make_atom(env, "general"); 1652 | am_reason = enif_make_atom(env, "reason"); 1653 | am_badarg = enif_make_atom(env, "badarg"); 1654 | am_undefined = enif_make_atom(env, "undefined"); 1655 | am_position = enif_make_atom(env, "position"); 1656 | am_out_of_memory = enif_make_atom(env, "out_of_memory"); 1657 | am_sqlite_error = enif_make_atom(env, "sqlite_error"); 1658 | am_blob = enif_make_atom(env, "blob"); 1659 | am_done = enif_make_atom(env, "done"); 1660 | am_busy = enif_make_atom(env, "busy"); 1661 | 1662 | am_mode = enif_make_atom(env, "mode"); 1663 | am_flags = enif_make_atom(env, "flags"); 1664 | am_uri = enif_make_atom(env, "uri"); 1665 | am_read_only = enif_make_atom(env, "read_only"); 1666 | am_read_write = enif_make_atom(env, "read_write"); 1667 | am_read_write_create = enif_make_atom(env, "read_write_create"); 1668 | am_shared = enif_make_atom(env, "shared"); 1669 | am_memory = enif_make_atom(env, "memory"); 1670 | am_busy_timeout = enif_make_atom(env, "busy_timeout"); 1671 | 1672 | am_persistent = enif_make_atom(env, "persistent"); 1673 | am_no_vtab = enif_make_atom(env, "no_vtab"); 1674 | 1675 | am_insert = enif_make_atom(env, "insert"); 1676 | am_update = enif_make_atom(env, "update"); 1677 | am_delete = enif_make_atom(env, "delete"); 1678 | 1679 | am_lookaside_memory = enif_make_atom(env, "lookaside_memory"); 1680 | am_pager_cache_memory = enif_make_atom(env, "pager_cache_memory"); 1681 | am_schema = enif_make_atom(env, "schema"); 1682 | am_statement = enif_make_atom(env, "statement"); 1683 | am_deferred_fks = enif_make_atom(env, "deferred_fks"); 1684 | 1685 | am_used = enif_make_atom(env, "used"); 1686 | am_max = enif_make_atom(env, "max"); 1687 | am_hit = enif_make_atom(env, "hit"); 1688 | am_miss_size = enif_make_atom(env, "miss_size"); 1689 | am_miss_full = enif_make_atom(env, "miss_full"); 1690 | am_shared = enif_make_atom(env, "shared"); 1691 | am_miss = enif_make_atom(env, "miss"); 1692 | am_write = enif_make_atom(env, "write"); 1693 | am_spill = enif_make_atom(env, "spill"); 1694 | 1695 | am_fullscan_step = enif_make_atom(env, "fullscan_step"); 1696 | am_sort = enif_make_atom(env, "sort"); 1697 | am_autoindex = enif_make_atom(env, "autoindex"); 1698 | am_vm_step = enif_make_atom(env, "vm_step"); 1699 | am_reprepare = enif_make_atom(env, "reprepare"); 1700 | am_run = enif_make_atom(env, "run"); 1701 | am_filter_miss = enif_make_atom(env, "filter_miss"); 1702 | am_filter_hit = enif_make_atom(env, "filter_hit"); 1703 | am_memory_used = enif_make_atom(env, "memory_used"); 1704 | 1705 | am_page_cache = enif_make_atom(env, "page_cache"); 1706 | am_malloc = enif_make_atom(env, "malloc"); 1707 | am_version = enif_make_atom(env, "version"); 1708 | 1709 | s_connection_resource = enif_open_resource_type(env, NULL, "sqlite_connection", sqlite_connection_destroy, ERL_NIF_RT_CREATE, NULL); 1710 | s_statement_resource = enif_open_resource_type(env, NULL, "sqlite_statement", sqlite_statement_destroy, ERL_NIF_RT_CREATE, NULL); 1711 | s_backup_resource = enif_open_resource_type(env, NULL, "sqlite_backup", sqlite_backup_destroy, ERL_NIF_RT_CREATE, NULL); 1712 | 1713 | if (!s_connection_resource || !s_statement_resource || !s_backup_resource) { 1714 | sqlite3_shutdown(); 1715 | return -4; 1716 | } 1717 | 1718 | s_delayed_close_mutex = enif_mutex_create("sqlite_delayed_close"); 1719 | ASSERT(!s_delayed_close_mutex); 1720 | s_delayed_close_queue = NULL; 1721 | 1722 | s_backup_finish_mutex = enif_mutex_create("sqlite_backup_finish"); 1723 | ASSERT(!s_backup_finish_mutex); 1724 | s_backup_finish_queue = NULL; 1725 | 1726 | return 0; 1727 | } 1728 | 1729 | static void unload(ErlNifEnv *env, void* priv_data) 1730 | { 1731 | sqlite3_shutdown(); 1732 | enif_mutex_destroy(s_delayed_close_mutex); 1733 | enif_mutex_destroy(s_backup_finish_mutex); 1734 | } 1735 | 1736 | static int upgrade(ErlNifEnv *env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) 1737 | { 1738 | return 0; 1739 | } 1740 | 1741 | ERL_NIF_INIT(sqlite, nif_funcs, load, NULL, upgrade, unload); -------------------------------------------------------------------------------- /priv/.gitignore: -------------------------------------------------------------------------------- 1 | sqlite.so 2 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info]}. 2 | {deps, []}. 3 | 4 | {minimum_otp_vsn, "24.0"}. 5 | 6 | {shell, [ 7 | % {config, "config/sys.config"}, 8 | {apps, [sqlite]} 9 | ]}. 10 | 11 | {pre_hooks, 12 | [{"(linux|darwin|solaris)", compile, "make -C c_src"}, 13 | {"(freebsd)", compile, "gmake -C c_src"}, 14 | {"(win32)", compile, "nmake -F c_src/Makefile.win"}]}. 15 | {post_hooks, 16 | [{"(linux|darwin|solaris)", clean, "make -C c_src clean"}, 17 | {"(freebsd)", clean, "gmake -C c_src clean"}, 18 | {"(win32)", clean, "nmake -F c_src/Makefile.win clean"}]}. 19 | 20 | %{cover_enabled, true}. 21 | %{cover_opts, [verbose]}. 22 | {ct_opts, [ 23 | %% {ct_hooks, [cth_surefire]}, %% for GitLab CI reports 24 | {keep_logs, 1} 25 | ]}. 26 | 27 | {hex, [ 28 | {doc, #{provider => ex_doc}} 29 | ]}. 30 | 31 | {project_plugins, [rebar3_ex_doc]}. 32 | 33 | {ex_doc, [ 34 | {extras, [ 35 | {"README.md", #{title => "Overview"}}, 36 | {"LICENSE.md", #{title => "License"}} 37 | ]}, 38 | {main, "README.md"}, 39 | {source_url, "https://github.com/max-au/sqlite"}, 40 | {source_ref, <<"master">>} 41 | ]}. 42 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | []. 2 | -------------------------------------------------------------------------------- /src/sqlite.app.src: -------------------------------------------------------------------------------- 1 | {application, sqlite, 2 | [{description, "sqlite: NIF bindings for Erlang"}, 3 | {vsn, "2.0.0"}, 4 | {registered, []}, 5 | {applications, [kernel, stdlib]}, 6 | {env, []}, 7 | {modules, []}, 8 | 9 | {licenses, ["BSD 3-clause clear"]}, 10 | {links, [{"GitHub", "https://github.com/max-au/sqlite"}]} 11 | ]}. 12 | -------------------------------------------------------------------------------- /src/sqlite.erl: -------------------------------------------------------------------------------- 1 | %%% @copyright (C) 2023 Maxim Fedorov 2 | %%% @doc 3 | %%% sqlite3 NIF bindings for Erlang. 4 | %%% 5 | %%% See {@link query/3} for the data type conversion rules. 6 | %%% 7 | %%% All APIs of this module provide extended error information for the shell. 8 | %%% This information is also accessible programmatically using `erl_error'. 9 | %%% ``` 10 | %%% try 11 | %%% sqlite:prepare(sqlite:open(""), "INSERT INTO kv (key, val) VALU1ES (2, 2)") 12 | %%% catch 13 | %%% Class:Reason:Stack -> 14 | %%% Formatted = erl_error:format_exception(Class, Reason, Stack), 15 | %%% io:format(lists:flatten(Formatted)) 16 | %%% end. 17 | %%% ''' 18 | %%% @end 19 | %%% 20 | %%% Extended errors: for SQL errors, "general" contains the error code text, 21 | %%% and the detailed text is tied to an argument. 22 | %%% 23 | -module(sqlite). 24 | -author("maximfca@gmail.com"). 25 | 26 | %% API 27 | -export([open/1, open/2, close/1, status/1, reset/1, clear/1, finish/1, 28 | query/2, query/3, prepare/2, prepare/3, bind/2, step/1, step/2, info/1, describe/1, execute/2, 29 | monitor/1, demonitor/1, interrupt/1, system_info/0, get_last_insert_rowid/1, 30 | backup/2, backup/3 31 | ]). 32 | 33 | %% Internal export for erl_error formatter 34 | -export([format_error/2]). 35 | 36 | %% NIF loading code 37 | -on_load(init/0). 38 | -nifs([sqlite_open_nif/2, sqlite_close_nif/1, sqlite_status_nif/1, sqlite_reset_nif/1, sqlite_clear_nif/1, 39 | sqlite_query_nif/3, sqlite_prepare_nif/3, sqlite_bind_nif/2, sqlite_info_nif/1, sqlite_execute_nif/2, 40 | sqlite_step_nif/2, sqlite_monitor_nif/2, sqlite_describe_nif/1, sqlite_interrupt_nif/1, 41 | sqlite_system_info_nif/0, sqlite_get_last_insert_rowid_nif/1, sqlite_finish_nif/1, 42 | sqlite_backup_init_nif/4, sqlite_backup_step_nif/2, sqlite_backup_finish_nif/1, 43 | sqlite_dirty_close_nif/0, sqlite_dirty_backup_finish_nif/0]). 44 | 45 | -define(nif_stub, nif_stub_error(?LINE)). 46 | nif_stub_error(Line) -> 47 | erlang:nif_error({nif_not_loaded,module, ?MODULE, line, Line}). 48 | 49 | -type connection() :: reference(). 50 | %% SQLite connection resource reference. 51 | 52 | -type connect_flag() :: memory | uri | shared. 53 | %% SQLite connections flags specification 54 | %% 55 | %% Normally used to specify an in-memory database: `flags => [memory]'. 56 | %% 57 | %% Example of an in-memory database shared between multiple connections: 58 | %% ``` 59 | %% Db1 = sqlite:open("file::mem:", #{flags => [shared, memory, uri]}), 60 | %% Db2 = sqlite:open("file::mem:", #{flags => [shared, memory, uri]}), 61 | %% ''' 62 | %%
    63 | %%
  • `memory': enables `SQLITE_OPEN_MEMORY' flag
    • 64 | %%
  • `uri': enables `SQLITE_OPEN_URI' flag
    • 65 | %%
  • `shared': enables `SQLITE_OPEN_SHAREDCACHE' flag
    • 66 | %%
67 | 68 | -type connect_options() :: #{ 69 | mode => read_only | read_write | read_write_create, 70 | flags => [connect_flag()], 71 | busy_timeout => non_neg_integer() 72 | }. 73 | %% SQLite connection options. 74 | %% 75 | %%
    76 | %%
  • `mode': file open mode, the default is `read_write_create'
    • 77 | %%
  • `flags': SQLite connection flags specification
    • 78 | %%
  • `busy_timeout': specifies SQLite busy timeout. NOTE: using this setting may 79 | %% easily exhaust all ERTS Dirty I/O schedulers. Prefer WAL mode instead
    • 80 | %%
81 | 82 | -type lookaside_memory() :: #{ 83 | used => integer(), 84 | max => integer(), 85 | hit => integer(), 86 | miss_size => integer(), 87 | miss_full => integer() 88 | }. 89 | %% SQLite connection lookaside memory usage 90 | %% 91 | %%
    92 | %%
  • `used': the current number of lookaside memory slots currently checked out 93 | %% (current value of `SQLITE_DBSTATUS_LOOKASIDE_USED')
    • 94 | %%
  • `max': high watermark of `SQLITE_DBSTATUS_LOOKASIDE_USED'
    • 95 | %%
  • `hit': number of malloc attempts that were satisfied using lookaside memory 96 | %% (high watermark of `SQLITE_DBSTATUS_LOOKASIDE_HIT')
    • 97 | %%
  • `miss_size': the number malloc attempts that might have been satisfied using lookaside 98 | %% memory but failed due to the amount of memory requested being larger than the lookaside slot size 99 | %% (high watermark of `SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE')
    • 100 | %%
  • `miss_full': the number malloc attempts that might have been satisfied using lookaside memory 101 | %% but failed due to all lookaside memory already being in use 102 | %% (high watermark of `SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL')
    • 103 | %%
104 | 105 | -type pager_cache_memory() :: #{ 106 | used => integer(), 107 | shared => integer(), 108 | hit => integer(), 109 | miss => integer(), 110 | write => integer(), 111 | spill => {Current :: integer(), Max :: integer()} 112 | }. 113 | %% SQLite connection pager cache memory usage 114 | %% 115 | %%
    116 | %%
  • `used': the approximate number of bytes of heap memory used by all pager caches 117 | %% associated with the database connection (current value of `SQLITE_DBSTATUS_CACHE_USED')
    • 118 | %%
  • `shared': current value of `SQLITE_DBSTATUS_CACHE_USED_SHARED'
    • 119 | %%
  • `hit': the number of pager cache hits that have occurred 120 | %% (current value of `SQLITE_DBSTATUS_CACHE_HIT')
    • 121 | %%
  • `miss': the number of pager cache misses that have occurred 122 | %% (current value of SQLITE_DBSTATUS_CACHE_MISS)
    • 123 | %%
  • `write': the number of dirty cache entries that have been written to disk 124 | %% (current value of `SQLITE_DBSTATUS_CACHE_WRITE')
    • 125 | %%
  • `spill': the number of dirty cache entries that have been written to disk in the middle 126 | %% of a transaction due to the page cache overflowing. Both values of `SQLITE_DBSTATUS_CACHE_SPILL'
    • 127 | %%
128 | 129 | 130 | -type connection_status() :: #{ 131 | lookaside_memory => lookaside_memory(), 132 | pager_cache_memory => pager_cache_memory(), 133 | schema => integer(), 134 | statement => integer(), 135 | deferred_fks => integer() 136 | }. 137 | %% SQLite connection status 138 | %% 139 | %%
    140 | %%
  • `schema': the approximate number of bytes of heap memory used to store the schema for 141 | %% all databases associated with the connection (current value of `SQLITE_DBSTATUS_SCHEMA_USED')
    • 142 | %%
  • `statement': the approximate number of bytes of heap and lookaside memory used by all 143 | %% prepared statements associated with the database connection 144 | %% (current value of `SQLITE_DBSTATUS_STMT_USED')
    • 145 | %%
  • `deferred_fks': returns zero for the current value if and only if all foreign key 146 | %% constraints (deferred or immediate) have been resolved (current value of `SQLITE_DBSTATUS_DEFERRED_FKS')
    • 147 | %%
148 | 149 | -type prepared_statement() :: reference(). 150 | %% Prepared statement reference. 151 | 152 | -type parameter() :: integer() | float() | binary() | string() | iolist() | {binary, binary()} | undefined. 153 | %% Erlang type allowed to be used in SQLite bindings. 154 | 155 | -type parameters() :: [parameter()] | #{atom() | iolist() | pos_integer() => parameter()}. 156 | %% Positional and named parameters. 157 | %% 158 | %% A list of parameters must match the expected number of parameters. 159 | %% A map of parameters may be used to define named parameters, or 160 | %% assign specific columns. When map key is an integer, it is treated 161 | %% as a column index. 162 | 163 | -type row() :: tuple(). 164 | 165 | -type prepare_options() :: #{ 166 | persistent => boolean(), 167 | no_vtab => boolean() 168 | }. 169 | %% Prepared statement creation flags 170 | %% 171 | %%
    172 | %%
  • `persistent': The SQLITE_PREPARE_PERSISTENT flag is a hint to the query planner that 173 | %% the prepared statement will be retained for a long time and probably reused many times
    • 174 | %%
  • `no_vtab': The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler to return an error 175 | %% (error code SQLITE_ERROR) if the statement uses any virtual tables
    • 176 | %%
177 | 178 | -type statement_info() :: #{ 179 | fullscan_step => integer(), 180 | sort => integer(), 181 | autoindex => integer(), 182 | vm_step => integer(), 183 | reprepare => integer(), 184 | run => integer(), 185 | filter_miss => integer(), 186 | filter_hit => integer(), 187 | memory_used => integer() 188 | }. 189 | %% Prepared statement statistics 190 | %% 191 | %% Performance counters for SQLite statement. See `sqlite3_stmt_status' 192 | %% function in SQLite reference. 193 | 194 | -type system_info() :: #{ 195 | memory_used => {Cur :: integer(), Max :: integer()}, 196 | page_cache => {Size :: integer(), Used :: integer(), Max :: integer(), Overflow :: integer(), OverflowMax :: integer()}, 197 | malloc => {Size :: integer(), Count :: integer(), Max :: integer()}, 198 | version => binary() 199 | }. 200 | %% Status information about the performance of SQLite 201 | %% 202 | %% See `sqlite3_status' in the SQLite reference for more details about 203 | %% exported statistics. 204 | 205 | -type progress_callback() :: fun((Remaining :: non_neg_integer(), Total :: non_neg_integer()) -> 206 | ok | {ok, NewStep :: pos_integer()} | stop). 207 | %% Backup progress callback. 208 | %% Must return `ok' for backup to continue, `{ok, Steps}` to change the steps amount. 209 | %% Return `stop' to stop the backup process. Throwing an exception also stops the process. 210 | 211 | -type backup_options() :: #{ 212 | from => unicode:chardata(), 213 | to => unicode:chardata(), 214 | step => pos_integer(), 215 | progress => progress_callback() 216 | }. 217 | %% Backup options 218 | %% 219 | %%
    220 | %%
  • `from': source database name, `<<"main">>' is the default
    • 221 | %%
  • `to': destination database name, takes `from' value as the default
    • 222 | %%
  • `stop': backup step, pages. Default is 64
    • 223 | %%
  • `progress': progress callback to invoke after finishing the next step
    • 224 | %%
225 | 226 | %% @equiv open(FileName, #{}) 227 | -spec open(file:filename_all()) -> connection(). 228 | open(FileName) -> 229 | open(FileName, #{}). 230 | 231 | %% @doc Open the connection to a specified SQLite database. 232 | -spec open(file:filename_all(), Options :: connect_options()) -> connection(). 233 | open(FileName, Options) -> 234 | case sqlite_open_nif(FileName, Options) of 235 | {error, Reason, ExtErr} -> 236 | erlang:error(Reason, [FileName, Options], [{error_info, #{cause => ExtErr}}]); 237 | Connection -> 238 | Connection 239 | end. 240 | 241 | %% @doc Closes the connection. 242 | %% 243 | %% Ensures that all prepared statements are deallocated. Forcibly 244 | %% stops all running backups (both source and destination). Releases 245 | %% the underlying database file. 246 | %% 247 | %% It is not necessary to call this function explicitly, as eventually 248 | %% all resources will be collected. However it might be useful if you 249 | %% need to unlock the database file immediately. 250 | %% 251 | %% Throws `badarg' if the connection is already closed. 252 | -spec close(connection()) -> ok. 253 | close(Connection) -> 254 | case sqlite_close_nif(Connection) of 255 | {error, Reason, ExtErr} -> 256 | erlang:error(Reason, [Connection], [{error_info, #{cause => ExtErr}}]); 257 | ok -> 258 | ok 259 | end. 260 | 261 | %% @doc Returns connection statistics. 262 | -spec status(connection()) -> connection_status(). 263 | status(Connection) -> 264 | case sqlite_status_nif(Connection) of 265 | {error, Reason, ExtErr} -> 266 | erlang:error(Reason, [Connection], [{error_info, #{cause => ExtErr}}]); 267 | Status -> 268 | Status 269 | end. 270 | 271 | %% @equiv query(Connection, Query, []) 272 | -spec query(connection(), iodata()) -> [row()]. 273 | query(Connection, Query) -> 274 | query(Connection, Query, []). 275 | 276 | %% @doc Runs an SQL query using specified connection 277 | %% 278 | %% Query may contain placeholders, e.g. `?', `?1', `?2'. Number of placeholders 279 | %% must match the length of the parameters list. However for a map-based 280 | %% parameters, there is no such limitation, and it is supported to bind a single 281 | %% column using the following syntax: `#{2 => "value"}'. 282 | %% 283 | %% Named parameters are also supported: 284 | %% ``` 285 | %% sqlite:query(Conn, "SELECT * FROM table WHERE key = :key", #{key => 1}). 286 | %% ''' 287 | %% Missing named parameters are treated as NULL. 288 | %% 289 | %% Parameters are converted to sqlite storage classed using following type mappings: 290 | %%
    291 | %%
  • `binary()': accepts an Erlang binary stored as a TEXT in sqlite. See `unicode' module for 292 | %% transformations between Unicode characters list and a binary
    • 293 | %%
  • `string()': accepts an Erlang string limited to `iolist()' type, stored as a TEXT
    • 294 | %%
  • `iolist()': accepts lists supported by {@link erlang:iolist_to_binary/1}, stored as a TEXT
    • 295 | %%
  • `integer()': accepts a 64-bit signed integer (BIGNUM is not supported), stored as an INTEGER
    • 296 | %%
  • `float()': accepts IEEE floating point number, stored as a FLOAT
    • 297 | %%
  • `undefined': stored as NULL
    • 298 | %%
  • `{blob, binary()}': accepts an Erlang binary stored as a BLOB
    • 299 | %%
300 | %% 301 | %% Returns a list of database rows, or an empty list if no rows are expected 302 | %% to be returned (e.g. `CREATE TABLE' statement). By default, all TEXT fields are 303 | %% returned as Erlang `binary()' type for performance reasons. 304 | %% 305 | %% BLOB is always returned as a binary. Note that wrapping in `{binary, Bin}' tuple is 306 | %% only necessary for parameters, to tell the actual binary from an UTF8 encoded string. 307 | -spec query(connection(), iodata(), parameters()) -> [row()]. 308 | query(Connection, Query, Parameter) -> 309 | case sqlite_query_nif(Connection, Query, Parameter) of 310 | {error, Reason, ExtErr} -> 311 | erlang:error(Reason, [Connection, Query, Parameter], [{error_info, #{cause => ExtErr}}]); 312 | Result when is_list(Result) -> 313 | lists:reverse(Result) 314 | end. 315 | 316 | %% @equiv prepare(Connection, Query, #{}) 317 | -spec prepare(connection(), iodata()) -> prepared_statement(). 318 | prepare(Connection, Query) -> 319 | prepare(Connection, Query, #{}). 320 | 321 | %% @doc Creates a prepared statement. 322 | %% 323 | %% Use {@link execute/2} to run the statement. 324 | %% Note that {@link close/1} invalidates all prepared statements created 325 | %% using the specified connection. 326 | %% 327 | %% By default, the prepared statement is not persistent. Note that 328 | %% if a single statement is going to be reused many times, it may 329 | %% prove useful to pass `persistent' option set to `true'. 330 | %% 331 | %% Running a single {@link query/3} once is more efficient than preparing 332 | %% a statement, executing it once and discarding. 333 | -spec prepare(connection(), iodata(), prepare_options()) -> prepared_statement(). 334 | prepare(Connection, Query, Options) -> 335 | case sqlite_prepare_nif(Connection, Query, Options) of 336 | {error, Reason, ExtErr} -> 337 | erlang:error(Reason, [Connection, Query, Options], [{error_info, #{cause => ExtErr}}]); 338 | Prepared -> 339 | Prepared 340 | end. 341 | 342 | %% @doc Binds arguments to a prepared statement. 343 | %% 344 | %% Does not reset the statement, and throws `badarg' with SQL_MISUSE 345 | %% explanation if the statement needs to be reset first. 346 | %% 347 | %% See {@link query/3} for types and bindings details. 348 | -spec bind(prepared_statement(), parameters()) -> ok. 349 | bind(Prepared, Parameters) -> 350 | case sqlite_bind_nif(Prepared, Parameters) of 351 | {error, Reason, ExtErr} -> 352 | erlang:error(Reason, [Prepared, Parameters], [{error_info, #{cause => ExtErr}}]); 353 | ok -> 354 | ok 355 | end. 356 | 357 | %% @doc Evaluates the prepared statement. 358 | %% 359 | %% Returns `done' when an operation has completed. 360 | %% May return `busy', see `SQLITE_BUSY' return code in 361 | %% the sqlite reference manual. 362 | -spec step(prepared_statement()) -> busy | done | row(). 363 | step(Prepared) -> 364 | case sqlite_step_nif(Prepared, 1) of 365 | {error, Reason, ExtErr} -> 366 | erlang:error(Reason, [Prepared], [{error_info, #{cause => ExtErr}}]); 367 | Result -> 368 | Result 369 | end. 370 | 371 | %% @doc Evaluates the prepared statement. 372 | %% 373 | %% Returns either a list of rows when there are more rows available, 374 | %% or `{done, Rows}` tuple with the list of the remaining rows. 375 | %% May return `{busy, Rows}', requesting the caller to handle 376 | %% `SQL_BUSY' according to sqlite recommendations. 377 | -spec step(prepared_statement(), pos_integer()) -> [row()] | {done, [row()]} | {busy, [row()]}. 378 | step(Prepared, Steps) when Steps > 1 -> 379 | case sqlite_step_nif(Prepared, Steps) of 380 | {error, Reason, ExtErr} -> 381 | erlang:error(Reason, [Prepared, Steps], [{error_info, #{cause => ExtErr}}]); 382 | {Code, Result} -> 383 | {Code, lists:reverse(Result)}; 384 | List -> 385 | lists:reverse(List) 386 | end. 387 | 388 | %% @doc Resets the prepared statement, but does not clear bindings 389 | -spec reset(prepared_statement()) -> ok. 390 | reset(Prepared) -> 391 | case sqlite_reset_nif(Prepared) of 392 | {error, Reason, ExtErr} -> 393 | erlang:error(Reason, [Prepared], [{error_info, #{cause => ExtErr}}]); 394 | Result -> 395 | Result 396 | end. 397 | 398 | %% @doc Clears bindings of the prepared statement 399 | -spec clear(prepared_statement()) -> ok. 400 | clear(Prepared) -> 401 | case sqlite_clear_nif(Prepared) of 402 | {error, Reason, ExtErr} -> 403 | erlang:error(Reason, [Prepared], [{error_info, #{cause => ExtErr}}]); 404 | Result -> 405 | Result 406 | end. 407 | 408 | %% @doc Finalises the prepared statement, freeing any resources allocated 409 | %% 410 | %% The purpose of this function is to cancel statements that are executed 411 | %% halfway through, and it is desirable to stop execution and free all 412 | %% allocated resources. 413 | -spec finish(prepared_statement()) -> ok. 414 | finish(Prepared) -> 415 | case sqlite_finish_nif(Prepared) of 416 | {error, Reason, ExtErr} -> 417 | erlang:error(Reason, [Prepared], [{error_info, #{cause => ExtErr}}]); 418 | Result -> 419 | Result 420 | end. 421 | 422 | %% @doc Returns column names and types for the prepared statement. 423 | %% 424 | %% SQLite uses dynamic type system. Types returned are column types specified 425 | %% in the `CREATE TABLE' statement. SQLite accepts any string as a type name, 426 | %% unless `STRICT' mode is used. 427 | -spec describe(prepared_statement()) -> [{Name :: binary(), Type :: binary()}]. 428 | describe(Prepared) -> 429 | case sqlite_describe_nif(Prepared) of 430 | {error, Reason, ExtErr} -> 431 | erlang:error(Reason, [Prepared], [{error_info, #{cause => ExtErr}}]); 432 | Description -> 433 | Description 434 | end. 435 | 436 | %% @doc Returns runtime statistics for the prepared statement. 437 | -spec info(prepared_statement()) -> statement_info(). 438 | info(Prepared) -> 439 | case sqlite_info_nif(Prepared) of 440 | {error, Reason, ExtErr} -> 441 | erlang:error(Reason, [Prepared], [{error_info, #{cause => ExtErr}}]); 442 | Description -> 443 | Description 444 | end. 445 | 446 | %% @doc Runs the prepared statement with new parameters bound. 447 | %% 448 | %% Resets the prepared statement, binds new parameters and 449 | %% steps until no more rows are available. Throws an error 450 | %% if at any step `SQLITE_BUSY' was returned, see {@link sqlite:connect_options()} 451 | %% for busy timeout handling. 452 | %% 453 | %% See {@link query/3} for types and bindings details. 454 | -spec execute(prepared_statement(), parameters()) -> [row()]. 455 | execute(Prepared, Parameters) -> 456 | case sqlite_execute_nif(Prepared, Parameters) of 457 | {error, Reason, ExtErr} -> 458 | erlang:error(Reason, [Prepared, Parameters], [{error_info, #{cause => ExtErr}}]); 459 | Result when is_list(Result) -> 460 | lists:reverse(Result) 461 | end. 462 | 463 | %% @doc EXPERIMENTAL: monitor updates happening through the connection. 464 | %% 465 | %% This function is intended for debugging INSERT, UPDATE and DELETE 466 | %% operations. The API is experimental and may change in the future 467 | %% without prior notice. 468 | %% 469 | %% Only one process may monitor a connection. Subsequent monitor calls 470 | %% replace the previously set process. 471 | %% 472 | %% Upon successful completion, calling process may start receiving 473 | %% messages of the following format: 474 | %% ``` 475 | %% {Ref, Op, Database, Table, RowID} 476 | %% Example: 477 | %% {#Ref<0.1954006965.2226257955.79689>, insert, <<"main">>, <<"table">>, 123} 478 | %% ''' 479 | %% `Ref' is the reference returned by `monitor/1' call, `Op' is one of 480 | %% `insert', `update' or `delete' operations, and `RowID' is the SQLite 481 | %% ROWID. 482 | -spec monitor(connection()) -> reference(). 483 | monitor(Connection) -> 484 | case sqlite_monitor_nif(Connection, self()) of 485 | {error, Reason, ExtErr} -> 486 | erlang:error(Reason, [Connection], [{error_info, #{cause => ExtErr}}]); 487 | Ref -> 488 | Ref 489 | end. 490 | 491 | %% @doc EXPERIMENTAL: Stops monitoring previously monitored connection. 492 | %% 493 | %% Does not flush messages that are already in transit. 494 | %% This API is experimental and may change in the future. 495 | -spec demonitor(reference()) -> ok. 496 | demonitor(Ref) -> 497 | case sqlite_monitor_nif(Ref, undefined) of 498 | {error, Reason, ExtErr} -> 499 | erlang:error(Reason, [Ref], [{error_info, #{cause => ExtErr}}]); 500 | ok -> 501 | ok 502 | end. 503 | 504 | %% @doc Interrupts the query running on this connection. 505 | %% 506 | %% WARNING: this function is unsafe to use concurrently with 507 | %% `close/1' on the same connection. 508 | -spec interrupt(connection()) -> ok. 509 | interrupt(Connection) -> 510 | sqlite_interrupt_nif(Connection). 511 | 512 | %% @doc Returns SQLite system information (memory usage, page cache usage, allocation statistics) 513 | -spec system_info() -> system_info(). 514 | system_info() -> 515 | sqlite_system_info_nif(). 516 | 517 | %% @doc Usually returns the ROWID of the most recent successful INSERT 518 | %% into a rowid table or virtual table on database connection. 519 | %% 520 | %% See `sqlite3_last_insert_rowid' in the SQLite reference. 521 | -spec get_last_insert_rowid(connection()) -> integer(). 522 | get_last_insert_rowid(Connection) -> 523 | sqlite_get_last_insert_rowid_nif(Connection). 524 | 525 | %% @doc Backs up the main database of the source to the destination. 526 | -spec backup(Source :: connection(), Destination :: connection()) -> ok. 527 | backup(Source, Destination) -> 528 | backup(Source, Destination, #{from => <<"main">>, to => <<"main">>}). 529 | 530 | %% @doc Runs a backup with the options specified 531 | %% 532 | %% Returns `ok' if the backup was successful, or `stop' 533 | %% if the process was requested to stop. 534 | -spec backup(Source :: connection(), Destination :: connection(), Options :: backup_options()) -> ok | stop. 535 | backup(Source, Destination, Options) -> 536 | SrcDb = maps:get(from, Options, <<"main">>), 537 | DstDb = maps:get(to, Options, SrcDb), 538 | Step = maps:get(step, Options, 64), 539 | case sqlite_backup_init_nif(Source, SrcDb, Destination, DstDb) of 540 | {error, Reason, ExtErr} -> 541 | erlang:error(Reason, [Source, Destination, Options], [{error_info, #{cause => ExtErr}}]); 542 | Ref -> 543 | try 544 | do_backup(Ref, Step, maps:get(progress, Options, undefined)) 545 | after 546 | sqlite_backup_finish_nif(Ref) 547 | end 548 | end. 549 | 550 | %%------------------------------------------------------------------- 551 | %% Internal implementation 552 | 553 | do_backup(Ref, Step, Progress) -> 554 | case sqlite_backup_step_nif(Ref, Step) of 555 | {_Remaining, _Total} when Progress =:= undefined -> 556 | do_backup(Ref, Step, Progress); 557 | {Remaining, Total} -> 558 | case Progress(Remaining, Total) of 559 | ok -> 560 | do_backup(Ref, Step, Progress); 561 | {ok, NewStep} -> 562 | do_backup(Ref, NewStep, Progress); 563 | stop -> 564 | stop 565 | end; 566 | ok -> 567 | ok; 568 | {error, Reason, ExtErr} -> 569 | erlang:error(Reason, [Ref, Step], [{error_info, #{cause => ExtErr}}]) 570 | end. 571 | 572 | %% @doc Formats exception according to EEP-54. 573 | %% 574 | %% Used internally by the shell exception handler. 575 | %% Note that NIFs are not yet supported, and therefore exceptions are 576 | %% thrown by Erlang code. Use `erl_error' module to provide human-readable 577 | %% exception explanations. 578 | format_error(_Reason, [{_M, _F, _As, Info} | _]) -> 579 | #{cause := Cause} = proplists:get_value(error_info, Info, #{}), 580 | Cause. 581 | 582 | 583 | %%------------------------------------------------------------------- 584 | %% NIF stubs 585 | 586 | sqlite_open_nif(_FileName, _Options) -> 587 | ?nif_stub. 588 | 589 | sqlite_close_nif(_Connection) -> 590 | ?nif_stub. 591 | 592 | sqlite_status_nif(_Connection) -> 593 | ?nif_stub. 594 | 595 | sqlite_query_nif(_Connection, _Query, _Parameters) -> 596 | ?nif_stub. 597 | 598 | sqlite_prepare_nif(_Connection, _Query, _Options) -> 599 | ?nif_stub. 600 | 601 | sqlite_bind_nif(_Prepared, _Parameters) -> 602 | ?nif_stub. 603 | 604 | sqlite_step_nif(_Prepared, _Steps) -> 605 | ?nif_stub. 606 | 607 | sqlite_reset_nif(_Prepared) -> 608 | ?nif_stub. 609 | 610 | sqlite_clear_nif(_Prepared) -> 611 | ?nif_stub. 612 | 613 | sqlite_finish_nif(_Prepared) -> 614 | ?nif_stub. 615 | 616 | sqlite_info_nif(_Prepared) -> 617 | ?nif_stub. 618 | 619 | sqlite_describe_nif(_Prepared) -> 620 | ?nif_stub. 621 | 622 | sqlite_execute_nif(_Prepared, _Parameters) -> 623 | ?nif_stub. 624 | 625 | sqlite_monitor_nif(_Connection, _Pid) -> 626 | ?nif_stub. 627 | 628 | sqlite_interrupt_nif(_Connection) -> 629 | ?nif_stub. 630 | 631 | sqlite_system_info_nif() -> 632 | ?nif_stub. 633 | 634 | sqlite_get_last_insert_rowid_nif(_Connection) -> 635 | ?nif_stub. 636 | 637 | sqlite_backup_init_nif(_Src, _SrcDb, _Dst, _DstDb) -> 638 | ?nif_stub. 639 | 640 | sqlite_backup_step_nif(_Backup, _Pages) -> 641 | ?nif_stub. 642 | 643 | sqlite_backup_finish_nif(_Backup) -> 644 | ?nif_stub. 645 | 646 | sqlite_dirty_close_nif() -> 647 | ?nif_stub. 648 | 649 | sqlite_dirty_backup_finish_nif() -> 650 | ?nif_stub. 651 | 652 | %%------------------------------------------------------------------- 653 | %% Internal implementation 654 | 655 | %% The delayed deallocation process is needed when all connection 656 | %% references were Garbage Collected, but the connection was not 657 | %% closed. It is unsafe to close the connection in the GC callback 658 | %% code, because it could lock the scheduler for an arbitrary amount 659 | %% of time, and may lock the entire VM as it relies on schedulers being 660 | %% responsive. 661 | %% So the trick is to have a process that will collect such orphan 662 | %% connections and run delayed deallocation in a dirty I/O scheduler. 663 | %% There is no other purpose of this process. 664 | %% If the module gets purged, this process is stopped forcibly, 665 | %% causing NIF to fall back to closing connection in the GC callback, 666 | %% printing a warning to stderr. 667 | delayed_dealloc() -> 668 | receive 669 | undefined -> 670 | sqlite_dirty_close_nif(), 671 | delayed_dealloc(); 672 | general -> 673 | sqlite_dirty_backup_finish_nif(), 674 | delayed_dealloc() 675 | end. 676 | 677 | init() -> 678 | Purger = erlang:spawn(fun delayed_dealloc/0), 679 | NifAlloc = application:get_env(sqlite, enif_alloc, true), 680 | Priv = code:priv_dir(sqlite), 681 | SoFile = filename:join(Priv, ?MODULE_STRING), 682 | case erlang:load_nif(SoFile, {Purger, NifAlloc}) of 683 | ok -> 684 | ok; 685 | {error, {load, "Library load-call unsuccessful (-1)."}} -> 686 | {error, {load, "Failed to configure memory allocation functions"}}; 687 | {error, {load, "Library load-call unsuccessful (-2)."}} -> 688 | {error, {load, "Linked sqlite version does not support multithreading"}}; 689 | {error, {load, "Library load-call unsuccessful (-3)."}} -> 690 | {error, {load, "Failed to initialize sqlite"}}; 691 | {error, {load, "Library load-call unsuccessful (-4)."}} -> 692 | {error, {load, "Out of memory creating resources"}}; 693 | {error, {load, "Library load-call unsuccessful (-6)."}} -> 694 | {error, {load, "Invalid NIF load_info passed, check if delayed dealloc process is alive"}}; 695 | Other -> 696 | Other 697 | end. 698 | -------------------------------------------------------------------------------- /test/sqlite_SUITE.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @copyright (C) 2023 Maxim Fedorov 3 | %%% Tests for SQLite Connection API 4 | -module(sqlite_SUITE). 5 | -author("maximfca@gmail.com"). 6 | 7 | %% Common Test API 8 | -export([suite/0, all/0, init_per_testcase/2, end_per_testcase/2]). 9 | 10 | %% Test cases 11 | -export([basic/1, 12 | bind_step/0, bind_step/1, 13 | shared/0, shared/1, 14 | shared_cache/0, shared_cache/1, 15 | interrupt/0, interrupt/1, 16 | crash/1, types/1, close/1, status/1, 17 | race_close_prepare/0, race_close_prepare/1, 18 | enif_alloc/0, enif_alloc/1, 19 | malloc/0, malloc/1, 20 | backup/1, backup_sanity/0, backup_sanity/1, 21 | named_parameters/0, named_parameters/1, 22 | prepared/1, errors/1, monitor/1, 23 | concurrency/0, concurrency/1, 24 | delayed_dealloc_kill/1]). 25 | 26 | -behaviour(ct_suite). 27 | 28 | -include_lib("stdlib/include/assert.hrl"). 29 | 30 | suite() -> 31 | [{timetrap, {seconds, 10000}}]. 32 | 33 | init_per_testcase(TC, Config) when TC =:= malloc -> 34 | ensure_unload(sqlite), 35 | Config; 36 | init_per_testcase(_TC, Config) -> 37 | Config. 38 | 39 | end_per_testcase(TC, Config) when TC =:= malloc; TC =:= delayed_dealloc_kill -> 40 | application:unset_env(sqlite, enif_alloc), 41 | ensure_unload(sqlite), 42 | Config; 43 | end_per_testcase(_TC, Config) -> 44 | Config. 45 | 46 | all() -> 47 | [basic, bind_step, shared, shared_cache, interrupt, crash, types, close, status, 48 | enif_alloc, prepared, errors, monitor, concurrency, race_close_prepare, 49 | delayed_dealloc_kill, malloc, backup, backup_sanity]. 50 | 51 | %%------------------------------------------------------------------- 52 | %% Convenience functions 53 | 54 | ensure_unload(Mod) -> 55 | erlang:module_loaded(Mod) andalso 56 | begin 57 | [garbage_collect(Pid) || Pid <- processes()], %% must GC all resource types of the library 58 | true = code:delete(Mod), 59 | code:purge(Mod) 60 | end, 61 | ?assertNot(erlang:module_loaded(Mod)). 62 | 63 | create_blob_db(BlobCount) -> 64 | Src = sqlite:open("", #{flags => [memory]}), 65 | sqlite:query(Src, "CREATE TABLE large (blob BLOB)"), 66 | [] = sqlite:query(Src, "BEGIN;"), 67 | [[] = sqlite:query(Src, "INSERT INTO large VALUES(randomblob(4072));") || _ <- lists:seq(1, BlobCount)], 68 | [] = sqlite:query(Src, "COMMIT;"), 69 | Src. 70 | 71 | start_backup() -> 72 | Src = create_blob_db(8), 73 | Dst = sqlite:open("", #{flags => [memory]}), 74 | %% start the backup in a separate process, to keep this one for tests 75 | Control = self(), 76 | Backup = spawn_link( 77 | fun () -> 78 | try 79 | Return = sqlite:backup(Src, Dst, #{step => 1, progress => 80 | fun (_Remain, _Total) -> 81 | Control ! in_backup, 82 | receive {continue, Ret} -> Ret end 83 | end}), 84 | Control ! {return, Return} 85 | catch 86 | Class:Reason:Stack -> 87 | Control ! {exception, Class, Reason, Stack} 88 | end 89 | end), 90 | receive in_backup -> ok end, 91 | {Src, Dst, Backup}. 92 | 93 | continue_backup(Backup) -> 94 | Backup ! {continue, ok}, 95 | receive Result -> Result end. 96 | 97 | %% extra assertion macro that verifies extended error information 98 | -define(assertExtended(Class, Term, ErrorInfo, Expr), 99 | begin 100 | ((fun () -> 101 | try (Expr) of 102 | X__V -> 103 | erlang:error({assertExtended, [{module, ?MODULE}, {line, ?LINE}, 104 | {expression, (??Expr)}, {pattern, "{ "++(??Class)++" , "++ (??Term) ++" , [...] }"}, 105 | {unexpected_success, X__V}]}) 106 | catch 107 | Class:Term:X__S -> 108 | {_, _, _, X__E} = hd(X__S), 109 | case lists:keyfind(error_info, 1, X__E) of 110 | {error_info, X__EWI} when X__EWI =:= ErrorInfo -> 111 | ok; 112 | {error_info, X__EWI} -> 113 | erlang:error({assertExtended, [{module, ?MODULE}, {line, ?LINE}, 114 | {expression, (??Expr)}, {pattern, "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, 115 | {unexpected_extended_error, {Class, Term, X__EWI}}]}); 116 | false -> 117 | erlang:error({assertExtended, [{module, ?MODULE}, {line, ?LINE}, 118 | {expression, (??Expr)}, {pattern, "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, 119 | {missing_extended_error, {Class, Term, X__E}}]}) 120 | end; 121 | X__C:X__T:X__S -> 122 | erlang:error({assertExtended, [{module, ?MODULE}, {line, ?LINE}, 123 | {expression, (??Expr)}, {pattern, "{ "++(??Class)++" , "++(??Term) ++" , [...] }"}, 124 | {unexpected_exception, {X__C, X__T, X__S}}]}) 125 | end 126 | end)()) 127 | end). 128 | 129 | %%------------------------------------------------------------------- 130 | %% TEST CASES 131 | 132 | basic(Config) when is_list(Config) -> 133 | Conn = sqlite:open("", #{flags => [memory]}), 134 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT"), 135 | %% create a table with key -> value columns 136 | %% insert 1 => str 137 | [] = sqlite:query(Conn, "INSERT INTO kv (key, val) VALUES (1, 'str')"), 138 | Unicode = "юникод", 139 | [] = sqlite:query(Conn, "INSERT INTO kv (key, val) VALUES (?1, ?2)", [2, unicode:characters_to_binary(Unicode)]), 140 | ?assertEqual(2, sqlite:get_last_insert_rowid(Conn)), 141 | %% select 142 | [{1, <<"str">>}, {2, Encoded}] = sqlite:query(Conn, "SELECT * from kv ORDER BY key"), 143 | ?assertEqual(Unicode, unicode:characters_to_list(Encoded)), 144 | [] = sqlite:query(Conn, "SELECT * from kv WHERE val = 'h'"). 145 | 146 | 147 | shared() -> 148 | [{doc, "Test file database shared between two connections"}]. 149 | 150 | shared(Config) when is_list(Config) -> 151 | File = filename:join(proplists:get_value(priv_dir, Config), ?FUNCTION_NAME), 152 | Count = 4, 153 | Conns = [{Seq, sqlite:open(File, #{mode => read_write_create})} || Seq <- lists:seq(1, Count)], 154 | {1, C1} = hd(Conns), 155 | [] = sqlite:query(C1, "CREATE TABLE kv (key INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER) STRICT"), 156 | [sqlite:query(Conn, "INSERT INTO kv (val) VALUES (?1)", [Seq]) || {Seq, Conn} <- Conns], 157 | Sel = [sqlite:query(Conn, "SELECT val FROM kv ORDER BY val") || {_, Conn} <- Conns], 158 | %% must be the same 159 | [Sorted] = lists:usort(Sel), 160 | ?assertEqual([{S} || S <- lists:seq(1, Count)], Sorted). 161 | 162 | shared_cache() -> 163 | [{doc, "Tests different open modes"}]. 164 | 165 | shared_cache(Config) when is_list(Config) -> 166 | Db1 = sqlite:open("file::mem:", #{flags => [shared, memory, uri]}), 167 | Db2 = sqlite:open("file::mem:", #{flags => [shared, memory, uri]}), 168 | [] = sqlite:query(Db1, "CREATE TABLE kv (key INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)"), 169 | [] = sqlite:query(Db1, "INSERT INTO kv (val) VALUES (?1)", [1]), 170 | ?assertEqual([{1}], sqlite:query(Db2, "SELECT val FROM kv ORDER BY val")), 171 | sqlite:close(Db1), sqlite:close(Db2). 172 | 173 | types(Config) when is_list(Config) -> 174 | Conn = sqlite:open("", #{flags => [memory]}), 175 | [] = sqlite:query(Conn, "CREATE TABLE types (t1 TEXT, b1 BLOB, i1 INTEGER, f1 REAL) STRICT"), 176 | %% TEXT storage class 177 | PreparedText = sqlite:prepare(Conn, "INSERT INTO types (t1) VALUES (?1)"), 178 | [] = sqlite:execute(PreparedText, [<<"TXTBIN">>]), %% accept a binary 179 | [] = sqlite:execute(PreparedText, ["TXTSTR"]), %% accept a string 180 | [] = sqlite:execute(PreparedText, [["T", ["XT"], <<"IO">>]]), %% accept an iolist 181 | %% get that back as a binary 182 | [{<<"TXTBIN">>}, {<<"TXTIO">>}, {<<"TXTSTR">>}] = 183 | sqlite:query(Conn, "SELECT t1 FROM types WHERE t1 IS NOT NULL ORDER BY t1"), 184 | %% test a wrapper turning binaries back to string 185 | %% 186 | %% BLOB storage class 187 | [] = sqlite:query(Conn, "INSERT INTO types (b1) VALUES (?1)", [{blob, <<1,2,3,4,5,6>>}]), 188 | [{<<1,2,3,4,5,6>>}] = sqlite:query(Conn, "SELECT b1 FROM types WHERE b1 IS NOT NULL"), 189 | %% INTEGER storage class 190 | PreparedInt = sqlite:prepare(Conn, "INSERT INTO types (i1) VALUES ($1)"), 191 | [] = sqlite:execute(PreparedInt, [576460752303423489]), %% bigint but fits 192 | [] = sqlite:execute(PreparedInt, [-576460752303423490]), %% bigint but fits 193 | [] = sqlite:execute(PreparedInt, [-576460752303423489]), %% small int 194 | [] = sqlite:execute(PreparedInt, [576460752303423488]), %% small int 195 | [{-576460752303423490}, {-576460752303423489}, {576460752303423488}, {576460752303423489}] = 196 | sqlite:query(Conn, "SELECT i1 FROM types WHERE i1 IS NOT NULL ORDER BY i1"), 197 | ?assertExtended(error, badarg, 198 | #{cause => #{2 => <<"bignum not supported">>, position => 1, general => <<"failed to bind parameter">>}}, 199 | sqlite:execute(PreparedInt, [18446744073709551615])), 200 | %% type error takes precedence over invalid column error 201 | ?assertExtended(error, badarg, 202 | #{cause => #{2 => <<"bignum not supported">>, position => 2, general => <<"failed to bind parameter">>}}, 203 | sqlite:execute(PreparedInt, #{2 => 18446744073709551615})), 204 | %% FLOAT storage class 205 | PreparedFloat = sqlite:prepare(Conn, "INSERT INTO types (f1) VALUES ($1)"), 206 | Pi = math:pi(), 207 | NegativePi = -Pi, 208 | [] = sqlite:execute(PreparedFloat, [Pi]), 209 | [] = sqlite:execute(PreparedFloat, [NegativePi]), 210 | [{NegativePi}, {Pi}] = sqlite:query(Conn, "SELECT f1 FROM types WHERE f1 IS NOT NULL ORDER BY f1"), 211 | %% null (undefined) 212 | %% atom 213 | Unsupported = #{cause => #{2 => <<"unsupported type">>, position => 1, general => <<"failed to bind parameter">>}}, 214 | %% reference/fun/pid/port/THING 215 | ?assertExtended(error, badarg, Unsupported, sqlite:execute(PreparedInt, [self()])), 216 | %% tuple/record 217 | ?assertExtended(error, badarg, Unsupported, sqlite:execute(PreparedInt, [{}])), 218 | %% map (JSON?) 219 | ?assertExtended(error, badarg, Unsupported, sqlite:execute(PreparedInt, [#{}])), 220 | %% list is a string, but not a list of large numbers 221 | ?assertExtended(error, badarg, Unsupported, sqlite:execute(PreparedInt, [[12345]])). 222 | 223 | status(Config) when is_list(Config) -> 224 | Conn = sqlite:open("", #{flags => [memory]}), 225 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT"), 226 | Prepared = sqlite:prepare(Conn, "INSERT INTO kv (key, val) VALUES (1, ?1)"), 227 | %% sanity check, that fields are present, but not beyond that. Should are real assertions here 228 | #{deferred_fks := 0, statement := _, lookaside_memory := Lookaside, 229 | pager_cache_memory := Cache} = sqlite:status(Conn), 230 | #{used := Used, shared := _Shared, hit := Hit, miss := _Miss, write := _Write, spill := _Spill} = Cache, 231 | #{used := _, max := _Max, hit := _, miss_size := _, miss_full := _} = Lookaside, 232 | ?assert(Used > 0, {used, Used}), 233 | ?assert(Hit > 0, {hit, Hit}), 234 | %% prepared statement stats 235 | #{fullscan_step := 0, sort := 0, autoindex := 0, vm_step := 0, reprepare := 0, 236 | run := 0, memory_used := _} = sqlite:info(Prepared), 237 | %% system stats, skip page_cache check for not all versions have it 238 | #{memory_used := {_, _}, page_cache := {_, _, _, _, _}, malloc := {_, _, _}, 239 | version := _Vsn} = sqlite:system_info(). 240 | 241 | enif_alloc() -> 242 | [{doc, "Tests that enif_alloc is used for SQLite allocations"}]. 243 | 244 | enif_alloc(Config) when is_list(Config) -> 245 | Conn = sqlite:open("", #{flags => [memory]}), 246 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT"), 247 | Bin = iolist_to_binary([<<"1234567890ABCDEF">> || _ <- lists:seq(1, 128)]), 248 | Repeats = 16, 249 | BinSize = byte_size(Bin) * Repeats, 250 | Statement = sqlite:prepare(Conn, "INSERT INTO kv (key, val) VALUES (?1, ?2)"), 251 | %% test that erlang:memory() reflects changes when large allocations are done 252 | Before = erlang:memory(system), 253 | [[] = sqlite:execute(Statement, [Seq, Bin]) || Seq <- lists:seq(1, Repeats)], 254 | After = erlang:memory(system), 255 | Diff = After - Before, 256 | %% system memory should grow ~2x of the binding size 257 | ?assert(Diff > BinSize andalso Diff < (3 * BinSize), {difference, Diff, byte_size, BinSize}), 258 | %% finalise the statement - release some sqlite memory (but keep some of NIF resources) 259 | ok = sqlite:finish(Statement), 260 | Diff2 = erlang:memory(system) - After, 261 | %% just a bit of slack 262 | ?assert(Diff2 < 0, {not_final, Diff2}). 263 | 264 | malloc() -> 265 | [{doc, "Tests that malloc is used for SQLite allocations"}]. 266 | 267 | malloc(Config) when is_list(Config) -> 268 | application:set_env(sqlite, enif_alloc, false), 269 | {module, sqlite} = code:load_file(sqlite), 270 | MemBefore = erlang:memory(system), 271 | Preps = alloc_some(), 272 | MemDiff = erlang:memory(system) - MemBefore, 273 | KeepRefs = tuple_size(Preps), %% only to avoid GC-ing "Preps" 274 | %% 200 prepared statements take > 100k, so if we test for 64k... 275 | ?assert(MemDiff < 64 * 1024, {unexpected_allocation, MemDiff}), 276 | ?assert(KeepRefs =:= 2). 277 | 278 | prepared(Config) when is_list(Config) -> 279 | Conn = sqlite:open("", #{flags => [memory]}), 280 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT"), 281 | PreparedInsert = sqlite:prepare(Conn, "INSERT INTO kv (key, val) VALUES (?1, ?2)"), 282 | [] = sqlite:execute(PreparedInsert, [1, "string"]), 283 | PreparedSel = sqlite:prepare(Conn, "SELECT * FROM kv WHERE key = ?1"), 284 | [{1, <<"string">>}] = sqlite:execute(PreparedSel, [1]), 285 | %% test statement into 286 | ?assertEqual([{<<"key">>,<<"INTEGER">>},{<<"val">>,<<"TEXT">>}], sqlite:describe(PreparedSel)). 287 | 288 | named_parameters() -> 289 | [{doc, "Tests that named parameters map is accepted"}]. 290 | 291 | named_parameters(Config) when is_list(Config) -> 292 | Conn = sqlite:open("", #{flags => [memory]}), 293 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER, val TEXT) STRICT"), 294 | Prep = sqlite:prepare(Conn, "INSERT INTO kv (key, val) VALUES (:key, :value)"), 295 | %% test extended errors 296 | ?assertExtended(error, badarg, #{cause => #{2 => <<"parameter not found">>, general => <<":wrong">>}}, 297 | sqlite:bind(Prep, #{wrong => 1})), 298 | ?assertExtended(error, {sqlite_error,25}, #{cause => #{2 => <<"column index out of range">>, position => 4, 299 | general => <<"failed to bind parameter">>}}, sqlite:bind(Prep, #{4 => "err"})), 300 | %% 301 | [] = sqlite:execute(Prep, #{key => 1, value => "text"}), 302 | [] = sqlite:execute(Prep, #{key => 2, value => "text2"}), 303 | [{1, <<"text">>}] = sqlite:query(Conn, "SELECT * FROM kv WHERE key = :col", #{<<"col">> => 1}), 304 | %% now try partial bindings - with some bits left from previous run 305 | ok = sqlite:reset(Prep), 306 | ok = sqlite:clear(Prep), 307 | ok = sqlite:bind(Prep, #{value => "two"}), %% key is unbound 308 | done = sqlite:step(Prep), 309 | ok = sqlite:reset(Prep), 310 | %% try using positional syntax for map binding 311 | ok = sqlite:bind(Prep, #{1 => 123}), %% now both key and value are bound 312 | done = sqlite:step(Prep), 313 | %% ensure we have all expected rows 314 | ?assertEqual([{1, <<"text">>}, {2, <<"text2">>}, {undefined, <<"two">>}, {123, <<"two">>}], 315 | sqlite:query(Conn, "SELECT * from kv")). 316 | 317 | bind_step() -> 318 | [{doc, "Tests bind/2 and step/1,2"}]. 319 | 320 | bind_step(Config) when is_list(Config) -> 321 | Conn = sqlite:open("", #{flags => [memory]}), 322 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT) STRICT"), 323 | PreparedInsert = sqlite:prepare(Conn, "INSERT INTO kv (val) VALUES (?1)"), 324 | ok = sqlite:bind(PreparedInsert, ["str"]), 325 | done = sqlite:step(PreparedInsert), 326 | done = sqlite:step(PreparedInsert), %% sqlite allows that, and inserts another row 327 | %% insert & select multiple rows 328 | PreparedMultiInsert = sqlite:prepare(Conn, "INSERT INTO kv (val) VALUES (?1), (?), (?)"), 329 | ok = sqlite:bind(PreparedMultiInsert, ["2", "3", "4"]), 330 | {done, []} = sqlite:step(PreparedMultiInsert, 10), 331 | %% reset and step again 332 | ok = sqlite:reset(PreparedMultiInsert), 333 | done = sqlite:step(PreparedMultiInsert), %% that's one statement, after all 334 | %% clear bindings 335 | ok = sqlite:clear(PreparedMultiInsert), 336 | %% do not bind anything, but run the statement, which inserts 3 more rows with undefined value 337 | done = sqlite:step(PreparedMultiInsert), 338 | %% select 339 | PreparedSelect = sqlite:prepare(Conn, "SELECT * FROM kv ORDER BY key"), 340 | [{1, <<"str">>}, {2, <<"str">>}, {3, <<"2">>}, {4, <<"3">>}] = sqlite:step(PreparedSelect, 4), 341 | [{5, <<"4">>}, {6, <<"2">>}, {7, <<"3">>}, {8, <<"4">>}] = sqlite:step(PreparedSelect, 4), 342 | {done, [{9, undefined}, {10, undefined}, {11, undefined}]} = sqlite:step(PreparedSelect, 10), 343 | sqlite:close(Conn). 344 | 345 | monitor(Config) when is_list(Config) -> 346 | Conn = sqlite:open("", #{flags => [memory]}), 347 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT"), 348 | Ref = sqlite:monitor(Conn), 349 | %% verify monitoring works 350 | [] = sqlite:query(Conn, "INSERT INTO kv (key, val) VALUES (1, 'str')"), 351 | receive 352 | {Ref, insert, <<"main">>, <<"kv">>, _RowId} -> ok 353 | after 10 -> ?assert(false, "monitoring did not work, expected message is not received") 354 | end, 355 | %% test demonitoring 356 | sqlite:demonitor(Ref), 357 | [] = sqlite:query(Conn, "INSERT INTO kv (key, val) VALUES (2, 'str')"), 358 | receive Unexpected2 -> ?assert(false, {unexpected, Unexpected2}) 359 | after 10 -> ok 360 | end. 361 | 362 | close(Config) when is_list(Config) -> 363 | Conn = sqlite:open("", #{flags => [memory]}), 364 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT"), 365 | ok = sqlite:close(Conn), 366 | ?assertExtended(error, badarg, #{cause => #{1 => <<"connection closed">>}}, 367 | sqlite:prepare(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT")), 368 | ?assertExtended(error, badarg, #{cause => #{1 => <<"connection already closed">>}}, sqlite:close(Conn)). 369 | 370 | 371 | interrupt() -> 372 | [{doc, "Tests that interrupt works, and hopefully not crashing the VM"}]. 373 | 374 | interrupt(Config) when is_list(Config) -> 375 | Conn = sqlite:open("", #{flags => [memory]}), 376 | [] = sqlite:query(Conn, "CREATE TABLE any (val NOT NULL)"), 377 | %% now do the test 378 | Control = self(), 379 | _Long = spawn_link( 380 | fun () -> 381 | Result = 382 | try 383 | sqlite:query(Conn, "WITH RECURSIVE 384 | for(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM for WHERE i < 10000000) 385 | INSERT INTO any SELECT i FROM for") 386 | catch 387 | error:Reason -> 388 | Reason 389 | end, 390 | Control ! {long, Result, os:system_time(millisecond)} 391 | end), 392 | timer:sleep(10), %% enough to get the Long process running 393 | %% close will hang until the interrupt comes, so spawn an extra process for it 394 | _Close = spawn_link( 395 | fun () -> 396 | sqlite:close(Conn), 397 | Control ! {close, os:system_time(millisecond)} 398 | end), 399 | %% interrupt in the test process after 100 ms 400 | timer:sleep(100), 401 | ok = sqlite:interrupt(Conn), 402 | %% get the interrupted & closing time (the latter must be greater) 403 | InterruptedAt = receive {long, Result, IntTime} -> ?assertEqual({sqlite_error, 9}, Result), IntTime end, 404 | %% get the closing time 405 | ClosedAt = receive {close, CloseTime} -> CloseTime end, 406 | ?assert(ClosedAt >= InterruptedAt, {interrupted, InterruptedAt, closed, ClosedAt, diff, ClosedAt - InterruptedAt}). 407 | 408 | crash(Config) when is_list(Config) -> 409 | %% ensures that even after GC prepared statement is working 410 | Conn = sqlite:open("", #{flags => [memory]}), 411 | Prepared = sqlite:prepare(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val TEXT) STRICT"), 412 | sqlite:close(Conn), 413 | erlang:garbage_collect(), 414 | attempt_crash(Prepared). 415 | 416 | attempt_crash(Prepared) -> 417 | ?assertExtended(error, badarg, #{cause => #{1 => <<"connection closed">>}}, 418 | sqlite:execute(Prepared, [<<"1">>])). 419 | 420 | delayed_dealloc_kill(Config) when is_list(Config) -> 421 | %% delete && purge the sqlite NIF module 422 | erlang:module_loaded(sqlite) andalso begin 423 | true = code:delete(sqlite), 424 | %% ensure that code purge kills the delayed_dealloc process 425 | Begin = processes(), 426 | code:purge(sqlite), 427 | WithoutPurger = Begin -- processes(), 428 | ?assert(length(WithoutPurger) =:= 1, WithoutPurger) 429 | end, 430 | %% ensure it's not loaded 431 | ?assertNot(erlang:module_loaded(sqlite)), 432 | Before = processes(), 433 | %% load implicitly, see the purger process there 434 | {module, sqlite} = code:load_file(sqlite), 435 | [Purger] = processes() -- Before, 436 | %% kill the purger 437 | MRef = erlang:monitor(process, Purger), 438 | erlang:exit(Purger, kill), 439 | receive {'DOWN', MRef, process, Purger, _Reason} -> ok end, 440 | %% trigger delayed dealloc 441 | MemBefore = erlang:memory(system), 442 | alloc_some(), 443 | erlang:garbage_collect(), %% GC references to prepared statements and connection 444 | timer:sleep(500), %% deallocation is delayed, maybe think of better way to ensure cleanup? 445 | %% TODO: ensure the error is logged, but how? redirecting stderr? 446 | %% ensure that actual resource was freed 447 | MemDiff = erlang:memory(system) - MemBefore, 448 | ?assert(MemDiff < 10 * 1024, {memory_leak, MemDiff}). 449 | 450 | alloc_some() -> 451 | Conn = sqlite:open("", #{flags => [memory]}), 452 | Preps = [sqlite:prepare(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val STRING)") || _ <- lists:seq(1, 200)], 453 | {Conn, Preps}. 454 | 455 | backup(Config) when is_list(Config) -> 456 | Src = create_blob_db(8), 457 | %% do the basic backup 458 | Dst = sqlite:open("", #{flags => [memory]}), 459 | ok = sqlite:backup(Src, Dst), 460 | ?assertEqual([{8}], sqlite:query(Dst, "SELECT COUNT(*) FROM large")), 461 | %% do the fancier backup, with progress tracking, step setting and exception 462 | %% aborting the backup 463 | Dst2 = sqlite:open("", #{flags => [memory]}), 464 | try sqlite:backup(Src, Dst2, #{from => "main", step => 2, progress => 465 | fun (Rem, _Total) when Rem > 6 -> ok; 466 | (Rem, _Total) when Rem > 4 -> {ok, 1}; 467 | (Rem, Total) -> throw({expected, Rem, Total}) 468 | end}) 469 | catch 470 | throw:{expected, Remaining, Total} -> 471 | ?assertEqual(4, Remaining), 472 | ?assert(Total > 0 andalso Total > Remaining) 473 | end. 474 | 475 | backup_sanity() -> 476 | [{doc, ""}]. 477 | 478 | backup_sanity(Config) when is_list(Config) -> 479 | {Src0, Dst0, Backup0} = start_backup(), 480 | sqlite:close(Src0), %% must force-close the backup (causing backup to abort) 481 | {exception, error, badarg, [{sqlite, _, _, Ext} | _]} = continue_backup(Backup0), 482 | ?assertEqual(#{1 => <<"backup aborted (connection closed)">>}, 483 | maps:get(cause, proplists:get_value(error_info, Ext))), 484 | sqlite:close(Dst0), 485 | {Src1, Dst1, Backup1} = start_backup(), 486 | %% ensure that destination is locked when backup is in progress 487 | ?assertExtended(error, badarg, #{cause => #{1 => <<"connection busy (backup destination)">>}}, 488 | sqlite:query(Dst1, "SELECT * from large")), 489 | %% check backup destination interaction with close/1 490 | sqlite:close(Dst1), 491 | sqlite:close(Src1), 492 | {exception, error, badarg, [{sqlite, _, _, Ext} | _]} = continue_backup(Backup1). 493 | 494 | errors(Config) when is_list(Config) -> 495 | %% test EPP-54 extended error information 496 | ?assertExtended(error, badarg, #{cause => #{2 => <<"not a map">>}}, sqlite:open("", 123)), 497 | ?assertExtended(error, badarg, #{cause => #{2 => <<"unsupported mode">>}}, sqlite:open("", #{mode => invalid})), 498 | ?assertExtended(error, {sqlite_error, 14}, #{cause => 499 | #{reason => <<"bad parameter or other API misuse">>, general => <<"unable to open database file">>}}, 500 | sqlite:open("not_exist", #{mode => read_only})), 501 | %% need a statement for testing 502 | Db = sqlite:open("", #{flags => [memory]}), 503 | [] = sqlite:query(Db, "CREATE TABLE kv (key INTEGER PRIMARY KEY, val STRING)"), 504 | %% non-existent table 505 | ?assertExtended(error, {sqlite_error, 1}, 506 | #{cause => #{2 => <<"no such table: non_existing">>, general => <<"SQL logic error">>}}, 507 | sqlite:query(Db, "SELECT * from non_existing")), 508 | %% errors while preparing query 509 | ?assertExtended(error, {sqlite_error, 1}, #{cause => 510 | #{2 => <<"near \"WHERE\": syntax error">>,position => 30, general => <<"SQL logic error">>}}, 511 | sqlite:query(Db, "SELECT * from kv ORDER BY key WHERE key = ?", [1])), 512 | ?assertExtended(error, badarg, #{cause => #{3 => <<"invalid persistent value">>}}, 513 | sqlite:prepare(Db, "SELECT * from kv ORDER BY key", #{persistent => maybe})), 514 | %% extra binding when not expected 515 | ?assertExtended(error, {sqlite_error, 25}, #{cause => 516 | #{3 => <<"column index out of range">>,position => 2, general => <<"column index out of range">>}}, 517 | sqlite:query(Db, "INSERT INTO kv (key) VALUES ($1)", [1, "1", 1])), 518 | %% not enough bindings 519 | ?assertExtended(error, badarg, #{cause => #{2 => <<"not enough parameters">>}}, 520 | sqlite:query(Db, "INSERT INTO kv (key, val) VALUES ($1, $2)", [1])), 521 | sqlite:close(Db). 522 | 523 | concurrency() -> 524 | [{doc, "Tests that concurrently opening, closing, and preparing does not crash"}, 525 | {timetrap, {seconds, 60}}]. 526 | 527 | concurrency(Config) when is_list(Config) -> 528 | Concurrency = erlang:system_info(dirty_io_schedulers), 529 | FileName = filename:join(proplists:get_value(priv_dir, Config), "db.bin"), 530 | %% create a DB with schema 531 | Src = sqlite:open(FileName, #{busy_timeout => 60000}), 532 | [] = sqlite:query(Src, "CREATE TABLE kv (key INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)"), 533 | %% use the connection by many workers 534 | Count = 500, 535 | StartOpt = #{control => self(), filename => FileName}, 536 | Workers = [erlang:spawn_link(fun() -> worker(Count, rand:seed(exrop, {Seq, 0, 0}), Src, StartOpt) end) 537 | || Seq <- lists:seq(1, Concurrency)], 538 | [W ! {workers, Workers} || W <- Workers], 539 | %% wait for all of them to complete (with no error, otherwise the link crashes test runner) 540 | progress(Count * Concurrency, maps:from_list([{W, Count} || W <- Workers])). 541 | 542 | progress(0, WP) -> 543 | Monitors = [{W, erlang:monitor(process, W)} || W <- maps:keys(WP)], 544 | [receive {'DOWN', Mon, process, Pid, Reason} -> Reason end || {Pid, Mon} <- Monitors]; 545 | progress(Total, WP) -> 546 | receive 547 | {progress, Worker, Left} -> 548 | #{Worker := Before} = WP, 549 | ?assert(Left =< Before, {left, Left, before, Before}), 550 | NewTotal = Total - (Before - Left), 551 | (Total div 1000) > (NewTotal div 1000) andalso 552 | io:format(user, " -> ~b", [NewTotal]), 553 | progress(NewTotal, WP#{Worker => Left}) 554 | end. 555 | 556 | worker(Count, Seed, Src, StartOpt) -> 557 | receive 558 | {workers, Workers} -> 559 | worker(Count, Seed, Src, [], StartOpt#{workers => Workers}) 560 | end. 561 | 562 | %% This is a mini-property based test, running random actions 563 | worker(0, _Seed, _Db, _Statements, #{control := Control}) -> 564 | Control ! {progress, self(), 0}; 565 | worker(Count, Seed, Db, Statements, #{control := Control} = StartOpt) -> 566 | {Next, NewSeed} = rand:uniform_s(51, Seed), 567 | Control ! {progress, self(), Count}, 568 | {NewCount, NS, NDb, NST} = 569 | case Next of 570 | 1 -> 571 | try 572 | ok = sqlite:close(Db), 573 | #{filename := FileName, workers := Workers} = StartOpt, 574 | NewDb = sqlite:open(FileName, #{busy_timeout => 60000, mode => read_write}), 575 | %% notify all other workers 576 | [W ! {new_db, NewDb} || W <- Workers, W =/= self()], 577 | {Count - 1, NewSeed, NewDb, []} 578 | catch 579 | error:badarg -> 580 | {Count, Seed, receive_db(), []} 581 | end; 582 | 2 -> 583 | Dst = sqlite:open("", #{flags => [memory]}), 584 | try sqlite:backup(Db, Dst), 585 | sqlite:close(Dst), 586 | {Count - 1, NewSeed, Db, Statements} 587 | catch 588 | error:badarg -> 589 | {Count, Seed, receive_db(), []} 590 | end; 591 | NewStatement when NewStatement < 12 -> 592 | try 593 | Prep = sqlite:prepare(Db, "INSERT INTO kv (val) VALUES (?1)", #{persistent => true}), 594 | {Count - 1, NewSeed, Db, [Prep | Statements]} 595 | catch 596 | error:badarg -> 597 | {Count, Seed, receive_db(), []} 598 | end; 599 | DelStatement when DelStatement < 22, length(Statements) > 0 -> 600 | NewStmts = lists:droplast(Statements), 601 | {Count - 1, NewSeed, Db, NewStmts}; 602 | RunStatement when RunStatement < 52, length(Statements) > 0 -> 603 | try sqlite:execute(hd(Statements), [1]), 604 | {Count - 1, NewSeed, Db, Statements} 605 | catch 606 | error:badarg -> 607 | {Count, Seed, receive_db(), []} 608 | end; 609 | _RunStatement -> 610 | %% skipping a step 611 | {Count, NewSeed, Db, Statements} 612 | end, 613 | worker(NewCount, NS, NDb, NST, StartOpt). 614 | 615 | receive_db() -> 616 | receive 617 | {new_db, NewDb} -> 618 | NewDb 619 | end. 620 | 621 | race_close_prepare() -> 622 | [{doc, "Tests that closing the connection while preparing the statement does not crash or leak"}, 623 | {timetrap, {seconds, 60}}]. 624 | 625 | race_close_prepare(Config) when is_list(Config) -> 626 | do_race(5000). 627 | 628 | do_race(0) -> ok; 629 | do_race(Count) -> 630 | Conn = sqlite:open("", #{flags => [memory]}), 631 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)"), 632 | %% 633 | Preparer = spawn_link(fun() -> do_prepare(1000, Conn) end), 634 | Closer = spawn_link(fun() -> timer:sleep(2), sqlite:close(Conn) end), 635 | %% 636 | Workers = [Preparer, Closer], 637 | Monitors = [{W, erlang:monitor(process, W)} || W <- Workers], 638 | %% wait for all of them to complete (with no error, otherwise the link crashes test runner) 639 | [receive {'DOWN', Mon, process, Pid, Reason} -> Reason end || {Pid, Mon} <- Monitors], 640 | do_race(Count - 1). 641 | 642 | do_prepare(0, _Conn) -> 643 | ?assert(too_quick); 644 | do_prepare(Count, Conn) -> 645 | try sqlite:prepare(Conn, <<"SELECT * FROM kv WHERE key=?1">>), do_prepare(Count - 1, Conn) 646 | catch error:badarg -> ok end. 647 | -------------------------------------------------------------------------------- /test/sqlite_bench_SUITE.erl: -------------------------------------------------------------------------------- 1 | %%%------------------------------------------------------------------- 2 | %%% @copyright (C) 2023 Maxim Fedorov 3 | %%% Benchmarks - use for testing performance 4 | -module(sqlite_bench_SUITE). 5 | -author("maximfca@gmail.com"). 6 | 7 | %% Common Test API 8 | -export([suite/0, all/0]). 9 | 10 | %% Test cases 11 | -export([basic/1, benchmark_prepared/1]). 12 | 13 | -behaviour(ct_suite). 14 | 15 | -include_lib("stdlib/include/assert.hrl"). 16 | 17 | suite() -> 18 | [{timetrap, {seconds, 600}}]. 19 | 20 | all() -> 21 | [basic, benchmark_prepared]. 22 | 23 | %%------------------------------------------------------------------- 24 | %% Convenience functions 25 | 26 | basic(Config) when is_list(Config) -> 27 | ok. 28 | 29 | %% benchmark for prepared statements 30 | -define(QUERY, <<"SELECT * FROM kv WHERE key=?1">>). 31 | benchmark_prepared(Config) when is_list(Config) -> 32 | measure_one(fun bench_query/2, "query"), 33 | measure_one(fun bench_prep/2, "prepare every time"), 34 | measure_one(fun bench_one/2, "prepare once"). 35 | 36 | measure_one(FunTo, Kind) -> 37 | Conn = sqlite:open("", #{flags => [memory]}), 38 | [] = sqlite:query(Conn, "CREATE TABLE kv (key INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)"), 39 | [erlang:garbage_collect(Pid) || Pid <- processes()], %% ensure no garbage in the system 40 | MemBefore = erlang:memory(system), 41 | {TimeUs, _} = timer:tc(fun () -> FunTo(100000, Conn), erlang:garbage_collect() end), 42 | MemDiff = erlang:memory(system) - MemBefore, 43 | sqlite:close(Conn), 44 | erlang:garbage_collect(), 45 | MemZero = erlang:memory(system) - MemBefore, 46 | ct:pal("~s: ~b ms, allocated ~b Kb (~b Kb)", [Kind, TimeUs div 1000, MemDiff div 1024, MemZero div 1024]), 47 | {TimeUs, MemDiff}. 48 | 49 | bench_query(0, _Conn) -> 50 | ok; 51 | bench_query(Count, Conn) -> 52 | sqlite:query(Conn, ?QUERY, [1]), 53 | bench_query(Count - 1, Conn). 54 | 55 | bench_prep(0, _Conn) -> 56 | ok; 57 | bench_prep(Count, Conn) -> 58 | Prep = sqlite:prepare(Conn, ?QUERY), 59 | sqlite:execute(Prep, [1]), 60 | bench_prep(Count - 1, Conn). 61 | 62 | bench_one(Count, Conn) -> 63 | do_bench_one(Count, sqlite:prepare(Conn, ?QUERY)). 64 | 65 | do_bench_one(0, _Prep) -> 66 | ok; 67 | do_bench_one(Count, Prep) -> 68 | sqlite:execute(Prep, [1]), 69 | do_bench_one(Count - 1, Prep). 70 | 71 | --------------------------------------------------------------------------------